Index: projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/cm.c =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/cm.c (revision 331317) +++ projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/cm.c (revision 331318) @@ -1,2553 +1,2964 @@ /* * Copyright (c) 2009-2013, 2016 Chelsio, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #ifdef TCP_OFFLOAD #include #include #include #include #include #include #include #include #include #include +#include #include #include +#include +#include #include #include #include #include #include struct sge_iq; struct rss_header; struct cpl_set_tcb_rpl; #include #include "offload.h" #include "tom/t4_tom.h" #define TOEPCB(so) ((struct toepcb *)(so_sototcpcb((so))->t_toe)) #include "iw_cxgbe.h" #include #include #include #include #include #include static spinlock_t req_lock; static TAILQ_HEAD(c4iw_ep_list, c4iw_ep_common) req_list; static struct work_struct c4iw_task; static struct workqueue_struct *c4iw_taskq; static LIST_HEAD(err_cqe_list); static spinlock_t err_cqe_lock; +static LIST_HEAD(listen_port_list); +static DEFINE_MUTEX(listen_port_mutex); static void process_req(struct work_struct *ctx); static void start_ep_timer(struct c4iw_ep *ep); static int stop_ep_timer(struct c4iw_ep *ep); static int set_tcpinfo(struct c4iw_ep *ep); static void process_timeout(struct c4iw_ep *ep); static void process_err_cqes(void); -static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc); -static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state tostate); -static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state tostate); static void *alloc_ep(int size, gfp_t flags); -static int find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, - __be16 peer_port, u8 tos, struct nhop4_extended *pnh4); static void close_socket(struct socket *so); static int send_mpa_req(struct c4iw_ep *ep); static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen); static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen); static void close_complete_upcall(struct c4iw_ep *ep, int status); static int send_abort(struct c4iw_ep *ep); static void peer_close_upcall(struct c4iw_ep *ep); static void peer_abort_upcall(struct c4iw_ep *ep); static void connect_reply_upcall(struct c4iw_ep *ep, int status); static int connect_request_upcall(struct c4iw_ep *ep); static void established_upcall(struct c4iw_ep *ep); static int process_mpa_reply(struct c4iw_ep *ep); static int process_mpa_request(struct c4iw_ep *ep); static void process_peer_close(struct c4iw_ep *ep); static void process_conn_error(struct c4iw_ep *ep); static void process_close_complete(struct c4iw_ep *ep); static void ep_timeout(unsigned long arg); static void setiwsockopt(struct socket *so); static void init_iwarp_socket(struct socket *so, void *arg); static void uninit_iwarp_socket(struct socket *so); static void process_data(struct c4iw_ep *ep); static void process_connected(struct c4iw_ep *ep); static int c4iw_so_upcall(struct socket *so, void *arg, int waitflag); static void process_socket_event(struct c4iw_ep *ep); static void release_ep_resources(struct c4iw_ep *ep); static int process_terminate(struct c4iw_ep *ep); static int terminate(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m); static int add_ep_to_req_list(struct c4iw_ep *ep, int ep_events); +static struct listen_port_info * +add_ep_to_listenlist(struct c4iw_listen_ep *lep); +static int rem_ep_from_listenlist(struct c4iw_listen_ep *lep); +static struct c4iw_listen_ep * +find_real_listen_ep(struct c4iw_listen_ep *master_lep, struct socket *so); +static int get_ifnet_from_raddr(struct sockaddr_storage *raddr, + struct ifnet **ifp); +static void process_newconn(struct c4iw_listen_ep *master_lep, + struct socket *new_so); #define START_EP_TIMER(ep) \ do { \ CTR3(KTR_IW_CXGBE, "start_ep_timer (%s:%d) ep %p", \ __func__, __LINE__, (ep)); \ start_ep_timer(ep); \ } while (0) #define STOP_EP_TIMER(ep) \ ({ \ CTR3(KTR_IW_CXGBE, "stop_ep_timer (%s:%d) ep %p", \ __func__, __LINE__, (ep)); \ stop_ep_timer(ep); \ }) +#define GET_LOCAL_ADDR(pladdr, so) \ + do { \ + struct sockaddr_storage *__a = NULL; \ + struct inpcb *__inp = sotoinpcb(so); \ + KASSERT(__inp != NULL, \ + ("GET_LOCAL_ADDR(%s):so:%p, inp = NULL", __func__, so)); \ + if (__inp->inp_vflag & INP_IPV4) \ + in_getsockaddr(so, (struct sockaddr **)&__a); \ + else \ + in6_getsockaddr(so, (struct sockaddr **)&__a); \ + *(pladdr) = *__a; \ + free(__a, M_SONAME); \ + } while (0) + +#define GET_REMOTE_ADDR(praddr, so) \ + do { \ + struct sockaddr_storage *__a = NULL; \ + struct inpcb *__inp = sotoinpcb(so); \ + KASSERT(__inp != NULL, \ + ("GET_REMOTE_ADDR(%s):so:%p, inp = NULL", __func__, so)); \ + if (__inp->inp_vflag & INP_IPV4) \ + in_getpeeraddr(so, (struct sockaddr **)&__a); \ + else \ + in6_getpeeraddr(so, (struct sockaddr **)&__a); \ + *(praddr) = *__a; \ + free(__a, M_SONAME); \ + } while (0) + #ifdef KTR static char *states[] = { "idle", "listen", "connecting", "mpa_wait_req", "mpa_req_sent", "mpa_req_rcvd", "mpa_rep_sent", "fpdu_mode", "aborting", "closing", "moribund", "dead", NULL, }; #endif - static void deref_cm_id(struct c4iw_ep_common *epc) { epc->cm_id->rem_ref(epc->cm_id); epc->cm_id = NULL; set_bit(CM_ID_DEREFED, &epc->history); } static void ref_cm_id(struct c4iw_ep_common *epc) { set_bit(CM_ID_REFED, &epc->history); epc->cm_id->add_ref(epc->cm_id); } static void deref_qp(struct c4iw_ep *ep) { c4iw_qp_rem_ref(&ep->com.qp->ibqp); clear_bit(QP_REFERENCED, &ep->com.flags); set_bit(QP_DEREFED, &ep->com.history); } static void ref_qp(struct c4iw_ep *ep) { set_bit(QP_REFERENCED, &ep->com.flags); set_bit(QP_REFED, &ep->com.history); c4iw_qp_add_ref(&ep->com.qp->ibqp); } +/* allocated per TCP port while listening */ +struct listen_port_info { + uint16_t port_num; /* TCP port address */ + struct list_head list; /* belongs to listen_port_list */ + struct list_head lep_list; /* per port lep list */ + uint32_t refcnt; /* number of lep's listening */ +}; +/* + * Following two lists are used to manage INADDR_ANY listeners: + * 1)listen_port_list + * 2)lep_list + * + * Below is the INADDR_ANY listener lists overview on a system with a two port + * adapter: + * |------------------| + * |listen_port_list | + * |------------------| + * | + * | |-----------| |-----------| + * | | port_num:X| | port_num:X| + * |--------------|-list------|-------|-list------|-------.... + * | lep_list----| | lep_list----| + * | refcnt | | | refcnt | | + * | | | | | | + * | | | | | | + * |-----------| | |-----------| | + * | | + * | | + * | | + * | | lep1 lep2 + * | | |----------------| |----------------| + * | |----| listen_ep_list |----| listen_ep_list | + * | |----------------| |----------------| + * | + * | + * | lep1 lep2 + * | |----------------| |----------------| + * |---| listen_ep_list |----| listen_ep_list | + * |----------------| |----------------| + * + * Because of two port adapter, the number of lep's are two(lep1 & lep2) for + * each TCP port number. + * + * Here 'lep1' is always marked as Master lep, because solisten() is always + * called through first lep. + * + */ +static struct listen_port_info * +add_ep_to_listenlist(struct c4iw_listen_ep *lep) +{ + uint16_t port; + struct listen_port_info *port_info = NULL; + struct sockaddr_storage *laddr = &lep->com.local_addr; + + port = (laddr->ss_family == AF_INET) ? + ((struct sockaddr_in *)laddr)->sin_port : + ((struct sockaddr_in6 *)laddr)->sin6_port; + + mutex_lock(&listen_port_mutex); + + list_for_each_entry(port_info, &listen_port_list, list) + if (port_info->port_num == port) + goto found_port; + + port_info = malloc(sizeof(*port_info), M_CXGBE, M_WAITOK); + port_info->port_num = port; + port_info->refcnt = 0; + + list_add_tail(&port_info->list, &listen_port_list); + INIT_LIST_HEAD(&port_info->lep_list); + +found_port: + port_info->refcnt++; + list_add_tail(&lep->listen_ep_list, &port_info->lep_list); + mutex_unlock(&listen_port_mutex); + return port_info; +} + +static int +rem_ep_from_listenlist(struct c4iw_listen_ep *lep) +{ + uint16_t port; + struct listen_port_info *port_info = NULL; + struct sockaddr_storage *laddr = &lep->com.local_addr; + int refcnt = 0; + + port = (laddr->ss_family == AF_INET) ? + ((struct sockaddr_in *)laddr)->sin_port : + ((struct sockaddr_in6 *)laddr)->sin6_port; + + mutex_lock(&listen_port_mutex); + + /* get the port_info structure based on the lep's port address */ + list_for_each_entry(port_info, &listen_port_list, list) { + if (port_info->port_num == port) { + port_info->refcnt--; + refcnt = port_info->refcnt; + /* remove the current lep from the listen list */ + list_del(&lep->listen_ep_list); + if (port_info->refcnt == 0) { + /* Remove this entry from the list as there + * are no more listeners for this port_num. + */ + list_del(&port_info->list); + kfree(port_info); + } + break; + } + } + mutex_unlock(&listen_port_mutex); + return refcnt; +} + +/* + * Find the lep that belongs to the ifnet on which the SYN frame was received. + */ +struct c4iw_listen_ep * +find_real_listen_ep(struct c4iw_listen_ep *master_lep, struct socket *so) +{ + struct adapter *adap = NULL; + struct c4iw_listen_ep *lep = NULL; + struct sockaddr_storage remote = { 0 }; + struct ifnet *new_conn_ifp = NULL; + struct listen_port_info *port_info = NULL; + int err = 0, i = 0, + found_portinfo = 0, found_lep = 0; + uint16_t port; + + /* STEP 1: get 'ifnet' based on socket's remote address */ + GET_REMOTE_ADDR(&remote, so); + + err = get_ifnet_from_raddr(&remote, &new_conn_ifp); + if (err) { + CTR4(KTR_IW_CXGBE, "%s: Failed to get ifnet, sock %p, " + "master_lep %p err %d", + __func__, so, master_lep, err); + return (NULL); + } + + /* STEP 2: Find 'port_info' with listener local port address. */ + port = (master_lep->com.local_addr.ss_family == AF_INET) ? + ((struct sockaddr_in *)&master_lep->com.local_addr)->sin_port : + ((struct sockaddr_in6 *)&master_lep->com.local_addr)->sin6_port; + + + mutex_lock(&listen_port_mutex); + list_for_each_entry(port_info, &listen_port_list, list) + if (port_info->port_num == port) { + found_portinfo =1; + break; + } + if (!found_portinfo) + goto out; + + /* STEP 3: Traverse through list of lep's that are bound to the current + * TCP port address and find the lep that belongs to the ifnet on which + * the SYN frame was received. + */ + list_for_each_entry(lep, &port_info->lep_list, listen_ep_list) { + adap = lep->com.dev->rdev.adap; + for_each_port(adap, i) { + if (new_conn_ifp == adap->port[i]->vi[0].ifp) { + found_lep =1; + goto out; + } + } + } +out: + mutex_unlock(&listen_port_mutex); + return found_lep ? lep : (NULL); +} + static void process_timeout(struct c4iw_ep *ep) { - struct c4iw_qp_attributes attrs; + struct c4iw_qp_attributes attrs = {0}; int abort = 1; - mutex_lock(&ep->com.mutex); CTR4(KTR_IW_CXGBE, "%s ep :%p, tid:%u, state %d", __func__, ep, ep->hwtid, ep->com.state); set_bit(TIMEDOUT, &ep->com.history); switch (ep->com.state) { case MPA_REQ_SENT: connect_reply_upcall(ep, -ETIMEDOUT); break; case MPA_REQ_WAIT: case MPA_REQ_RCVD: case MPA_REP_SENT: case FPDU_MODE: break; case CLOSING: case MORIBUND: if (ep->com.cm_id && ep->com.qp) { attrs.next_state = C4IW_QP_STATE_ERROR; c4iw_modify_qp(ep->com.dev, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); } close_complete_upcall(ep, -ETIMEDOUT); break; case ABORTING: case DEAD: /* * These states are expected if the ep timed out at the same * time as another thread was calling stop_ep_timer(). * So we silently do nothing for these states. */ abort = 0; break; default: CTR4(KTR_IW_CXGBE, "%s unexpected state ep %p tid %u state %u\n" , __func__, ep, ep->hwtid, ep->com.state); abort = 0; } - mutex_unlock(&ep->com.mutex); if (abort) c4iw_ep_disconnect(ep, 1, GFP_KERNEL); c4iw_put_ep(&ep->com); return; } struct cqe_list_entry { struct list_head entry; struct c4iw_dev *rhp; struct t4_cqe err_cqe; }; static void process_err_cqes(void) { unsigned long flag; struct cqe_list_entry *cle; spin_lock_irqsave(&err_cqe_lock, flag); while (!list_empty(&err_cqe_list)) { struct list_head *tmp; tmp = err_cqe_list.next; list_del(tmp); tmp->next = tmp->prev = NULL; spin_unlock_irqrestore(&err_cqe_lock, flag); cle = list_entry(tmp, struct cqe_list_entry, entry); c4iw_ev_dispatch(cle->rhp, &cle->err_cqe); free(cle, M_CXGBE); spin_lock_irqsave(&err_cqe_lock, flag); } spin_unlock_irqrestore(&err_cqe_lock, flag); return; } static void process_req(struct work_struct *ctx) { struct c4iw_ep_common *epc; unsigned long flag; int ep_events; process_err_cqes(); spin_lock_irqsave(&req_lock, flag); while (!TAILQ_EMPTY(&req_list)) { epc = TAILQ_FIRST(&req_list); TAILQ_REMOVE(&req_list, epc, entry); epc->entry.tqe_prev = NULL; ep_events = epc->ep_events; epc->ep_events = 0; spin_unlock_irqrestore(&req_lock, flag); - CTR4(KTR_IW_CXGBE, "%s: so %p, ep %p, events 0x%x", __func__, - epc->so, epc, ep_events); + mutex_lock(&epc->mutex); + CTR5(KTR_IW_CXGBE, "%s: so %p, ep %p, ep_state %s events 0x%x", + __func__, epc->so, epc, states[epc->state], ep_events); if (ep_events & C4IW_EVENT_TERM) process_terminate((struct c4iw_ep *)epc); if (ep_events & C4IW_EVENT_TIMEOUT) process_timeout((struct c4iw_ep *)epc); if (ep_events & C4IW_EVENT_SOCKET) process_socket_event((struct c4iw_ep *)epc); + mutex_unlock(&epc->mutex); c4iw_put_ep(epc); process_err_cqes(); spin_lock_irqsave(&req_lock, flag); } spin_unlock_irqrestore(&req_lock, flag); } /* * XXX: doesn't belong here in the iWARP driver. * XXX: assumes that the connection was offloaded by cxgbe/t4_tom if TF_TOE is * set. Is this a valid assumption for active open? */ static int set_tcpinfo(struct c4iw_ep *ep) { struct socket *so = ep->com.so; struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp; struct toepcb *toep; int rc = 0; INP_WLOCK(inp); tp = intotcpcb(inp); if ((tp->t_flags & TF_TOE) == 0) { rc = EINVAL; log(LOG_ERR, "%s: connection not offloaded (so %p, ep %p)\n", __func__, so, ep); goto done; } toep = TOEPCB(so); ep->hwtid = toep->tid; ep->snd_seq = tp->snd_nxt; ep->rcv_seq = tp->rcv_nxt; ep->emss = max(tp->t_maxseg, 128); done: INP_WUNLOCK(inp); return (rc); } - static int -find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, - __be16 peer_port, u8 tos, struct nhop4_extended *pnh4) +get_ifnet_from_raddr(struct sockaddr_storage *raddr, struct ifnet **ifp) { - struct in_addr addr; - int err; + int err = 0; - CTR5(KTR_IW_CXGBE, "%s:frtB %x, %x, %d, %d", __func__, local_ip, - peer_ip, ntohs(local_port), ntohs(peer_port)); + if (raddr->ss_family == AF_INET) { + struct sockaddr_in *raddr4 = (struct sockaddr_in *)raddr; + struct nhop4_extended nh4 = {0}; - addr.s_addr = peer_ip; - err = fib4_lookup_nh_ext(RT_DEFAULT_FIB, addr, NHR_REF, 0, pnh4); + err = fib4_lookup_nh_ext(RT_DEFAULT_FIB, raddr4->sin_addr, + NHR_REF, 0, &nh4); + *ifp = nh4.nh_ifp; + if (err) + fib4_free_nh_ext(RT_DEFAULT_FIB, &nh4); + } else { + struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)raddr; + struct nhop6_extended nh6 = {0}; + struct in6_addr addr6; + uint32_t scopeid; - CTR2(KTR_IW_CXGBE, "%s:frtE %d", __func__, err); + memset(&addr6, 0, sizeof(addr6)); + in6_splitscope((struct in6_addr *)&raddr6->sin6_addr, + &addr6, &scopeid); + err = fib6_lookup_nh_ext(RT_DEFAULT_FIB, &addr6, scopeid, + NHR_REF, 0, &nh6); + *ifp = nh6.nh_ifp; + if (err) + fib6_free_nh_ext(RT_DEFAULT_FIB, &nh6); + } + + CTR2(KTR_IW_CXGBE, "%s: return: %d", __func__, err); return err; } static void close_socket(struct socket *so) { - uninit_iwarp_socket(so); - sodisconnect(so); + soclose(so); } static void process_peer_close(struct c4iw_ep *ep) { - struct c4iw_qp_attributes attrs; + struct c4iw_qp_attributes attrs = {0}; int disconnect = 1; int release = 0; CTR4(KTR_IW_CXGBE, "%s:ppcB ep %p so %p state %s", __func__, ep, ep->com.so, states[ep->com.state]); - mutex_lock(&ep->com.mutex); switch (ep->com.state) { case MPA_REQ_WAIT: - CTR2(KTR_IW_CXGBE, "%s:ppc1 %p MPA_REQ_WAIT CLOSING", + CTR2(KTR_IW_CXGBE, "%s:ppc1 %p MPA_REQ_WAIT DEAD", __func__, ep); - __state_set(&ep->com, CLOSING); - break; - + /* Fallthrough */ case MPA_REQ_SENT: - CTR2(KTR_IW_CXGBE, "%s:ppc2 %p MPA_REQ_SENT CLOSING", + CTR2(KTR_IW_CXGBE, "%s:ppc2 %p MPA_REQ_SENT DEAD", __func__, ep); - __state_set(&ep->com, DEAD); + ep->com.state = DEAD; connect_reply_upcall(ep, -ECONNABORTED); disconnect = 0; STOP_EP_TIMER(ep); close_socket(ep->com.so); deref_cm_id(&ep->com); release = 1; break; case MPA_REQ_RCVD: /* * We're gonna mark this puppy DEAD, but keep * the reference on it until the ULP accepts or * rejects the CR. */ CTR2(KTR_IW_CXGBE, "%s:ppc3 %p MPA_REQ_RCVD CLOSING", __func__, ep); - __state_set(&ep->com, CLOSING); - c4iw_get_ep(&ep->com); + ep->com.state = CLOSING; break; case MPA_REP_SENT: CTR2(KTR_IW_CXGBE, "%s:ppc4 %p MPA_REP_SENT CLOSING", __func__, ep); - __state_set(&ep->com, CLOSING); + ep->com.state = CLOSING; break; case FPDU_MODE: CTR2(KTR_IW_CXGBE, "%s:ppc5 %p FPDU_MODE CLOSING", __func__, ep); START_EP_TIMER(ep); - __state_set(&ep->com, CLOSING); + ep->com.state = CLOSING; attrs.next_state = C4IW_QP_STATE_CLOSING; c4iw_modify_qp(ep->com.dev, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); peer_close_upcall(ep); break; case ABORTING: CTR2(KTR_IW_CXGBE, "%s:ppc6 %p ABORTING (disconn)", __func__, ep); disconnect = 0; break; case CLOSING: CTR2(KTR_IW_CXGBE, "%s:ppc7 %p CLOSING MORIBUND", __func__, ep); - __state_set(&ep->com, MORIBUND); + ep->com.state = MORIBUND; disconnect = 0; break; case MORIBUND: CTR2(KTR_IW_CXGBE, "%s:ppc8 %p MORIBUND DEAD", __func__, ep); STOP_EP_TIMER(ep); if (ep->com.cm_id && ep->com.qp) { attrs.next_state = C4IW_QP_STATE_IDLE; c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); } close_socket(ep->com.so); close_complete_upcall(ep, 0); - __state_set(&ep->com, DEAD); + ep->com.state = DEAD; release = 1; disconnect = 0; break; case DEAD: CTR2(KTR_IW_CXGBE, "%s:ppc9 %p DEAD (disconn)", __func__, ep); disconnect = 0; break; default: panic("%s: ep %p state %d", __func__, ep, ep->com.state); break; } - mutex_unlock(&ep->com.mutex); if (disconnect) { CTR2(KTR_IW_CXGBE, "%s:ppca %p", __func__, ep); c4iw_ep_disconnect(ep, 0, M_NOWAIT); } if (release) { CTR2(KTR_IW_CXGBE, "%s:ppcb %p", __func__, ep); c4iw_put_ep(&ep->com); } CTR2(KTR_IW_CXGBE, "%s:ppcE %p", __func__, ep); return; } static void process_conn_error(struct c4iw_ep *ep) { - struct c4iw_qp_attributes attrs; + struct c4iw_qp_attributes attrs = {0}; int ret; int state; - mutex_lock(&ep->com.mutex); state = ep->com.state; CTR5(KTR_IW_CXGBE, "%s:pceB ep %p so %p so->so_error %u state %s", __func__, ep, ep->com.so, ep->com.so->so_error, states[ep->com.state]); switch (state) { case MPA_REQ_WAIT: STOP_EP_TIMER(ep); + c4iw_put_ep(&ep->parent_ep->com); break; case MPA_REQ_SENT: STOP_EP_TIMER(ep); connect_reply_upcall(ep, -ECONNRESET); break; case MPA_REP_SENT: ep->com.rpl_err = ECONNRESET; CTR1(KTR_IW_CXGBE, "waking up ep %p", ep); break; case MPA_REQ_RCVD: - - /* - * We're gonna mark this puppy DEAD, but keep - * the reference on it until the ULP accepts or - * rejects the CR. - */ - c4iw_get_ep(&ep->com); break; case MORIBUND: case CLOSING: STOP_EP_TIMER(ep); /*FALLTHROUGH*/ case FPDU_MODE: if (ep->com.cm_id && ep->com.qp) { attrs.next_state = C4IW_QP_STATE_ERROR; ret = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); if (ret) log(LOG_ERR, "%s - qp <- error failed!\n", __func__); } peer_abort_upcall(ep); break; case ABORTING: break; case DEAD: CTR2(KTR_IW_CXGBE, "%s so_error %d IN DEAD STATE!!!!", __func__, ep->com.so->so_error); - mutex_unlock(&ep->com.mutex); return; default: panic("%s: ep %p state %d", __func__, ep, state); break; } if (state != ABORTING) { close_socket(ep->com.so); - __state_set(&ep->com, DEAD); + ep->com.state = DEAD; c4iw_put_ep(&ep->com); } - mutex_unlock(&ep->com.mutex); CTR2(KTR_IW_CXGBE, "%s:pceE %p", __func__, ep); return; } static void process_close_complete(struct c4iw_ep *ep) { - struct c4iw_qp_attributes attrs; + struct c4iw_qp_attributes attrs = {0}; int release = 0; CTR4(KTR_IW_CXGBE, "%s:pccB ep %p so %p state %s", __func__, ep, ep->com.so, states[ep->com.state]); /* The cm_id may be null if we failed to connect */ - mutex_lock(&ep->com.mutex); set_bit(CLOSE_CON_RPL, &ep->com.history); switch (ep->com.state) { case CLOSING: CTR2(KTR_IW_CXGBE, "%s:pcc1 %p CLOSING MORIBUND", __func__, ep); - __state_set(&ep->com, MORIBUND); + ep->com.state = MORIBUND; break; case MORIBUND: CTR2(KTR_IW_CXGBE, "%s:pcc1 %p MORIBUND DEAD", __func__, ep); STOP_EP_TIMER(ep); if ((ep->com.cm_id) && (ep->com.qp)) { CTR2(KTR_IW_CXGBE, "%s:pcc2 %p QP_STATE_IDLE", __func__, ep); attrs.next_state = C4IW_QP_STATE_IDLE; c4iw_modify_qp(ep->com.dev, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); } close_socket(ep->com.so); close_complete_upcall(ep, 0); - __state_set(&ep->com, DEAD); + ep->com.state = DEAD; release = 1; break; case ABORTING: CTR2(KTR_IW_CXGBE, "%s:pcc5 %p ABORTING", __func__, ep); break; case DEAD: CTR2(KTR_IW_CXGBE, "%s:pcc6 %p DEAD", __func__, ep); break; default: CTR2(KTR_IW_CXGBE, "%s:pcc7 %p unknown ep state", __func__, ep); panic("%s:pcc6 %p unknown ep state", __func__, ep); break; } - mutex_unlock(&ep->com.mutex); if (release) { CTR2(KTR_IW_CXGBE, "%s:pcc8 %p", __func__, ep); - c4iw_put_ep(&ep->com); + release_ep_resources(ep); } CTR2(KTR_IW_CXGBE, "%s:pccE %p", __func__, ep); return; } static void setiwsockopt(struct socket *so) { int rc; struct sockopt sopt; int on = 1; sopt.sopt_dir = SOPT_SET; sopt.sopt_level = IPPROTO_TCP; sopt.sopt_name = TCP_NODELAY; sopt.sopt_val = (caddr_t)&on; sopt.sopt_valsize = sizeof on; sopt.sopt_td = NULL; rc = sosetopt(so, &sopt); if (rc) { log(LOG_ERR, "%s: can't set TCP_NODELAY on so %p (%d)\n", __func__, so, rc); } } static void init_iwarp_socket(struct socket *so, void *arg) { - - SOCKBUF_LOCK(&so->so_rcv); - soupcall_set(so, SO_RCV, c4iw_so_upcall, arg); - so->so_state |= SS_NBIO; - SOCKBUF_UNLOCK(&so->so_rcv); + if (SOLISTENING(so)) { + SOLISTEN_LOCK(so); + solisten_upcall_set(so, c4iw_so_upcall, arg); + so->so_state |= SS_NBIO; + SOLISTEN_UNLOCK(so); + } else { + SOCKBUF_LOCK(&so->so_rcv); + soupcall_set(so, SO_RCV, c4iw_so_upcall, arg); + so->so_state |= SS_NBIO; + SOCKBUF_UNLOCK(&so->so_rcv); + } } static void uninit_iwarp_socket(struct socket *so) { - - SOCKBUF_LOCK(&so->so_rcv); - soupcall_clear(so, SO_RCV); - SOCKBUF_UNLOCK(&so->so_rcv); + if (SOLISTENING(so)) { + SOLISTEN_LOCK(so); + solisten_upcall_set(so, NULL, NULL); + SOLISTEN_UNLOCK(so); + } else { + SOCKBUF_LOCK(&so->so_rcv); + soupcall_clear(so, SO_RCV); + SOCKBUF_UNLOCK(&so->so_rcv); + } } static void process_data(struct c4iw_ep *ep) { - struct sockaddr_in *local, *remote; int disconnect = 0; CTR5(KTR_IW_CXGBE, "%s: so %p, ep %p, state %s, sbused %d", __func__, ep->com.so, ep, states[ep->com.state], sbused(&ep->com.so->so_rcv)); - switch (state_read(&ep->com)) { + switch (ep->com.state) { case MPA_REQ_SENT: disconnect = process_mpa_reply(ep); break; case MPA_REQ_WAIT: - in_getsockaddr(ep->com.so, (struct sockaddr **)&local); - in_getpeeraddr(ep->com.so, (struct sockaddr **)&remote); - ep->com.local_addr = *local; - ep->com.remote_addr = *remote; - free(local, M_SONAME); - free(remote, M_SONAME); disconnect = process_mpa_request(ep); + if (disconnect) + /* Refered in process_newconn() */ + c4iw_put_ep(&ep->parent_ep->com); break; default: if (sbused(&ep->com.so->so_rcv)) log(LOG_ERR, "%s: Unexpected streaming data. ep %p, " "state %d, so %p, so_state 0x%x, sbused %u\n", - __func__, ep, state_read(&ep->com), ep->com.so, + __func__, ep, ep->com.state, ep->com.so, ep->com.so->so_state, sbused(&ep->com.so->so_rcv)); break; } if (disconnect) c4iw_ep_disconnect(ep, disconnect == 2, GFP_KERNEL); } static void process_connected(struct c4iw_ep *ep) { struct socket *so = ep->com.so; if ((so->so_state & SS_ISCONNECTED) && !so->so_error) { if (send_mpa_req(ep)) goto err; } else { connect_reply_upcall(ep, -so->so_error); goto err; } return; err: close_socket(so); - state_set(&ep->com, DEAD); + ep->com.state = DEAD; c4iw_put_ep(&ep->com); return; } -void -process_newconn(struct iw_cm_id *parent_cm_id, struct socket *child_so) +static inline int c4iw_zero_addr(struct sockaddr *addr) { - struct c4iw_ep *child_ep; - struct sockaddr_in *local; - struct sockaddr_in *remote; - struct c4iw_ep *parent_ep = parent_cm_id->provider_data; + struct in6_addr *ip6; + + if (addr->sa_family == AF_INET) + return IN_ZERONET( + ntohl(((struct sockaddr_in *)addr)->sin_addr.s_addr)); + else { + ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr; + return (ip6->s6_addr32[0] | ip6->s6_addr32[1] | + ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0; + } +} + +static inline int c4iw_loopback_addr(struct sockaddr *addr) +{ + if (addr->sa_family == AF_INET) + return IN_LOOPBACK( + ntohl(((struct sockaddr_in *) addr)->sin_addr.s_addr)); + else + return IN6_IS_ADDR_LOOPBACK( + &((struct sockaddr_in6 *) addr)->sin6_addr); +} + +static inline int c4iw_any_addr(struct sockaddr *addr) +{ + return c4iw_zero_addr(addr) || c4iw_loopback_addr(addr); +} + +static void +process_newconn(struct c4iw_listen_ep *master_lep, struct socket *new_so) +{ + struct c4iw_listen_ep *real_lep = NULL; + struct c4iw_ep *new_ep = NULL; + struct sockaddr_in *remote = NULL; int ret = 0; - MPASS(child_so != NULL); + MPASS(new_so != NULL); - child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); + if (c4iw_any_addr((struct sockaddr *)&master_lep->com.local_addr)) { + /* Here we need to find the 'real_lep' that belongs to the + * incomming socket's network interface, such that the newly + * created 'ep' can be attached to the real 'lep'. + */ + real_lep = find_real_listen_ep(master_lep, new_so); + if (real_lep == NULL) { + CTR2(KTR_IW_CXGBE, "%s: Could not find the real listen " + "ep for sock: %p", __func__, new_so); + log(LOG_ERR,"%s: Could not find the real listen ep for " + "sock: %p\n", __func__, new_so); + /* FIXME: properly free the 'new_so' in failure case. + * Use of soabort() and soclose() are not legal + * here(before soaccept()). + */ + return; + } + } else /* for Non-Wildcard address, master_lep is always the real_lep */ + real_lep = master_lep; - CTR5(KTR_IW_CXGBE, - "%s: parent so %p, parent ep %p, child so %p, child ep %p", - __func__, parent_ep->com.so, parent_ep, child_so, child_ep); + new_ep = alloc_ep(sizeof(*new_ep), GFP_KERNEL); - in_getsockaddr(child_so, (struct sockaddr **)&local); - in_getpeeraddr(child_so, (struct sockaddr **)&remote); + CTR6(KTR_IW_CXGBE, "%s: master_lep %p, real_lep: %p, new ep %p, " + "listening so %p, new so %p", __func__, master_lep, real_lep, + new_ep, master_lep->com.so, new_so); - child_ep->com.local_addr = *local; - child_ep->com.remote_addr = *remote; - child_ep->com.dev = parent_ep->com.dev; - child_ep->com.so = child_so; - child_ep->com.cm_id = NULL; - child_ep->com.thread = parent_ep->com.thread; - child_ep->parent_ep = parent_ep; + new_ep->com.dev = real_lep->com.dev; + new_ep->com.so = new_so; + new_ep->com.cm_id = NULL; + new_ep->com.thread = real_lep->com.thread; + new_ep->parent_ep = real_lep; - free(local, M_SONAME); + GET_LOCAL_ADDR(&new_ep->com.local_addr, new_so); + GET_REMOTE_ADDR(&new_ep->com.remote_addr, new_so); + c4iw_get_ep(&real_lep->com); + init_timer(&new_ep->timer); + new_ep->com.state = MPA_REQ_WAIT; + START_EP_TIMER(new_ep); + + setiwsockopt(new_so); + ret = soaccept(new_so, (struct sockaddr **)&remote); + if (ret != 0) { + CTR4(KTR_IW_CXGBE, + "%s:listen sock:%p, new sock:%p, ret:%d\n", + __func__, master_lep->com.so, new_so, ret); + if (remote != NULL) + free(remote, M_SONAME); + uninit_iwarp_socket(new_so); + soclose(new_so); + c4iw_put_ep(&new_ep->com); + c4iw_put_ep(&real_lep->com); + return; + } free(remote, M_SONAME); - setiwsockopt(child_so); - init_iwarp_socket(child_so, &child_ep->com); - c4iw_get_ep(&parent_ep->com); - init_timer(&child_ep->timer); - state_set(&child_ep->com, MPA_REQ_WAIT); - START_EP_TIMER(child_ep); + /* MPA request might have been queued up on the socket already, so we + * initialize the socket/upcall_handler under lock to prevent processing + * MPA request on another thread(via process_req()) simultaniously. + */ + c4iw_get_ep(&new_ep->com); /* Dereferenced at the end below, this is to + avoid freeing of ep before ep unlock. */ + mutex_lock(&new_ep->com.mutex); + init_iwarp_socket(new_so, &new_ep->com); - /* maybe the request has already been queued up on the socket... */ - ret = process_mpa_request(child_ep); - if (ret == 2) + ret = process_mpa_request(new_ep); + if (ret) { /* ABORT */ - c4iw_ep_disconnect(child_ep, 1, GFP_KERNEL); - else if (ret == 1) - /* CLOSE */ - c4iw_ep_disconnect(child_ep, 0, GFP_KERNEL); - + c4iw_ep_disconnect(new_ep, 1, GFP_KERNEL); + c4iw_put_ep(&real_lep->com); + } + mutex_unlock(&new_ep->com.mutex); + c4iw_put_ep(&new_ep->com); return; } static int add_ep_to_req_list(struct c4iw_ep *ep, int new_ep_event) { unsigned long flag; spin_lock_irqsave(&req_lock, flag); if (ep && ep->com.so) { ep->com.ep_events |= new_ep_event; if (!ep->com.entry.tqe_prev) { c4iw_get_ep(&ep->com); TAILQ_INSERT_TAIL(&req_list, &ep->com, entry); queue_work(c4iw_taskq, &c4iw_task); } } spin_unlock_irqrestore(&req_lock, flag); return (0); } static int c4iw_so_upcall(struct socket *so, void *arg, int waitflag) { struct c4iw_ep *ep = arg; CTR6(KTR_IW_CXGBE, "%s: so %p, so_state 0x%x, ep %p, ep_state %s, tqe_prev %p", __func__, so, so->so_state, ep, states[ep->com.state], ep->com.entry.tqe_prev); MPASS(ep->com.so == so); + /* + * Wake up any threads waiting in rdma_init()/rdma_fini(), + * with locks held. + */ + if (so->so_error) + c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); add_ep_to_req_list(ep, C4IW_EVENT_SOCKET); return (SU_OK); } static int terminate(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_rdma_terminate *cpl = mtod(m, const void *); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); struct socket *so; struct c4iw_ep *ep; INP_WLOCK(toep->inp); so = inp_inpcbtosocket(toep->inp); ep = so->so_rcv.sb_upcallarg; INP_WUNLOCK(toep->inp); CTR3(KTR_IW_CXGBE, "%s: so %p, ep %p", __func__, so, ep); add_ep_to_req_list(ep, C4IW_EVENT_TERM); return 0; } static void process_socket_event(struct c4iw_ep *ep) { - int state = state_read(&ep->com); + int state = ep->com.state; struct socket *so = ep->com.so; + if (ep->com.state == DEAD) { + CTR3(KTR_IW_CXGBE, "%s: Pending socket event discarded " + "ep %p ep_state %s", __func__, ep, states[state]); + return; + } + CTR6(KTR_IW_CXGBE, "process_socket_event: so %p, so_state 0x%x, " "so_err %d, sb_state 0x%x, ep %p, ep_state %s", so, so->so_state, so->so_error, so->so_rcv.sb_state, ep, states[state]); if (state == CONNECTING) { process_connected(ep); return; } if (state == LISTEN) { - /* socket listening events are handled at IWCM */ - CTR3(KTR_IW_CXGBE, "%s Invalid ep state:%u, ep:%p", __func__, - ep->com.state, ep); - BUG(); + struct c4iw_listen_ep *lep = (struct c4iw_listen_ep *)ep; + struct socket *listen_so = so, *new_so = NULL; + int error = 0; + + SOLISTEN_LOCK(listen_so); + do { + error = solisten_dequeue(listen_so, &new_so, + SOCK_NONBLOCK); + if (error) { + CTR4(KTR_IW_CXGBE, "%s: lep %p listen_so %p " + "error %d", __func__, lep, listen_so, + error); + return; + } + process_newconn(lep, new_so); + + /* solisten_dequeue() unlocks while return, so aquire + * lock again for sol_qlen and also for next iteration. + */ + SOLISTEN_LOCK(listen_so); + } while (listen_so->sol_qlen); + SOLISTEN_UNLOCK(listen_so); + return; } /* connection error */ if (so->so_error) { process_conn_error(ep); return; } /* peer close */ if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && state <= CLOSING) { process_peer_close(ep); /* * check whether socket disconnect event is pending before * returning. Fallthrough if yes. */ if (!(so->so_state & SS_ISDISCONNECTED)) return; } /* close complete */ if (so->so_state & SS_ISDISCONNECTED) { process_close_complete(ep); return; } /* rx data */ process_data(ep); } SYSCTL_NODE(_hw, OID_AUTO, iw_cxgbe, CTLFLAG_RD, 0, "iw_cxgbe driver parameters"); static int dack_mode = 0; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, dack_mode, CTLFLAG_RWTUN, &dack_mode, 0, "Delayed ack mode (default = 0)"); int c4iw_max_read_depth = 8; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_max_read_depth, CTLFLAG_RWTUN, &c4iw_max_read_depth, 0, "Per-connection max ORD/IRD (default = 8)"); static int enable_tcp_timestamps; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_timestamps, CTLFLAG_RWTUN, &enable_tcp_timestamps, 0, "Enable tcp timestamps (default = 0)"); static int enable_tcp_sack; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_sack, CTLFLAG_RWTUN, &enable_tcp_sack, 0, "Enable tcp SACK (default = 0)"); static int enable_tcp_window_scaling = 1; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_window_scaling, CTLFLAG_RWTUN, &enable_tcp_window_scaling, 0, "Enable tcp window scaling (default = 1)"); int c4iw_debug = 1; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_debug, CTLFLAG_RWTUN, &c4iw_debug, 0, "Enable debug logging (default = 0)"); static int peer2peer = 1; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, peer2peer, CTLFLAG_RWTUN, &peer2peer, 0, "Support peer2peer ULPs (default = 1)"); static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, p2p_type, CTLFLAG_RWTUN, &p2p_type, 0, "RDMAP opcode to use for the RTR message: 1 = RDMA_READ 0 = RDMA_WRITE (default 1)"); static int ep_timeout_secs = 60; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, ep_timeout_secs, CTLFLAG_RWTUN, &ep_timeout_secs, 0, "CM Endpoint operation timeout in seconds (default = 60)"); static int mpa_rev = 1; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, mpa_rev, CTLFLAG_RWTUN, &mpa_rev, 0, "MPA Revision, 0 supports amso1100, 1 is RFC5044 spec compliant, 2 is IETF MPA Peer Connect Draft compliant (default = 1)"); static int markers_enabled; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, markers_enabled, CTLFLAG_RWTUN, &markers_enabled, 0, "Enable MPA MARKERS (default(0) = disabled)"); static int crc_enabled = 1; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, crc_enabled, CTLFLAG_RWTUN, &crc_enabled, 0, "Enable MPA CRC (default(1) = enabled)"); static int rcv_win = 256 * 1024; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, rcv_win, CTLFLAG_RWTUN, &rcv_win, 0, "TCP receive window in bytes (default = 256KB)"); static int snd_win = 128 * 1024; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, snd_win, CTLFLAG_RWTUN, &snd_win, 0, "TCP send window in bytes (default = 128KB)"); static void start_ep_timer(struct c4iw_ep *ep) { if (timer_pending(&ep->timer)) { CTR2(KTR_IW_CXGBE, "%s: ep %p, already started", __func__, ep); printk(KERN_ERR "%s timer already started! ep %p\n", __func__, ep); return; } clear_bit(TIMEOUT, &ep->com.flags); c4iw_get_ep(&ep->com); ep->timer.expires = jiffies + ep_timeout_secs * HZ; ep->timer.data = (unsigned long)ep; ep->timer.function = ep_timeout; add_timer(&ep->timer); } static int stop_ep_timer(struct c4iw_ep *ep) { del_timer_sync(&ep->timer); if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { c4iw_put_ep(&ep->com); return 0; } return 1; } -static enum -c4iw_ep_state state_read(struct c4iw_ep_common *epc) -{ - enum c4iw_ep_state state; - - mutex_lock(&epc->mutex); - state = epc->state; - mutex_unlock(&epc->mutex); - - return (state); -} - -static void -__state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) -{ - - epc->state = new; -} - -static void -state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) -{ - - mutex_lock(&epc->mutex); - __state_set(epc, new); - mutex_unlock(&epc->mutex); -} - static void * alloc_ep(int size, gfp_t gfp) { struct c4iw_ep_common *epc; epc = kzalloc(size, gfp); if (epc == NULL) return (NULL); kref_init(&epc->kref); mutex_init(&epc->mutex); c4iw_init_wr_wait(&epc->wr_wait); return (epc); } void _c4iw_free_ep(struct kref *kref) { struct c4iw_ep *ep; struct c4iw_ep_common *epc; ep = container_of(kref, struct c4iw_ep, com.kref); epc = &ep->com; KASSERT(!epc->entry.tqe_prev, ("%s epc %p still on req list", __func__, epc)); if (test_bit(QP_REFERENCED, &ep->com.flags)) deref_qp(ep); kfree(ep); } static void release_ep_resources(struct c4iw_ep *ep) { CTR2(KTR_IW_CXGBE, "%s:rerB %p", __func__, ep); set_bit(RELEASE_RESOURCES, &ep->com.flags); c4iw_put_ep(&ep->com); CTR2(KTR_IW_CXGBE, "%s:rerE %p", __func__, ep); } static int send_mpa_req(struct c4iw_ep *ep) { int mpalen; struct mpa_message *mpa; struct mpa_v2_conn_params mpa_v2_params; struct mbuf *m; char mpa_rev_to_use = mpa_rev; int err = 0; if (ep->retry_with_mpa_v1) mpa_rev_to_use = 1; mpalen = sizeof(*mpa) + ep->plen; if (mpa_rev_to_use == 2) mpalen += sizeof(struct mpa_v2_conn_params); mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); if (mpa == NULL) { err = -ENOMEM; CTR3(KTR_IW_CXGBE, "%s:smr1 ep: %p , error: %d", __func__, ep, err); goto err; } memset(mpa, 0, mpalen); memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); mpa->flags = (crc_enabled ? MPA_CRC : 0) | (markers_enabled ? MPA_MARKERS : 0) | (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); mpa->private_data_size = htons(ep->plen); mpa->revision = mpa_rev_to_use; if (mpa_rev_to_use == 1) { ep->tried_with_mpa_v1 = 1; ep->retry_with_mpa_v1 = 0; } if (mpa_rev_to_use == 2) { - mpa->private_data_size += - htons(sizeof(struct mpa_v2_conn_params)); + mpa->private_data_size = htons(ntohs(mpa->private_data_size) + + sizeof(struct mpa_v2_conn_params)); mpa_v2_params.ird = htons((u16)ep->ird); mpa_v2_params.ord = htons((u16)ep->ord); if (peer2peer) { mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { mpa_v2_params.ord |= htons(MPA_V2_RDMA_WRITE_RTR); } else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { mpa_v2_params.ord |= htons(MPA_V2_RDMA_READ_RTR); } } memcpy(mpa->private_data, &mpa_v2_params, sizeof(struct mpa_v2_conn_params)); if (ep->plen) { memcpy(mpa->private_data + sizeof(struct mpa_v2_conn_params), ep->mpa_pkt + sizeof(*mpa), ep->plen); } } else { if (ep->plen) memcpy(mpa->private_data, ep->mpa_pkt + sizeof(*mpa), ep->plen); CTR2(KTR_IW_CXGBE, "%s:smr7 %p", __func__, ep); } m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); if (m == NULL) { err = -ENOMEM; CTR3(KTR_IW_CXGBE, "%s:smr2 ep: %p , error: %d", __func__, ep, err); free(mpa, M_CXGBE); goto err; } m_copyback(m, 0, mpalen, (void *)mpa); free(mpa, M_CXGBE); err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread); if (err) { CTR3(KTR_IW_CXGBE, "%s:smr3 ep: %p , error: %d", __func__, ep, err); goto err; } START_EP_TIMER(ep); - state_set(&ep->com, MPA_REQ_SENT); + ep->com.state = MPA_REQ_SENT; ep->mpa_attr.initiator = 1; CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); return 0; err: connect_reply_upcall(ep, err); CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); return err; } static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) { int mpalen ; struct mpa_message *mpa; struct mpa_v2_conn_params mpa_v2_params; struct mbuf *m; int err; CTR4(KTR_IW_CXGBE, "%s:smrejB %p %u %d", __func__, ep, ep->hwtid, ep->plen); mpalen = sizeof(*mpa) + plen; if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { mpalen += sizeof(struct mpa_v2_conn_params); CTR4(KTR_IW_CXGBE, "%s:smrej1 %p %u %d", __func__, ep, ep->mpa_attr.version, mpalen); } mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); if (mpa == NULL) return (-ENOMEM); memset(mpa, 0, mpalen); memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); mpa->flags = MPA_REJECT; mpa->revision = mpa_rev; mpa->private_data_size = htons(plen); if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { mpa->flags |= MPA_ENHANCED_RDMA_CONN; - mpa->private_data_size += - htons(sizeof(struct mpa_v2_conn_params)); + mpa->private_data_size = htons(ntohs(mpa->private_data_size) + + sizeof(struct mpa_v2_conn_params)); mpa_v2_params.ird = htons(((u16)ep->ird) | (peer2peer ? MPA_V2_PEER2PEER_MODEL : 0)); mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE ? MPA_V2_RDMA_WRITE_RTR : p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ ? MPA_V2_RDMA_READ_RTR : 0) : 0)); memcpy(mpa->private_data, &mpa_v2_params, sizeof(struct mpa_v2_conn_params)); if (ep->plen) memcpy(mpa->private_data + - sizeof(struct mpa_v2_conn_params), pdata, plen); + sizeof(struct mpa_v2_conn_params), pdata, plen); CTR5(KTR_IW_CXGBE, "%s:smrej3 %p %d %d %d", __func__, ep, mpa_v2_params.ird, mpa_v2_params.ord, ep->plen); } else if (plen) memcpy(mpa->private_data, pdata, plen); m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); if (m == NULL) { free(mpa, M_CXGBE); return (-ENOMEM); } m_copyback(m, 0, mpalen, (void *)mpa); free(mpa, M_CXGBE); err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread); if (!err) ep->snd_seq += mpalen; CTR4(KTR_IW_CXGBE, "%s:smrejE %p %u %d", __func__, ep, ep->hwtid, err); return err; } static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) { int mpalen; struct mpa_message *mpa; struct mbuf *m; struct mpa_v2_conn_params mpa_v2_params; int err; CTR2(KTR_IW_CXGBE, "%s:smrepB %p", __func__, ep); mpalen = sizeof(*mpa) + plen; if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { CTR3(KTR_IW_CXGBE, "%s:smrep1 %p %d", __func__, ep, ep->mpa_attr.version); mpalen += sizeof(struct mpa_v2_conn_params); } mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); if (mpa == NULL) return (-ENOMEM); memset(mpa, 0, sizeof(*mpa)); memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | (markers_enabled ? MPA_MARKERS : 0); mpa->revision = ep->mpa_attr.version; mpa->private_data_size = htons(plen); if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { mpa->flags |= MPA_ENHANCED_RDMA_CONN; mpa->private_data_size += htons(sizeof(struct mpa_v2_conn_params)); mpa_v2_params.ird = htons((u16)ep->ird); mpa_v2_params.ord = htons((u16)ep->ord); CTR5(KTR_IW_CXGBE, "%s:smrep3 %p %d %d %d", __func__, ep, ep->mpa_attr.version, mpa_v2_params.ird, mpa_v2_params.ord); if (peer2peer && (ep->mpa_attr.p2p_type != FW_RI_INIT_P2PTYPE_DISABLED)) { mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { mpa_v2_params.ord |= htons(MPA_V2_RDMA_WRITE_RTR); CTR5(KTR_IW_CXGBE, "%s:smrep4 %p %d %d %d", __func__, ep, p2p_type, mpa_v2_params.ird, mpa_v2_params.ord); } else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { mpa_v2_params.ord |= htons(MPA_V2_RDMA_READ_RTR); CTR5(KTR_IW_CXGBE, "%s:smrep5 %p %d %d %d", __func__, ep, p2p_type, mpa_v2_params.ird, mpa_v2_params.ord); } } memcpy(mpa->private_data, &mpa_v2_params, sizeof(struct mpa_v2_conn_params)); if (ep->plen) memcpy(mpa->private_data + sizeof(struct mpa_v2_conn_params), pdata, plen); } else if (plen) memcpy(mpa->private_data, pdata, plen); m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); if (m == NULL) { free(mpa, M_CXGBE); return (-ENOMEM); } m_copyback(m, 0, mpalen, (void *)mpa); free(mpa, M_CXGBE); - state_set(&ep->com, MPA_REP_SENT); + ep->com.state = MPA_REP_SENT; ep->snd_seq += mpalen; err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread); CTR3(KTR_IW_CXGBE, "%s:smrepE %p %d", __func__, ep, err); return err; } static void close_complete_upcall(struct c4iw_ep *ep, int status) { struct iw_cm_event event; CTR2(KTR_IW_CXGBE, "%s:ccuB %p", __func__, ep); memset(&event, 0, sizeof(event)); event.event = IW_CM_EVENT_CLOSE; event.status = status; if (ep->com.cm_id) { CTR2(KTR_IW_CXGBE, "%s:ccu1 %1", __func__, ep); ep->com.cm_id->event_handler(ep->com.cm_id, &event); deref_cm_id(&ep->com); set_bit(CLOSE_UPCALL, &ep->com.history); } CTR2(KTR_IW_CXGBE, "%s:ccuE %p", __func__, ep); } static int send_abort(struct c4iw_ep *ep) { struct socket *so = ep->com.so; struct sockopt sopt; int rc; struct linger l; CTR5(KTR_IW_CXGBE, "%s ep %p so %p state %s tid %d", __func__, ep, so, states[ep->com.state], ep->hwtid); l.l_onoff = 1; l.l_linger = 0; /* linger_time of 0 forces RST to be sent */ sopt.sopt_dir = SOPT_SET; sopt.sopt_level = SOL_SOCKET; sopt.sopt_name = SO_LINGER; sopt.sopt_val = (caddr_t)&l; sopt.sopt_valsize = sizeof l; sopt.sopt_td = NULL; rc = sosetopt(so, &sopt); if (rc != 0) { log(LOG_ERR, "%s: sosetopt(%p, linger = 0) failed with %d.\n", __func__, so, rc); } uninit_iwarp_socket(so); - sodisconnect(so); + soclose(so); set_bit(ABORT_CONN, &ep->com.history); /* * TBD: iw_cxgbe driver should receive ABORT reply for every ABORT * request it has sent. But the current TOE driver is not propagating * this ABORT reply event (via do_abort_rpl) to iw_cxgbe. So as a work- - * around de-refer 'ep' (which was refered before sending ABORT request) - * here instead of doing it in abort_rpl() handler of iw_cxgbe driver. + * around de-refererece 'ep' here instead of doing it in abort_rpl() + * handler(not yet implemented) of iw_cxgbe driver. */ - c4iw_put_ep(&ep->com); + release_ep_resources(ep); return (0); } static void peer_close_upcall(struct c4iw_ep *ep) { struct iw_cm_event event; CTR2(KTR_IW_CXGBE, "%s:pcuB %p", __func__, ep); memset(&event, 0, sizeof(event)); event.event = IW_CM_EVENT_DISCONNECT; if (ep->com.cm_id) { CTR2(KTR_IW_CXGBE, "%s:pcu1 %p", __func__, ep); ep->com.cm_id->event_handler(ep->com.cm_id, &event); set_bit(DISCONN_UPCALL, &ep->com.history); } CTR2(KTR_IW_CXGBE, "%s:pcuE %p", __func__, ep); } static void peer_abort_upcall(struct c4iw_ep *ep) { struct iw_cm_event event; CTR2(KTR_IW_CXGBE, "%s:pauB %p", __func__, ep); memset(&event, 0, sizeof(event)); event.event = IW_CM_EVENT_CLOSE; event.status = -ECONNRESET; if (ep->com.cm_id) { CTR2(KTR_IW_CXGBE, "%s:pau1 %p", __func__, ep); ep->com.cm_id->event_handler(ep->com.cm_id, &event); deref_cm_id(&ep->com); set_bit(ABORT_UPCALL, &ep->com.history); } CTR2(KTR_IW_CXGBE, "%s:pauE %p", __func__, ep); } static void connect_reply_upcall(struct c4iw_ep *ep, int status) { struct iw_cm_event event; CTR3(KTR_IW_CXGBE, "%s:cruB %p, status: %d", __func__, ep, status); memset(&event, 0, sizeof(event)); event.event = IW_CM_EVENT_CONNECT_REPLY; event.status = ((status == -ECONNABORTED) || (status == -EPIPE)) ? -ECONNRESET : status; event.local_addr = ep->com.local_addr; event.remote_addr = ep->com.remote_addr; if ((status == 0) || (status == -ECONNREFUSED)) { if (!ep->tried_with_mpa_v1) { CTR2(KTR_IW_CXGBE, "%s:cru1 %p", __func__, ep); /* this means MPA_v2 is used */ + event.ord = ep->ird; + event.ird = ep->ord; event.private_data_len = ep->plen - sizeof(struct mpa_v2_conn_params); event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + sizeof(struct mpa_v2_conn_params); } else { CTR2(KTR_IW_CXGBE, "%s:cru2 %p", __func__, ep); /* this means MPA_v1 is used */ + event.ord = c4iw_max_read_depth; + event.ird = c4iw_max_read_depth; event.private_data_len = ep->plen; event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); } } if (ep->com.cm_id) { CTR2(KTR_IW_CXGBE, "%s:cru3 %p", __func__, ep); set_bit(CONN_RPL_UPCALL, &ep->com.history); ep->com.cm_id->event_handler(ep->com.cm_id, &event); } if(status == -ECONNABORTED) { CTR3(KTR_IW_CXGBE, "%s:cruE %p %d", __func__, ep, status); return; } if (status < 0) { CTR3(KTR_IW_CXGBE, "%s:cru4 %p %d", __func__, ep, status); deref_cm_id(&ep->com); } CTR2(KTR_IW_CXGBE, "%s:cruE %p", __func__, ep); } static int connect_request_upcall(struct c4iw_ep *ep) { struct iw_cm_event event; int ret; CTR3(KTR_IW_CXGBE, "%s: ep %p, mpa_v1 %d", __func__, ep, ep->tried_with_mpa_v1); memset(&event, 0, sizeof(event)); event.event = IW_CM_EVENT_CONNECT_REQUEST; event.local_addr = ep->com.local_addr; event.remote_addr = ep->com.remote_addr; event.provider_data = ep; - event.so = ep->com.so; if (!ep->tried_with_mpa_v1) { /* this means MPA_v2 is used */ event.ord = ep->ord; event.ird = ep->ird; event.private_data_len = ep->plen - sizeof(struct mpa_v2_conn_params); event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + sizeof(struct mpa_v2_conn_params); } else { /* this means MPA_v1 is used. Send max supported */ event.ord = c4iw_max_read_depth; event.ird = c4iw_max_read_depth; event.private_data_len = ep->plen; event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); } c4iw_get_ep(&ep->com); ret = ep->parent_ep->com.cm_id->event_handler(ep->parent_ep->com.cm_id, &event); - if(ret) + if(ret) { + CTR3(KTR_IW_CXGBE, "%s: ep %p, Failure while notifying event to" + " IWCM, err:%d", __func__, ep, ret); c4iw_put_ep(&ep->com); + } else + /* Dereference parent_ep only in success case. + * In case of failure, parent_ep is dereferenced by the caller + * of process_mpa_request(). + */ + c4iw_put_ep(&ep->parent_ep->com); set_bit(CONNREQ_UPCALL, &ep->com.history); - c4iw_put_ep(&ep->parent_ep->com); return ret; } static void established_upcall(struct c4iw_ep *ep) { struct iw_cm_event event; CTR2(KTR_IW_CXGBE, "%s:euB %p", __func__, ep); memset(&event, 0, sizeof(event)); event.event = IW_CM_EVENT_ESTABLISHED; - event.ird = ep->ird; - event.ord = ep->ord; + event.ird = ep->ord; + event.ord = ep->ird; if (ep->com.cm_id) { CTR2(KTR_IW_CXGBE, "%s:eu1 %p", __func__, ep); ep->com.cm_id->event_handler(ep->com.cm_id, &event); set_bit(ESTAB_UPCALL, &ep->com.history); } CTR2(KTR_IW_CXGBE, "%s:euE %p", __func__, ep); } +#define RELAXED_IRD_NEGOTIATION 1 + /* * process_mpa_reply - process streaming mode MPA reply * * Returns: * * 0 upon success indicating a connect request was delivered to the ULP * or the mpa request is incomplete but valid so far. * * 1 if a failure requires the caller to close the connection. * * 2 if a failure requires the caller to abort the connection. */ static int process_mpa_reply(struct c4iw_ep *ep) { struct mpa_message *mpa; struct mpa_v2_conn_params *mpa_v2_params; u16 plen; u16 resp_ird, resp_ord; u8 rtr_mismatch = 0, insuff_ird = 0; - struct c4iw_qp_attributes attrs; + struct c4iw_qp_attributes attrs = {0}; enum c4iw_qp_attr_mask mask; int err; struct mbuf *top, *m; int flags = MSG_DONTWAIT; struct uio uio; int disconnect = 0; CTR2(KTR_IW_CXGBE, "%s:pmrB %p", __func__, ep); /* * Stop mpa timer. If it expired, then * we ignore the MPA reply. process_timeout() * will abort the connection. */ if (STOP_EP_TIMER(ep)) return 0; uio.uio_resid = 1000000; uio.uio_td = ep->com.thread; err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags); if (err) { if (err == EWOULDBLOCK) { CTR2(KTR_IW_CXGBE, "%s:pmr1 %p", __func__, ep); START_EP_TIMER(ep); return 0; } err = -err; CTR2(KTR_IW_CXGBE, "%s:pmr2 %p", __func__, ep); goto err; } if (ep->com.so->so_rcv.sb_mb) { CTR2(KTR_IW_CXGBE, "%s:pmr3 %p", __func__, ep); printf("%s data after soreceive called! so %p sb_mb %p top %p\n", __func__, ep->com.so, ep->com.so->so_rcv.sb_mb, top); } m = top; do { CTR2(KTR_IW_CXGBE, "%s:pmr4 %p", __func__, ep); /* * If we get more than the supported amount of private data * then we must fail this connection. */ if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) { CTR3(KTR_IW_CXGBE, "%s:pmr5 %p %d", __func__, ep, ep->mpa_pkt_len + m->m_len); err = (-EINVAL); goto err_stop_timer; } /* * copy the new data into our accumulation buffer. */ m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len])); ep->mpa_pkt_len += m->m_len; if (!m->m_next) m = m->m_nextpkt; else m = m->m_next; } while (m); m_freem(top); /* * if we don't even have the mpa message, then bail. */ if (ep->mpa_pkt_len < sizeof(*mpa)) { return 0; } mpa = (struct mpa_message *) ep->mpa_pkt; /* Validate MPA header. */ if (mpa->revision > mpa_rev) { CTR4(KTR_IW_CXGBE, "%s:pmr6 %p %d %d", __func__, ep, mpa->revision, mpa_rev); printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d, " " Received = %d\n", __func__, mpa_rev, mpa->revision); err = -EPROTO; goto err_stop_timer; } if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { CTR2(KTR_IW_CXGBE, "%s:pmr7 %p", __func__, ep); err = -EPROTO; goto err_stop_timer; } plen = ntohs(mpa->private_data_size); /* * Fail if there's too much private data. */ if (plen > MPA_MAX_PRIVATE_DATA) { CTR2(KTR_IW_CXGBE, "%s:pmr8 %p", __func__, ep); err = -EPROTO; goto err_stop_timer; } /* * If plen does not account for pkt size */ if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { CTR2(KTR_IW_CXGBE, "%s:pmr9 %p", __func__, ep); STOP_EP_TIMER(ep); err = -EPROTO; goto err_stop_timer; } ep->plen = (u8) plen; /* * If we don't have all the pdata yet, then bail. * We'll continue process when more data arrives. */ if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) { CTR2(KTR_IW_CXGBE, "%s:pmra %p", __func__, ep); return 0; } if (mpa->flags & MPA_REJECT) { CTR2(KTR_IW_CXGBE, "%s:pmrb %p", __func__, ep); err = -ECONNREFUSED; goto err_stop_timer; } /* * If we get here we have accumulated the entire mpa * start reply message including private data. And * the MPA header is valid. */ - state_set(&ep->com, FPDU_MODE); + ep->com.state = FPDU_MODE; ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; ep->mpa_attr.recv_marker_enabled = markers_enabled; ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; ep->mpa_attr.version = mpa->revision; ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; if (mpa->revision == 2) { CTR2(KTR_IW_CXGBE, "%s:pmrc %p", __func__, ep); ep->mpa_attr.enhanced_rdma_conn = mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; if (ep->mpa_attr.enhanced_rdma_conn) { CTR2(KTR_IW_CXGBE, "%s:pmrd %p", __func__, ep); mpa_v2_params = (struct mpa_v2_conn_params *) (ep->mpa_pkt + sizeof(*mpa)); resp_ird = ntohs(mpa_v2_params->ird) & MPA_V2_IRD_ORD_MASK; resp_ord = ntohs(mpa_v2_params->ord) & MPA_V2_IRD_ORD_MASK; /* * This is a double-check. Ideally, below checks are * not required since ird/ord stuff has been taken * care of in c4iw_accept_cr */ - if ((ep->ird < resp_ord) || (ep->ord > resp_ird)) { - - CTR2(KTR_IW_CXGBE, "%s:pmre %p", __func__, ep); + if (ep->ird < resp_ord) { + if (RELAXED_IRD_NEGOTIATION && resp_ord <= + ep->com.dev->rdev.adap->params.max_ordird_qp) + ep->ird = resp_ord; + else + insuff_ird = 1; + } else if (ep->ird > resp_ord) { + ep->ird = resp_ord; + } + if (ep->ord > resp_ird) { + if (RELAXED_IRD_NEGOTIATION) + ep->ord = resp_ird; + else + insuff_ird = 1; + } + if (insuff_ird) { err = -ENOMEM; ep->ird = resp_ord; ep->ord = resp_ird; - insuff_ird = 1; } if (ntohs(mpa_v2_params->ird) & MPA_V2_PEER2PEER_MODEL) { CTR2(KTR_IW_CXGBE, "%s:pmrf %p", __func__, ep); if (ntohs(mpa_v2_params->ord) & MPA_V2_RDMA_WRITE_RTR) { CTR2(KTR_IW_CXGBE, "%s:pmrg %p", __func__, ep); ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_RDMA_WRITE; } else if (ntohs(mpa_v2_params->ord) & MPA_V2_RDMA_READ_RTR) { CTR2(KTR_IW_CXGBE, "%s:pmrh %p", __func__, ep); ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; } } } } else { CTR2(KTR_IW_CXGBE, "%s:pmri %p", __func__, ep); if (mpa->revision == 1) { CTR2(KTR_IW_CXGBE, "%s:pmrj %p", __func__, ep); if (peer2peer) { CTR2(KTR_IW_CXGBE, "%s:pmrk %p", __func__, ep); ep->mpa_attr.p2p_type = p2p_type; } } } if (set_tcpinfo(ep)) { CTR2(KTR_IW_CXGBE, "%s:pmrl %p", __func__, ep); printf("%s set_tcpinfo error\n", __func__); err = -ECONNRESET; goto err; } CTR6(KTR_IW_CXGBE, "%s - crc_enabled = %d, recv_marker_enabled = %d, " "xmit_marker_enabled = %d, version = %d p2p_type = %d", __func__, ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, ep->mpa_attr.p2p_type); /* * If responder's RTR does not match with that of initiator, assign * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not * generated when moving QP to RTS state. * A TERM message will be sent after QP has moved to RTS state */ if ((ep->mpa_attr.version == 2) && peer2peer && (ep->mpa_attr.p2p_type != p2p_type)) { CTR2(KTR_IW_CXGBE, "%s:pmrm %p", __func__, ep); ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; rtr_mismatch = 1; } //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; attrs.mpa_attr = ep->mpa_attr; attrs.max_ird = ep->ird; attrs.max_ord = ep->ord; attrs.llp_stream_handle = ep; attrs.next_state = C4IW_QP_STATE_RTS; mask = C4IW_QP_ATTR_NEXT_STATE | C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; /* bind QP and TID with INIT_WR */ err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); if (err) { CTR2(KTR_IW_CXGBE, "%s:pmrn %p", __func__, ep); goto err; } /* * If responder's RTR requirement did not match with what initiator * supports, generate TERM message */ if (rtr_mismatch) { CTR2(KTR_IW_CXGBE, "%s:pmro %p", __func__, ep); printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); attrs.layer_etype = LAYER_MPA | DDP_LLP; attrs.ecode = MPA_NOMATCH_RTR; attrs.next_state = C4IW_QP_STATE_TERMINATE; + attrs.send_term = 1; err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, - C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); + C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); err = -ENOMEM; disconnect = 1; goto out; } /* * Generate TERM if initiator IRD is not sufficient for responder * provided ORD. Currently, we do the same behaviour even when * responder provided IRD is also not sufficient as regards to * initiator ORD. */ if (insuff_ird) { CTR2(KTR_IW_CXGBE, "%s:pmrp %p", __func__, ep); printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", __func__); attrs.layer_etype = LAYER_MPA | DDP_LLP; attrs.ecode = MPA_INSUFF_IRD; attrs.next_state = C4IW_QP_STATE_TERMINATE; + attrs.send_term = 1; err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, - C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); + C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); err = -ENOMEM; disconnect = 1; goto out; } goto out; err_stop_timer: STOP_EP_TIMER(ep); err: disconnect = 2; out: connect_reply_upcall(ep, err); CTR2(KTR_IW_CXGBE, "%s:pmrE %p", __func__, ep); return disconnect; } /* * process_mpa_request - process streaming mode MPA request * * Returns: * * 0 upon success indicating a connect request was delivered to the ULP * or the mpa request is incomplete but valid so far. * * 1 if a failure requires the caller to close the connection. * * 2 if a failure requires the caller to abort the connection. */ static int process_mpa_request(struct c4iw_ep *ep) { struct mpa_message *mpa; + struct mpa_v2_conn_params *mpa_v2_params; u16 plen; int flags = MSG_DONTWAIT; int rc; struct iovec iov; struct uio uio; - enum c4iw_ep_state state = state_read(&ep->com); + enum c4iw_ep_state state = ep->com.state; CTR3(KTR_IW_CXGBE, "%s: ep %p, state %s", __func__, ep, states[state]); if (state != MPA_REQ_WAIT) return 0; iov.iov_base = &ep->mpa_pkt[ep->mpa_pkt_len]; iov.iov_len = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; uio.uio_iov = &iov; uio.uio_iovcnt = 1; uio.uio_offset = 0; uio.uio_resid = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_READ; uio.uio_td = NULL; /* uio.uio_td = ep->com.thread; */ rc = soreceive(ep->com.so, NULL, &uio, NULL, NULL, &flags); if (rc == EAGAIN) return 0; else if (rc) goto err_stop_timer; KASSERT(uio.uio_offset > 0, ("%s: sorecieve on so %p read no data", __func__, ep->com.so)); ep->mpa_pkt_len += uio.uio_offset; /* * If we get more than the supported amount of private data then we must * fail this connection. XXX: check so_rcv->sb_cc, or peek with another * soreceive, or increase the size of mpa_pkt by 1 and abort if the last * byte is filled by the soreceive above. */ /* Don't even have the MPA message. Wait for more data to arrive. */ if (ep->mpa_pkt_len < sizeof(*mpa)) return 0; mpa = (struct mpa_message *) ep->mpa_pkt; /* * Validate MPA Header. */ if (mpa->revision > mpa_rev) { log(LOG_ERR, "%s: MPA version mismatch. Local = %d," " Received = %d\n", __func__, mpa_rev, mpa->revision); goto err_stop_timer; } if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) goto err_stop_timer; /* * Fail if there's too much private data. */ plen = ntohs(mpa->private_data_size); if (plen > MPA_MAX_PRIVATE_DATA) goto err_stop_timer; /* * If plen does not account for pkt size */ if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) goto err_stop_timer; ep->plen = (u8) plen; /* * If we don't have all the pdata yet, then bail. */ if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) return 0; /* * If we get here we have accumulated the entire mpa * start reply message including private data. */ ep->mpa_attr.initiator = 0; ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; ep->mpa_attr.recv_marker_enabled = markers_enabled; ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; ep->mpa_attr.version = mpa->revision; if (mpa->revision == 1) ep->tried_with_mpa_v1 = 1; ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; if (mpa->revision == 2) { ep->mpa_attr.enhanced_rdma_conn = mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; if (ep->mpa_attr.enhanced_rdma_conn) { - struct mpa_v2_conn_params *mpa_v2_params; - u16 ird, ord; - - mpa_v2_params = (void *)&ep->mpa_pkt[sizeof(*mpa)]; - ird = ntohs(mpa_v2_params->ird); - ord = ntohs(mpa_v2_params->ord); - - ep->ird = ird & MPA_V2_IRD_ORD_MASK; - ep->ord = ord & MPA_V2_IRD_ORD_MASK; - if (ird & MPA_V2_PEER2PEER_MODEL && peer2peer) { - if (ord & MPA_V2_RDMA_WRITE_RTR) { - ep->mpa_attr.p2p_type = - FW_RI_INIT_P2PTYPE_RDMA_WRITE; - } else if (ord & MPA_V2_RDMA_READ_RTR) { - ep->mpa_attr.p2p_type = - FW_RI_INIT_P2PTYPE_READ_REQ; + mpa_v2_params = (struct mpa_v2_conn_params *) + (ep->mpa_pkt + sizeof(*mpa)); + ep->ird = ntohs(mpa_v2_params->ird) & + MPA_V2_IRD_ORD_MASK; + ep->ird = min_t(u32, ep->ird, + cur_max_read_depth(ep->com.dev)); + ep->ord = ntohs(mpa_v2_params->ord) & + MPA_V2_IRD_ORD_MASK; + ep->ord = min_t(u32, ep->ord, + cur_max_read_depth(ep->com.dev)); + CTR3(KTR_IW_CXGBE, "%s initiator ird %u ord %u\n", + __func__, ep->ird, ep->ord); + if (ntohs(mpa_v2_params->ird) & MPA_V2_PEER2PEER_MODEL) + if (peer2peer) { + if (ntohs(mpa_v2_params->ord) & + MPA_V2_RDMA_WRITE_RTR) + ep->mpa_attr.p2p_type = + FW_RI_INIT_P2PTYPE_RDMA_WRITE; + else if (ntohs(mpa_v2_params->ord) & + MPA_V2_RDMA_READ_RTR) + ep->mpa_attr.p2p_type = + FW_RI_INIT_P2PTYPE_READ_REQ; } - } } } else if (mpa->revision == 1 && peer2peer) ep->mpa_attr.p2p_type = p2p_type; if (set_tcpinfo(ep)) goto err_stop_timer; CTR5(KTR_IW_CXGBE, "%s: crc_enabled = %d, recv_marker_enabled = %d, " "xmit_marker_enabled = %d, version = %d", __func__, ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); - state_set(&ep->com, MPA_REQ_RCVD); + ep->com.state = MPA_REQ_RCVD; STOP_EP_TIMER(ep); /* drive upcall */ - mutex_lock(&ep->parent_ep->com.mutex); - if (ep->parent_ep->com.state != DEAD) { + if (ep->parent_ep->com.state != DEAD) if (connect_request_upcall(ep)) - goto err_unlock_parent; - } else - goto err_unlock_parent; - mutex_unlock(&ep->parent_ep->com.mutex); + goto err_out; return 0; -err_unlock_parent: - mutex_unlock(&ep->parent_ep->com.mutex); - goto err_out; err_stop_timer: STOP_EP_TIMER(ep); err_out: return 2; } /* * Upcall from the adapter indicating data has been transmitted. * For us its just the single MPA request or reply. We can now free * the skb holding the mpa message. */ int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) { int err; struct c4iw_ep *ep = to_ep(cm_id); - CTR2(KTR_IW_CXGBE, "%s:crcB %p", __func__, ep); int abort = 0; - if ((state_read(&ep->com) == DEAD) || - (state_read(&ep->com) != MPA_REQ_RCVD)) { + mutex_lock(&ep->com.mutex); + CTR2(KTR_IW_CXGBE, "%s:crcB %p", __func__, ep); + if ((ep->com.state == DEAD) || + (ep->com.state != MPA_REQ_RCVD)) { + CTR2(KTR_IW_CXGBE, "%s:crc1 %p", __func__, ep); + mutex_unlock(&ep->com.mutex); c4iw_put_ep(&ep->com); return -ECONNRESET; } set_bit(ULP_REJECT, &ep->com.history); if (mpa_rev == 0) { CTR2(KTR_IW_CXGBE, "%s:crc2 %p", __func__, ep); abort = 1; } else { CTR2(KTR_IW_CXGBE, "%s:crc3 %p", __func__, ep); abort = send_mpa_reject(ep, pdata, pdata_len); } - stop_ep_timer(ep); + STOP_EP_TIMER(ep); err = c4iw_ep_disconnect(ep, abort != 0, GFP_KERNEL); + mutex_unlock(&ep->com.mutex); c4iw_put_ep(&ep->com); CTR3(KTR_IW_CXGBE, "%s:crc4 %p, err: %d", __func__, ep, err); return 0; } int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) { int err; - struct c4iw_qp_attributes attrs; + struct c4iw_qp_attributes attrs = {0}; enum c4iw_qp_attr_mask mask; struct c4iw_ep *ep = to_ep(cm_id); struct c4iw_dev *h = to_c4iw_dev(cm_id->device); struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); int abort = 0; + mutex_lock(&ep->com.mutex); CTR2(KTR_IW_CXGBE, "%s:cacB %p", __func__, ep); - if (state_read(&ep->com) == DEAD) { + if ((ep->com.state == DEAD) || + (ep->com.state != MPA_REQ_RCVD)) { CTR2(KTR_IW_CXGBE, "%s:cac1 %p", __func__, ep); err = -ECONNRESET; goto err_out; } - BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); BUG_ON(!qp); set_bit(ULP_ACCEPT, &ep->com.history); if ((conn_param->ord > c4iw_max_read_depth) || (conn_param->ird > c4iw_max_read_depth)) { CTR2(KTR_IW_CXGBE, "%s:cac2 %p", __func__, ep); err = -EINVAL; goto err_abort; } if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { CTR2(KTR_IW_CXGBE, "%s:cac3 %p", __func__, ep); if (conn_param->ord > ep->ird) { - - CTR2(KTR_IW_CXGBE, "%s:cac4 %p", __func__, ep); - ep->ird = conn_param->ird; - ep->ord = conn_param->ord; - send_mpa_reject(ep, conn_param->private_data, - conn_param->private_data_len); - err = -ENOMEM; - goto err_abort; - } - - if (conn_param->ird > ep->ord) { - - CTR2(KTR_IW_CXGBE, "%s:cac5 %p", __func__, ep); - - if (!ep->ord) { - - CTR2(KTR_IW_CXGBE, "%s:cac6 %p", __func__, ep); - conn_param->ird = 1; + if (RELAXED_IRD_NEGOTIATION) { + conn_param->ord = ep->ird; + } else { + ep->ird = conn_param->ird; + ep->ord = conn_param->ord; + send_mpa_reject(ep, conn_param->private_data, + conn_param->private_data_len); + err = -ENOMEM; + goto err_abort; } - else { - CTR2(KTR_IW_CXGBE, "%s:cac7 %p", __func__, ep); + } + if (conn_param->ird < ep->ord) { + if (RELAXED_IRD_NEGOTIATION && + ep->ord <= h->rdev.adap->params.max_ordird_qp) { + conn_param->ird = ep->ord; + } else { err = -ENOMEM; goto err_abort; } } - } ep->ird = conn_param->ird; ep->ord = conn_param->ord; - if (ep->mpa_attr.version != 2) { - - CTR2(KTR_IW_CXGBE, "%s:cac8 %p", __func__, ep); - - if (peer2peer && ep->ird == 0) { - - CTR2(KTR_IW_CXGBE, "%s:cac9 %p", __func__, ep); + if (ep->mpa_attr.version == 1) { + if (peer2peer && ep->ird == 0) ep->ird = 1; - } + } else { + if (peer2peer && + (ep->mpa_attr.p2p_type != FW_RI_INIT_P2PTYPE_DISABLED) && + (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) && ep->ird == 0) + ep->ird = 1; } + CTR4(KTR_IW_CXGBE, "%s %d ird %d ord %d\n", __func__, __LINE__, + ep->ird, ep->ord); ep->com.cm_id = cm_id; ref_cm_id(&ep->com); ep->com.qp = qp; ref_qp(ep); //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; /* bind QP to EP and move to RTS */ attrs.mpa_attr = ep->mpa_attr; attrs.max_ird = ep->ird; attrs.max_ord = ep->ord; attrs.llp_stream_handle = ep; attrs.next_state = C4IW_QP_STATE_RTS; /* bind QP and TID with INIT_WR */ mask = C4IW_QP_ATTR_NEXT_STATE | C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); - if (err) { - - CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); + CTR3(KTR_IW_CXGBE, "%s:caca %p, err: %d", __func__, ep, err); goto err_defef_cm_id; } + err = send_mpa_reply(ep, conn_param->private_data, conn_param->private_data_len); - if (err) { - - CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); + CTR3(KTR_IW_CXGBE, "%s:cacb %p, err: %d", __func__, ep, err); goto err_defef_cm_id; } - state_set(&ep->com, FPDU_MODE); + ep->com.state = FPDU_MODE; established_upcall(ep); + mutex_unlock(&ep->com.mutex); c4iw_put_ep(&ep->com); CTR2(KTR_IW_CXGBE, "%s:cacE %p", __func__, ep); return 0; err_defef_cm_id: deref_cm_id(&ep->com); err_abort: abort = 1; err_out: if (abort) c4iw_ep_disconnect(ep, 1, GFP_KERNEL); + mutex_unlock(&ep->com.mutex); c4iw_put_ep(&ep->com); CTR2(KTR_IW_CXGBE, "%s:cacE err %p", __func__, ep); return err; } +static int +c4iw_sock_create(struct sockaddr_storage *laddr, struct socket **so) +{ + int ret; + int size; + struct socket *sock = NULL; + ret = sock_create_kern(laddr->ss_family, + SOCK_STREAM, IPPROTO_TCP, &sock); + if (ret) { + CTR2(KTR_IW_CXGBE, "%s:Failed to create TCP socket. err %d", + __func__, ret); + return ret; + } + ret = sobind(sock, (struct sockaddr *)laddr, curthread); + if (ret) { + CTR2(KTR_IW_CXGBE, "%s:Failed to bind socket. err %p", + __func__, ret); + sock_release(sock); + return ret; + } + + size = laddr->ss_family == AF_INET6 ? + sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in); + ret = sock_getname(sock, (struct sockaddr *)laddr, &size, 0); + if (ret) { + CTR2(KTR_IW_CXGBE, "%s:sock_getname failed. err %p", + __func__, ret); + sock_release(sock); + return ret; + } + + *so = sock; + return 0; +} + int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) { int err = 0; struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); struct c4iw_ep *ep = NULL; - struct nhop4_extended nh4; + struct ifnet *nh_ifp; /* Logical egress interface */ CTR2(KTR_IW_CXGBE, "%s:ccB %p", __func__, cm_id); + if ((conn_param->ord > c4iw_max_read_depth) || (conn_param->ird > c4iw_max_read_depth)) { CTR2(KTR_IW_CXGBE, "%s:cc1 %p", __func__, cm_id); err = -EINVAL; goto out; } ep = alloc_ep(sizeof(*ep), GFP_KERNEL); + init_timer(&ep->timer); ep->plen = conn_param->private_data_len; if (ep->plen) { CTR2(KTR_IW_CXGBE, "%s:cc3 %p", __func__, ep); memcpy(ep->mpa_pkt + sizeof(struct mpa_message), conn_param->private_data, ep->plen); } ep->ird = conn_param->ird; ep->ord = conn_param->ord; if (peer2peer && ep->ord == 0) { CTR2(KTR_IW_CXGBE, "%s:cc4 %p", __func__, ep); ep->ord = 1; } ep->com.dev = dev; ep->com.cm_id = cm_id; ref_cm_id(&ep->com); ep->com.qp = get_qhp(dev, conn_param->qpn); if (!ep->com.qp) { CTR2(KTR_IW_CXGBE, "%s:cc5 %p", __func__, ep); err = -EINVAL; - goto fail2; + goto fail; } ref_qp(ep); ep->com.thread = curthread; - ep->com.so = cm_id->so; - /* find a route */ - err = find_route( - cm_id->local_addr.sin_addr.s_addr, - cm_id->remote_addr.sin_addr.s_addr, - cm_id->local_addr.sin_port, - cm_id->remote_addr.sin_port, 0, &nh4); - + err = get_ifnet_from_raddr(&cm_id->remote_addr, &nh_ifp); if (err) { CTR2(KTR_IW_CXGBE, "%s:cc7 %p", __func__, ep); printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); - err = -EHOSTUNREACH; - goto fail2; + err = EHOSTUNREACH; + return err; } - if (!(nh4.nh_ifp->if_capenable & IFCAP_TOE) || - TOEDEV(nh4.nh_ifp) == NULL) { + if (!(nh_ifp->if_capenable & IFCAP_TOE) || + TOEDEV(nh_ifp) == NULL) { err = -ENOPROTOOPT; - goto fail3; + goto fail; } - fib4_free_nh_ext(RT_DEFAULT_FIB, &nh4); - - setiwsockopt(cm_id->so); - state_set(&ep->com, CONNECTING); + ep->com.state = CONNECTING; ep->tos = 0; ep->com.local_addr = cm_id->local_addr; ep->com.remote_addr = cm_id->remote_addr; + + err = c4iw_sock_create(&cm_id->local_addr, &ep->com.so); + if (err) + goto fail; + + setiwsockopt(ep->com.so); err = -soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr, ep->com.thread); - if (!err) { - init_iwarp_socket(cm_id->so, &ep->com); + init_iwarp_socket(ep->com.so, &ep->com); goto out; - } else { - goto fail2; - } + } else + goto fail_free_so; -fail3: - fib4_free_nh_ext(RT_DEFAULT_FIB, &nh4); -fail2: +fail_free_so: + sock_release(ep->com.so); +fail: deref_cm_id(&ep->com); c4iw_put_ep(&ep->com); - ep = NULL; /* CTR shouldn't display already-freed ep. */ + ep = NULL; out: - CTR2(KTR_IW_CXGBE, "%s:ccE %p", __func__, ep); + CTR2(KTR_IW_CXGBE, "%s:ccE ret:%d", __func__, err); return err; } /* - * iwcm->create_listen_ep. Returns -errno on failure. + * iwcm->create_listen. Returns -errno on failure. */ int -c4iw_create_listen_ep(struct iw_cm_id *cm_id, int backlog) +c4iw_create_listen(struct iw_cm_id *cm_id, int backlog) { struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); - struct c4iw_listen_ep *ep; - struct socket *so = cm_id->so; + struct c4iw_listen_ep *lep = NULL; + struct listen_port_info *port_info = NULL; + int rc = 0; - ep = alloc_ep(sizeof(*ep), GFP_KERNEL); - ep->com.cm_id = cm_id; - ref_cm_id(&ep->com); - ep->com.dev = dev; - ep->backlog = backlog; - ep->com.local_addr = cm_id->local_addr; - ep->com.thread = curthread; - state_set(&ep->com, LISTEN); - ep->com.so = so; + CTR3(KTR_IW_CXGBE, "%s: cm_id %p, backlog %s", __func__, cm_id, + backlog); + lep = alloc_ep(sizeof(*lep), GFP_KERNEL); + lep->com.cm_id = cm_id; + ref_cm_id(&lep->com); + lep->com.dev = dev; + lep->backlog = backlog; + lep->com.local_addr = cm_id->local_addr; + lep->com.thread = curthread; + cm_id->provider_data = lep; + lep->com.state = LISTEN; - cm_id->provider_data = ep; - return (0); + /* In case of INDADDR_ANY, ibcore creates cmid for each device and + * invokes iw_cxgbe listener callbacks assuming that iw_cxgbe creates + * HW listeners for each device seperately. But toecore expects single + * solisten() call with INADDR_ANY address to create HW listeners on + * all devices for a given port number. So iw_cxgbe driver calls + * solisten() only once for INADDR_ANY(usually done at first time + * listener callback from ibcore). And all the subsequent INADDR_ANY + * listener callbacks from ibcore(for the same port address) do not + * invoke solisten() as first listener callback has already created + * listeners for all other devices(via solisten). + */ + if (c4iw_any_addr((struct sockaddr *)&lep->com.local_addr)) { + port_info = add_ep_to_listenlist(lep); + /* skip solisten() if refcnt > 1, as the listeners were + * alredy created by 'Master lep' + */ + if (port_info->refcnt > 1) { + /* As there will be only one listener socket for a TCP + * port, copy Master lep's socket pointer to other lep's + * that are belonging to same TCP port. + */ + struct c4iw_listen_ep *head_lep = + container_of(port_info->lep_list.next, + struct c4iw_listen_ep, listen_ep_list); + lep->com.so = head_lep->com.so; + goto out; + } + } + rc = c4iw_sock_create(&cm_id->local_addr, &lep->com.so); + if (rc) { + CTR2(KTR_IW_CXGBE, "%s:Failed to create socket. err %d", + __func__, rc); + goto fail; + } + + rc = solisten(lep->com.so, backlog, curthread); + if (rc) { + CTR3(KTR_IW_CXGBE, "%s:Failed to listen on sock:%p. err %d", + __func__, lep->com.so, rc); + goto fail_free_so; + } + init_iwarp_socket(lep->com.so, &lep->com); +out: + return 0; + +fail_free_so: + sock_release(lep->com.so); +fail: + if (port_info) + rem_ep_from_listenlist(lep); + deref_cm_id(&lep->com); + c4iw_put_ep(&lep->com); + return rc; } -void -c4iw_destroy_listen_ep(struct iw_cm_id *cm_id) +int +c4iw_destroy_listen(struct iw_cm_id *cm_id) { - struct c4iw_listen_ep *ep = to_listen_ep(cm_id); + struct c4iw_listen_ep *lep = to_listen_ep(cm_id); - CTR4(KTR_IW_CXGBE, "%s: cm_id %p, so %p, state %s", __func__, cm_id, - cm_id->so, states[ep->com.state]); + mutex_lock(&lep->com.mutex); + CTR3(KTR_IW_CXGBE, "%s: cm_id %p, state %s", __func__, cm_id, + states[lep->com.state]); - state_set(&ep->com, DEAD); - deref_cm_id(&ep->com); - c4iw_put_ep(&ep->com); + lep->com.state = DEAD; + if (c4iw_any_addr((struct sockaddr *)&lep->com.local_addr)) { + /* if no refcount then close listen socket */ + if (!rem_ep_from_listenlist(lep)) + close_socket(lep->com.so); + } else + close_socket(lep->com.so); + deref_cm_id(&lep->com); + mutex_unlock(&lep->com.mutex); + c4iw_put_ep(&lep->com); + return 0; +} - return; +int __c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) +{ + int ret; + mutex_lock(&ep->com.mutex); + ret = c4iw_ep_disconnect(ep, abrupt, gfp); + mutex_unlock(&ep->com.mutex); + return ret; } int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) { int ret = 0; int close = 0; int fatal = 0; struct c4iw_rdev *rdev; - mutex_lock(&ep->com.mutex); CTR2(KTR_IW_CXGBE, "%s:cedB %p", __func__, ep); rdev = &ep->com.dev->rdev; if (c4iw_fatal_error(rdev)) { CTR2(KTR_IW_CXGBE, "%s:ced1 %p", __func__, ep); fatal = 1; close_complete_upcall(ep, -ECONNRESET); + send_abort(ep); ep->com.state = DEAD; } CTR3(KTR_IW_CXGBE, "%s:ced2 %p %s", __func__, ep, states[ep->com.state]); + /* + * Ref the ep here in case we have fatal errors causing the + * ep to be released and freed. + */ + c4iw_get_ep(&ep->com); switch (ep->com.state) { case MPA_REQ_WAIT: case MPA_REQ_SENT: case MPA_REQ_RCVD: case MPA_REP_SENT: case FPDU_MODE: close = 1; if (abrupt) ep->com.state = ABORTING; else { ep->com.state = CLOSING; START_EP_TIMER(ep); } set_bit(CLOSE_SENT, &ep->com.flags); break; case CLOSING: if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { close = 1; if (abrupt) { STOP_EP_TIMER(ep); ep->com.state = ABORTING; } else ep->com.state = MORIBUND; } break; case MORIBUND: case ABORTING: case DEAD: CTR3(KTR_IW_CXGBE, "%s ignoring disconnect ep %p state %u", __func__, ep, ep->com.state); break; default: BUG(); break; } - mutex_unlock(&ep->com.mutex); if (close) { CTR2(KTR_IW_CXGBE, "%s:ced3 %p", __func__, ep); if (abrupt) { CTR2(KTR_IW_CXGBE, "%s:ced4 %p", __func__, ep); set_bit(EP_DISC_ABORT, &ep->com.history); close_complete_upcall(ep, -ECONNRESET); ret = send_abort(ep); if (ret) fatal = 1; } else { CTR2(KTR_IW_CXGBE, "%s:ced5 %p", __func__, ep); set_bit(EP_DISC_CLOSE, &ep->com.history); if (!ep->parent_ep) - __state_set(&ep->com, MORIBUND); + ep->com.state = MORIBUND; sodisconnect(ep->com.so); } } if (fatal) { set_bit(EP_DISC_FAIL, &ep->com.history); if (!abrupt) { STOP_EP_TIMER(ep); close_complete_upcall(ep, -EIO); } if (ep->com.qp) { - struct c4iw_qp_attributes attrs; + struct c4iw_qp_attributes attrs = {0}; attrs.next_state = C4IW_QP_STATE_ERROR; ret = c4iw_modify_qp(ep->com.dev, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); if (ret) { CTR2(KTR_IW_CXGBE, "%s:ced7 %p", __func__, ep); printf("%s - qp <- error failed!\n", __func__); } } release_ep_resources(ep); ep->com.state = DEAD; CTR2(KTR_IW_CXGBE, "%s:ced6 %p", __func__, ep); } + c4iw_put_ep(&ep->com); CTR2(KTR_IW_CXGBE, "%s:cedE %p", __func__, ep); return ret; } #ifdef C4IW_EP_REDIRECT int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new, struct l2t_entry *l2t) { struct c4iw_ep *ep = ctx; if (ep->dst != old) return 0; PDBG("%s ep %p redirect to dst %p l2t %p\n", __func__, ep, new, l2t); dst_hold(new); cxgb4_l2t_release(ep->l2t); ep->l2t = l2t; dst_release(old); ep->dst = new; return 1; } #endif static void ep_timeout(unsigned long arg) { struct c4iw_ep *ep = (struct c4iw_ep *)arg; if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { /* * Only insert if it is not already on the list. */ if (!(ep->com.ep_events & C4IW_EVENT_TIMEOUT)) { CTR2(KTR_IW_CXGBE, "%s:et1 %p", __func__, ep); add_ep_to_req_list(ep, C4IW_EVENT_TIMEOUT); } } } static int fw6_wr_rpl(struct adapter *sc, const __be64 *rpl) { uint64_t val = be64toh(*rpl); int ret; struct c4iw_wr_wait *wr_waitp; ret = (int)((val >> 8) & 0xff); wr_waitp = (struct c4iw_wr_wait *)rpl[1]; CTR3(KTR_IW_CXGBE, "%s wr_waitp %p ret %u", __func__, wr_waitp, ret); if (wr_waitp) c4iw_wake_up(wr_waitp, ret ? -ret : 0); return (0); } static int fw6_cqe_handler(struct adapter *sc, const __be64 *rpl) { struct cqe_list_entry *cle; unsigned long flag; cle = malloc(sizeof(*cle), M_CXGBE, M_NOWAIT); cle->rhp = sc->iwarp_softc; cle->err_cqe = *(const struct t4_cqe *)(&rpl[0]); spin_lock_irqsave(&err_cqe_lock, flag); list_add_tail(&cle->entry, &err_cqe_list); queue_work(c4iw_taskq, &c4iw_task); spin_unlock_irqrestore(&err_cqe_lock, flag); return (0); } static int process_terminate(struct c4iw_ep *ep) { - struct c4iw_qp_attributes attrs; + struct c4iw_qp_attributes attrs = {0}; CTR2(KTR_IW_CXGBE, "%s:tB %p %d", __func__, ep); if (ep && ep->com.qp) { printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", ep->hwtid, ep->com.qp->wq.sq.qid); attrs.next_state = C4IW_QP_STATE_TERMINATE; c4iw_modify_qp(ep->com.dev, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); } else printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", ep->hwtid); CTR2(KTR_IW_CXGBE, "%s:tE %p %d", __func__, ep); return 0; } int __init c4iw_cm_init(void) { t4_register_cpl_handler(CPL_RDMA_TERMINATE, terminate); t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, fw6_wr_rpl); t4_register_fw_msg_handler(FW6_TYPE_CQE, fw6_cqe_handler); t4_register_an_handler(c4iw_ev_handler); TAILQ_INIT(&req_list); spin_lock_init(&req_lock); INIT_LIST_HEAD(&err_cqe_list); spin_lock_init(&err_cqe_lock); INIT_WORK(&c4iw_task, process_req); c4iw_taskq = create_singlethread_workqueue("iw_cxgbe"); if (!c4iw_taskq) return -ENOMEM; return 0; } void __exit c4iw_cm_term(void) { WARN_ON(!TAILQ_EMPTY(&req_list)); WARN_ON(!list_empty(&err_cqe_list)); flush_workqueue(c4iw_taskq); destroy_workqueue(c4iw_taskq); t4_register_cpl_handler(CPL_RDMA_TERMINATE, NULL); t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, NULL); t4_register_fw_msg_handler(FW6_TYPE_CQE, NULL); t4_register_an_handler(NULL); } #endif Index: projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/cq.c =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/cq.c (revision 331317) +++ projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/cq.c (revision 331318) @@ -1,941 +1,1056 @@ /* * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #ifdef TCP_OFFLOAD #include #include #include #include #include #include #include #include #include #include #include "iw_cxgbe.h" #include "user.h" static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, struct c4iw_dev_ucontext *uctx) { struct adapter *sc = rdev->adap; + struct c4iw_dev *rhp = rdev_to_c4iw_dev(rdev); struct fw_ri_res_wr *res_wr; struct fw_ri_res *res; int wr_len; struct c4iw_wr_wait wr_wait; struct wrqe *wr; wr_len = sizeof *res_wr + sizeof *res; wr = alloc_wrqe(wr_len, &sc->sge.mgmtq); if (wr == NULL) return (0); res_wr = wrtod(wr); memset(res_wr, 0, wr_len); res_wr->op_nres = cpu_to_be32( V_FW_WR_OP(FW_RI_RES_WR) | V_FW_RI_RES_WR_NRES(1) | F_FW_WR_COMPL); res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16)); res_wr->cookie = (unsigned long) &wr_wait; res = res_wr->res; res->u.cq.restype = FW_RI_RES_TYPE_CQ; res->u.cq.op = FW_RI_RES_OP_RESET; res->u.cq.iqid = cpu_to_be32(cq->cqid); c4iw_init_wr_wait(&wr_wait); t4_wrq_tx(sc, wr); - c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__); + c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, NULL, __func__); kfree(cq->sw_queue); - contigfree(cq->queue, cq->memsize, M_DEVBUF); + dma_free_coherent(rhp->ibdev.dma_device, + cq->memsize, cq->queue, + dma_unmap_addr(cq, mapping)); c4iw_put_cqid(rdev, cq->cqid, uctx); return 0; } static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, struct c4iw_dev_ucontext *uctx) { struct adapter *sc = rdev->adap; + struct c4iw_dev *rhp = rdev_to_c4iw_dev(rdev); struct fw_ri_res_wr *res_wr; struct fw_ri_res *res; int wr_len; int user = (uctx != &rdev->uctx); struct c4iw_wr_wait wr_wait; int ret; struct wrqe *wr; + u64 cq_bar2_qoffset = 0; cq->cqid = c4iw_get_cqid(rdev, uctx); if (!cq->cqid) { ret = -ENOMEM; goto err1; } if (!user) { cq->sw_queue = kzalloc(cq->memsize, GFP_KERNEL); if (!cq->sw_queue) { ret = -ENOMEM; goto err2; } } - - cq->queue = contigmalloc(cq->memsize, M_DEVBUF, M_NOWAIT, 0ul, ~0ul, - PAGE_SIZE, 0); - if (cq->queue) - cq->dma_addr = vtophys(cq->queue); - else { + cq->queue = dma_alloc_coherent(rhp->ibdev.dma_device, cq->memsize, + &cq->dma_addr, GFP_KERNEL); + if (!cq->queue) { ret = -ENOMEM; - goto err3; + goto err3; } - - pci_unmap_addr_set(cq, mapping, cq->dma_addr); + dma_unmap_addr_set(cq, mapping, cq->dma_addr); memset(cq->queue, 0, cq->memsize); /* build fw_ri_res_wr */ wr_len = sizeof *res_wr + sizeof *res; wr = alloc_wrqe(wr_len, &sc->sge.mgmtq); if (wr == NULL) return (0); res_wr = wrtod(wr); memset(res_wr, 0, wr_len); res_wr->op_nres = cpu_to_be32( V_FW_WR_OP(FW_RI_RES_WR) | V_FW_RI_RES_WR_NRES(1) | F_FW_WR_COMPL); res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16)); res_wr->cookie = (unsigned long) &wr_wait; res = res_wr->res; res->u.cq.restype = FW_RI_RES_TYPE_CQ; res->u.cq.op = FW_RI_RES_OP_WRITE; res->u.cq.iqid = cpu_to_be32(cq->cqid); //Fixme: Always use first queue id for IQANDSTINDEX. Linux does the same. res->u.cq.iqandst_to_iqandstindex = cpu_to_be32( V_FW_RI_RES_WR_IQANUS(0) | V_FW_RI_RES_WR_IQANUD(1) | F_FW_RI_RES_WR_IQANDST | V_FW_RI_RES_WR_IQANDSTINDEX(sc->sge.ofld_rxq[0].iq.abs_id)); res->u.cq.iqdroprss_to_iqesize = cpu_to_be16( F_FW_RI_RES_WR_IQDROPRSS | V_FW_RI_RES_WR_IQPCIECH(2) | V_FW_RI_RES_WR_IQINTCNTTHRESH(0) | F_FW_RI_RES_WR_IQO | V_FW_RI_RES_WR_IQESIZE(1)); res->u.cq.iqsize = cpu_to_be16(cq->size); res->u.cq.iqaddr = cpu_to_be64(cq->dma_addr); c4iw_init_wr_wait(&wr_wait); t4_wrq_tx(sc, wr); CTR2(KTR_IW_CXGBE, "%s wait_event wr_wait %p", __func__, &wr_wait); - ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__); + ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, NULL, __func__); if (ret) goto err4; cq->gen = 1; - cq->gts = (void *)((unsigned long)rman_get_virtual(sc->regs_res) + - sc->sge_gts_reg); cq->rdev = rdev; - if (user) { - cq->ugts = (u64)((char*)rman_get_virtual(sc->udbs_res) + - (cq->cqid << rdev->cqshift)); - cq->ugts &= PAGE_MASK; - CTR5(KTR_IW_CXGBE, - "%s: UGTS %p cqid %x cqshift %d page_mask %x", __func__, - cq->ugts, cq->cqid, rdev->cqshift, PAGE_MASK); - } + /* Determine the BAR2 queue offset and qid. */ + t4_bar2_sge_qregs(rdev->adap, cq->cqid, T4_BAR2_QTYPE_INGRESS, user, + &cq_bar2_qoffset, &cq->bar2_qid); + + /* If user mapping then compute the page-aligned physical + * address for mapping. + */ + if (user) + cq->bar2_pa = (rdev->bar2_pa + cq_bar2_qoffset) & PAGE_MASK; + else + cq->bar2_va = (void __iomem *)((u64)rdev->bar2_kva + + cq_bar2_qoffset); + return 0; err4: - contigfree(cq->queue, cq->memsize, M_DEVBUF); + dma_free_coherent(rhp->ibdev.dma_device, cq->memsize, cq->queue, + dma_unmap_addr(cq, mapping)); err3: kfree(cq->sw_queue); err2: c4iw_put_cqid(rdev, cq->cqid, uctx); err1: return ret; } static void insert_recv_cqe(struct t4_wq *wq, struct t4_cq *cq) { struct t4_cqe cqe; CTR5(KTR_IW_CXGBE, "%s wq %p cq %p sw_cidx %u sw_pidx %u", __func__, wq, cq, cq->sw_cidx, cq->sw_pidx); memset(&cqe, 0, sizeof(cqe)); cqe.header = cpu_to_be32(V_CQE_STATUS(T4_ERR_SWFLUSH) | V_CQE_OPCODE(FW_RI_SEND) | V_CQE_TYPE(0) | V_CQE_SWCQE(1) | V_CQE_QPID(wq->sq.qid)); cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen)); cq->sw_queue[cq->sw_pidx] = cqe; t4_swcq_produce(cq); } int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count) { int flushed = 0; int in_use = wq->rq.in_use - count; BUG_ON(in_use < 0); CTR5(KTR_IW_CXGBE, "%s wq %p cq %p rq.in_use %u skip count %u", __func__, wq, cq, wq->rq.in_use, count); while (in_use--) { insert_recv_cqe(wq, cq); flushed++; } return flushed; } static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq, struct t4_swsqe *swcqe) { struct t4_cqe cqe; CTR5(KTR_IW_CXGBE, "%s wq %p cq %p sw_cidx %u sw_pidx %u", __func__, wq, cq, cq->sw_cidx, cq->sw_pidx); memset(&cqe, 0, sizeof(cqe)); cqe.header = cpu_to_be32(V_CQE_STATUS(T4_ERR_SWFLUSH) | V_CQE_OPCODE(swcqe->opcode) | V_CQE_TYPE(1) | V_CQE_SWCQE(1) | V_CQE_QPID(wq->sq.qid)); CQE_WRID_SQ_IDX(&cqe) = swcqe->idx; cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen)); cq->sw_queue[cq->sw_pidx] = cqe; t4_swcq_produce(cq); } -int c4iw_flush_sq(struct t4_wq *wq, struct t4_cq *cq, int count) +static void advance_oldest_read(struct t4_wq *wq); + +int c4iw_flush_sq(struct c4iw_qp *qhp) { int flushed = 0; - struct t4_swsqe *swsqe = &wq->sq.sw_sq[wq->sq.cidx + count]; - int in_use = wq->sq.in_use - count; + struct t4_wq *wq = &qhp->wq; + struct c4iw_cq *chp = to_c4iw_cq(qhp->ibqp.send_cq); + struct t4_cq *cq = &chp->cq; + int idx; + struct t4_swsqe *swsqe; - BUG_ON(in_use < 0); - while (in_use--) { - swsqe->signaled = 0; + if (wq->sq.flush_cidx == -1) + wq->sq.flush_cidx = wq->sq.cidx; + idx = wq->sq.flush_cidx; + BUG_ON(idx >= wq->sq.size); + while (idx != wq->sq.pidx) { + swsqe = &wq->sq.sw_sq[idx]; + BUG_ON(swsqe->flushed); + swsqe->flushed = 1; insert_sq_cqe(wq, cq, swsqe); - swsqe++; - if (swsqe == (wq->sq.sw_sq + wq->sq.size)) - swsqe = wq->sq.sw_sq; + if (wq->sq.oldest_read == swsqe) { + BUG_ON(swsqe->opcode != FW_RI_READ_REQ); + advance_oldest_read(wq); + } flushed++; + if (++idx == wq->sq.size) + idx = 0; } + wq->sq.flush_cidx += flushed; + if (wq->sq.flush_cidx >= wq->sq.size) + wq->sq.flush_cidx -= wq->sq.size; return flushed; } +static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq) +{ + struct t4_swsqe *swsqe; + int cidx; + + if (wq->sq.flush_cidx == -1) + wq->sq.flush_cidx = wq->sq.cidx; + cidx = wq->sq.flush_cidx; + BUG_ON(cidx > wq->sq.size); + + while (cidx != wq->sq.pidx) { + swsqe = &wq->sq.sw_sq[cidx]; + if (!swsqe->signaled) { + if (++cidx == wq->sq.size) + cidx = 0; + } else if (swsqe->complete) { + + BUG_ON(swsqe->flushed); + + /* + * Insert this completed cqe into the swcq. + */ + CTR3(KTR_IW_CXGBE, + "%s moving cqe into swcq sq idx %u cq idx %u\n", + __func__, cidx, cq->sw_pidx); + swsqe->cqe.header |= htonl(V_CQE_SWCQE(1)); + cq->sw_queue[cq->sw_pidx] = swsqe->cqe; + t4_swcq_produce(cq); + swsqe->flushed = 1; + if (++cidx == wq->sq.size) + cidx = 0; + wq->sq.flush_cidx = cidx; + } else + break; + } +} + +static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe, + struct t4_cqe *read_cqe) +{ + read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx; + read_cqe->len = htonl(wq->sq.oldest_read->read_len); + read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(hw_cqe)) | + V_CQE_SWCQE(SW_CQE(hw_cqe)) | + V_CQE_OPCODE(FW_RI_READ_REQ) | + V_CQE_TYPE(1)); + read_cqe->bits_type_ts = hw_cqe->bits_type_ts; +} + +static void advance_oldest_read(struct t4_wq *wq) +{ + + u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1; + + if (rptr == wq->sq.size) + rptr = 0; + while (rptr != wq->sq.pidx) { + wq->sq.oldest_read = &wq->sq.sw_sq[rptr]; + + if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ) + return; + if (++rptr == wq->sq.size) + rptr = 0; + } + wq->sq.oldest_read = NULL; +} + /* * Move all CQEs from the HWCQ into the SWCQ. + * Deal with out-of-order and/or completions that complete + * prior unsignalled WRs. */ -void c4iw_flush_hw_cq(struct t4_cq *cq) +void c4iw_flush_hw_cq(struct c4iw_cq *chp) { - struct t4_cqe *cqe = NULL, *swcqe; + struct t4_cqe *hw_cqe, *swcqe, read_cqe; + struct c4iw_qp *qhp; + struct t4_swsqe *swsqe; int ret; - CTR3(KTR_IW_CXGBE, "%s cq %p cqid 0x%x", __func__, cq, cq->cqid); - ret = t4_next_hw_cqe(cq, &cqe); + CTR3(KTR_IW_CXGBE, "%s cq %p cqid 0x%x", __func__, &chp->cq, + chp->cq.cqid); + ret = t4_next_hw_cqe(&chp->cq, &hw_cqe); + + /* + * This logic is similar to poll_cq(), but not quite the same + * unfortunately. Need to move pertinent HW CQEs to the SW CQ but + * also do any translation magic that poll_cq() normally does. + */ while (!ret) { - CTR3(KTR_IW_CXGBE, "%s flushing hwcq cidx 0x%x swcq pidx 0x%x", - __func__, cq->cidx, cq->sw_pidx); - swcqe = &cq->sw_queue[cq->sw_pidx]; - *swcqe = *cqe; - swcqe->header |= cpu_to_be32(V_CQE_SWCQE(1)); - t4_swcq_produce(cq); - t4_hwcq_consume(cq); - ret = t4_next_hw_cqe(cq, &cqe); + qhp = get_qhp(chp->rhp, CQE_QPID(hw_cqe)); + + /* + * drop CQEs with no associated QP + */ + if (qhp == NULL) + goto next_cqe; + + if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) + goto next_cqe; + + if (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP) { + + /* If we have reached here because of async + * event or other error, and have egress error + * then drop + */ + if (CQE_TYPE(hw_cqe) == 1) + goto next_cqe; + + /* drop peer2peer RTR reads. + */ + if (CQE_WRID_STAG(hw_cqe) == 1) + goto next_cqe; + + /* + * Eat completions for unsignaled read WRs. + */ + if (!qhp->wq.sq.oldest_read->signaled) { + advance_oldest_read(&qhp->wq); + goto next_cqe; + } + + /* + * Don't write to the HWCQ, create a new read req CQE + * in local memory and move it into the swcq. + */ + create_read_req_cqe(&qhp->wq, hw_cqe, &read_cqe); + hw_cqe = &read_cqe; + advance_oldest_read(&qhp->wq); + } + + /* if its a SQ completion, then do the magic to move all the + * unsignaled and now in-order completions into the swcq. + */ + if (SQ_TYPE(hw_cqe)) { + swsqe = &qhp->wq.sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)]; + swsqe->cqe = *hw_cqe; + swsqe->complete = 1; + flush_completed_wrs(&qhp->wq, &chp->cq); + } else { + swcqe = &chp->cq.sw_queue[chp->cq.sw_pidx]; + *swcqe = *hw_cqe; + swcqe->header |= cpu_to_be32(V_CQE_SWCQE(1)); + t4_swcq_produce(&chp->cq); + } +next_cqe: + t4_hwcq_consume(&chp->cq); + ret = t4_next_hw_cqe(&chp->cq, &hw_cqe); } } static int cqe_completes_wr(struct t4_cqe *cqe, struct t4_wq *wq) { if (CQE_OPCODE(cqe) == FW_RI_TERMINATE) return 0; if ((CQE_OPCODE(cqe) == FW_RI_RDMA_WRITE) && RQ_TYPE(cqe)) return 0; if ((CQE_OPCODE(cqe) == FW_RI_READ_RESP) && SQ_TYPE(cqe)) return 0; if (CQE_SEND_OPCODE(cqe) && RQ_TYPE(cqe) && t4_rq_empty(wq)) return 0; return 1; } -void c4iw_count_scqes(struct t4_cq *cq, struct t4_wq *wq, int *count) -{ - struct t4_cqe *cqe; - u32 ptr; - - *count = 0; - ptr = cq->sw_cidx; - while (ptr != cq->sw_pidx) { - cqe = &cq->sw_queue[ptr]; - if ((SQ_TYPE(cqe) || ((CQE_OPCODE(cqe) == FW_RI_READ_RESP) && - wq->sq.oldest_read)) && - (CQE_QPID(cqe) == wq->sq.qid)) - (*count)++; - if (++ptr == cq->size) - ptr = 0; - } - CTR3(KTR_IW_CXGBE, "%s cq %p count %d", __func__, cq, *count); -} - void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count) { struct t4_cqe *cqe; u32 ptr; *count = 0; CTR2(KTR_IW_CXGBE, "%s count zero %d", __func__, *count); ptr = cq->sw_cidx; while (ptr != cq->sw_pidx) { cqe = &cq->sw_queue[ptr]; if (RQ_TYPE(cqe) && (CQE_OPCODE(cqe) != FW_RI_READ_RESP) && (CQE_QPID(cqe) == wq->sq.qid) && cqe_completes_wr(cqe, wq)) (*count)++; if (++ptr == cq->size) ptr = 0; } CTR3(KTR_IW_CXGBE, "%s cq %p count %d", __func__, cq, *count); } -static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq) -{ - struct t4_swsqe *swsqe; - u16 ptr = wq->sq.cidx; - int count = wq->sq.in_use; - int unsignaled = 0; - - swsqe = &wq->sq.sw_sq[ptr]; - while (count--) - if (!swsqe->signaled) { - if (++ptr == wq->sq.size) - ptr = 0; - swsqe = &wq->sq.sw_sq[ptr]; - unsignaled++; - } else if (swsqe->complete) { - - /* - * Insert this completed cqe into the swcq. - */ - CTR3(KTR_IW_CXGBE, - "%s moving cqe into swcq sq idx %u cq idx %u", - __func__, ptr, cq->sw_pidx); - swsqe->cqe.header |= htonl(V_CQE_SWCQE(1)); - cq->sw_queue[cq->sw_pidx] = swsqe->cqe; - t4_swcq_produce(cq); - swsqe->signaled = 0; - wq->sq.in_use -= unsignaled; - break; - } else - break; -} - -static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe, - struct t4_cqe *read_cqe) -{ - read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx; - read_cqe->len = cpu_to_be32(wq->sq.oldest_read->read_len); - read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(hw_cqe)) | - V_CQE_SWCQE(SW_CQE(hw_cqe)) | - V_CQE_OPCODE(FW_RI_READ_REQ) | - V_CQE_TYPE(1)); - read_cqe->bits_type_ts = hw_cqe->bits_type_ts; -} - /* - * Return a ptr to the next read wr in the SWSQ or NULL. - */ -static void advance_oldest_read(struct t4_wq *wq) -{ - - u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1; - - if (rptr == wq->sq.size) - rptr = 0; - while (rptr != wq->sq.pidx) { - wq->sq.oldest_read = &wq->sq.sw_sq[rptr]; - - if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ) - return; - if (++rptr == wq->sq.size) - rptr = 0; - } - wq->sq.oldest_read = NULL; -} - -/* * poll_cq * * Caller must: * check the validity of the first CQE, * supply the wq assicated with the qpid. * * credit: cq credit to return to sge. * cqe_flushed: 1 iff the CQE is flushed. * cqe: copy of the polled CQE. * * return value: * 0 CQE returned ok. * -EAGAIN CQE skipped, try again. * -EOVERFLOW CQ overflow detected. */ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe, u8 *cqe_flushed, u64 *cookie, u32 *credit) { int ret = 0; struct t4_cqe *hw_cqe, read_cqe; *cqe_flushed = 0; *credit = 0; ret = t4_next_cqe(cq, &hw_cqe); if (ret) return ret; CTR6(KTR_IW_CXGBE, "%s CQE OVF %u qpid 0x%0x genbit %u type %u status 0x%0x", __func__, CQE_OVFBIT(hw_cqe), CQE_QPID(hw_cqe), CQE_GENBIT(hw_cqe), CQE_TYPE(hw_cqe), CQE_STATUS(hw_cqe)); CTR5(KTR_IW_CXGBE, "%s opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x", __func__, CQE_OPCODE(hw_cqe), CQE_LEN(hw_cqe), CQE_WRID_HI(hw_cqe), CQE_WRID_LOW(hw_cqe)); /* * skip cqe's not affiliated with a QP. */ if (wq == NULL) { ret = -EAGAIN; goto skip_cqe; } /* + * skip hw cqe's if the wq is flushed. + */ + if (wq->flushed && !SW_CQE(hw_cqe)) { + ret = -EAGAIN; + goto skip_cqe; + } + + /* + * skip TERMINATE cqes... + */ + if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) { + ret = -EAGAIN; + goto skip_cqe; + } + + /* * Special cqe for drain WR completions... */ if (CQE_OPCODE(hw_cqe) == C4IW_DRAIN_OPCODE) { *cookie = CQE_DRAIN_COOKIE(hw_cqe); *cqe = *hw_cqe; goto skip_cqe; } /* * Gotta tweak READ completions: * 1) the cqe doesn't contain the sq_wptr from the wr. * 2) opcode not reflected from the wr. * 3) read_len not reflected from the wr. * 4) cq_type is RQ_TYPE not SQ_TYPE. */ if (RQ_TYPE(hw_cqe) && (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP)) { - /* - * If this is an unsolicited read response, then the read + /* If we have reached here because of async + * event or other error, and have egress error + * then drop + */ + if (CQE_TYPE(hw_cqe) == 1) { + if (CQE_STATUS(hw_cqe)) + t4_set_wq_in_error(wq); + ret = -EAGAIN; + goto skip_cqe; + } + + /* If this is an unsolicited read response, then the read * was generated by the kernel driver as part of peer-2-peer * connection setup. So ignore the completion. */ - if (!wq->sq.oldest_read) { + if (CQE_WRID_STAG(hw_cqe) == 1) { if (CQE_STATUS(hw_cqe)) t4_set_wq_in_error(wq); ret = -EAGAIN; goto skip_cqe; } /* + * Eat completions for unsignaled read WRs. + */ + if (!wq->sq.oldest_read->signaled) { + advance_oldest_read(wq); + ret = -EAGAIN; + goto skip_cqe; + } + + /* * Don't write to the HWCQ, so create a new read req CQE * in local memory. */ create_read_req_cqe(wq, hw_cqe, &read_cqe); hw_cqe = &read_cqe; advance_oldest_read(wq); } if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) { - *cqe_flushed = t4_wq_in_error(wq); + *cqe_flushed = (CQE_STATUS(hw_cqe) == T4_ERR_SWFLUSH); t4_set_wq_in_error(wq); - goto proc_cqe; } - if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) { - ret = -EAGAIN; - goto skip_cqe; - } - /* * RECV completion. */ if (RQ_TYPE(hw_cqe)) { /* * HW only validates 4 bits of MSN. So we must validate that * the MSN in the SEND is the next expected MSN. If its not, * then we complete this with T4_ERR_MSN and mark the wq in * error. */ if (t4_rq_empty(wq)) { t4_set_wq_in_error(wq); ret = -EAGAIN; goto skip_cqe; } if (unlikely((CQE_WRID_MSN(hw_cqe) != (wq->rq.msn)))) { t4_set_wq_in_error(wq); hw_cqe->header |= htonl(V_CQE_STATUS(T4_ERR_MSN)); goto proc_cqe; } goto proc_cqe; } /* * If we get here its a send completion. * * Handle out of order completion. These get stuffed * in the SW SQ. Then the SW SQ is walked to move any * now in-order completions into the SW CQ. This handles * 2 cases: * 1) reaping unsignaled WRs when the first subsequent * signaled WR is completed. * 2) out of order read completions. */ if (!SW_CQE(hw_cqe) && (CQE_WRID_SQ_IDX(hw_cqe) != wq->sq.cidx)) { struct t4_swsqe *swsqe; CTR2(KTR_IW_CXGBE, "%s out of order completion going in sw_sq at idx %u", __func__, CQE_WRID_SQ_IDX(hw_cqe)); swsqe = &wq->sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)]; swsqe->cqe = *hw_cqe; swsqe->complete = 1; ret = -EAGAIN; goto flush_wq; } proc_cqe: *cqe = *hw_cqe; /* * Reap the associated WR(s) that are freed up with this * completion. */ if (SQ_TYPE(hw_cqe)) { - wq->sq.cidx = CQE_WRID_SQ_IDX(hw_cqe); - CTR2(KTR_IW_CXGBE, "%s completing sq idx %u", - __func__, wq->sq.cidx); + int idx = CQE_WRID_SQ_IDX(hw_cqe); + BUG_ON(idx >= wq->sq.size); + + /* + * Account for any unsignaled completions completed by + * this signaled completion. In this case, cidx points + * to the first unsignaled one, and idx points to the + * signaled one. So adjust in_use based on this delta. + * if this is not completing any unsigned wrs, then the + * delta will be 0. Handle wrapping also! + */ + if (idx < wq->sq.cidx) + wq->sq.in_use -= wq->sq.size + idx - wq->sq.cidx; + else + wq->sq.in_use -= idx - wq->sq.cidx; + BUG_ON(wq->sq.in_use <= 0 && wq->sq.in_use >= wq->sq.size); + + wq->sq.cidx = (uint16_t)idx; + CTR2(KTR_IW_CXGBE, "%s completing sq idx %u\n", + __func__, wq->sq.cidx); *cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id; t4_sq_consume(wq); } else { CTR2(KTR_IW_CXGBE, "%s completing rq idx %u", __func__, wq->rq.cidx); *cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id; BUG_ON(t4_rq_empty(wq)); t4_rq_consume(wq); + goto skip_cqe; } flush_wq: /* * Flush any completed cqes that are now in-order. */ flush_completed_wrs(wq, cq); skip_cqe: if (SW_CQE(hw_cqe)) { CTR4(KTR_IW_CXGBE, "%s cq %p cqid 0x%x skip sw cqe cidx %u", __func__, cq, cq->cqid, cq->sw_cidx); t4_swcq_consume(cq); } else { CTR4(KTR_IW_CXGBE, "%s cq %p cqid 0x%x skip hw cqe cidx %u", __func__, cq, cq->cqid, cq->cidx); t4_hwcq_consume(cq); } return ret; } /* * Get one cq entry from c4iw and map it to openib. * * Returns: * 0 cqe returned * -ENODATA EMPTY; * -EAGAIN caller must try again * any other -errno fatal error */ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc) { struct c4iw_qp *qhp = NULL; struct t4_cqe cqe = {0, 0}, *rd_cqe; struct t4_wq *wq; u32 credit = 0; u8 cqe_flushed; u64 cookie = 0; int ret; ret = t4_next_cqe(&chp->cq, &rd_cqe); if (ret) return ret; qhp = get_qhp(chp->rhp, CQE_QPID(rd_cqe)); if (!qhp) wq = NULL; else { spin_lock(&qhp->lock); wq = &(qhp->wq); } ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit); if (ret) goto out; wc->wr_id = cookie; wc->qp = &qhp->ibqp; wc->vendor_err = CQE_STATUS(&cqe); wc->wc_flags = 0; CTR5(KTR_IW_CXGBE, "%s qpid 0x%x type %d opcode %d status 0x%x", __func__, CQE_QPID(&cqe), CQE_TYPE(&cqe), CQE_OPCODE(&cqe), CQE_STATUS(&cqe)); CTR5(KTR_IW_CXGBE, "%s len %u wrid hi 0x%x lo 0x%x cookie 0x%llx", __func__, CQE_LEN(&cqe), CQE_WRID_HI(&cqe), CQE_WRID_LOW(&cqe), (unsigned long long)cookie); if (CQE_TYPE(&cqe) == 0) { if (!CQE_STATUS(&cqe)) wc->byte_len = CQE_LEN(&cqe); else wc->byte_len = 0; wc->opcode = IB_WC_RECV; if (CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_INV || CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_SE_INV) { wc->ex.invalidate_rkey = CQE_WRID_STAG(&cqe); wc->wc_flags |= IB_WC_WITH_INVALIDATE; + c4iw_invalidate_mr(qhp->rhp, wc->ex.invalidate_rkey); } } else { switch (CQE_OPCODE(&cqe)) { case FW_RI_RDMA_WRITE: wc->opcode = IB_WC_RDMA_WRITE; break; case FW_RI_READ_REQ: wc->opcode = IB_WC_RDMA_READ; wc->byte_len = CQE_LEN(&cqe); break; case FW_RI_SEND_WITH_INV: case FW_RI_SEND_WITH_SE_INV: wc->opcode = IB_WC_SEND; wc->wc_flags |= IB_WC_WITH_INVALIDATE; break; case FW_RI_SEND: case FW_RI_SEND_WITH_SE: wc->opcode = IB_WC_SEND; break; - case FW_RI_BIND_MW: - wc->opcode = IB_WC_BIND_MW; - break; - case FW_RI_LOCAL_INV: wc->opcode = IB_WC_LOCAL_INV; break; case FW_RI_FAST_REGISTER: - wc->opcode = IB_WC_FAST_REG_MR; + wc->opcode = IB_WC_REG_MR; + + /* Invalidate the MR if the fastreg failed */ + if (CQE_STATUS(&cqe) != T4_ERR_SUCCESS) + c4iw_invalidate_mr(qhp->rhp, + CQE_WRID_FR_STAG(&cqe)); break; case C4IW_DRAIN_OPCODE: wc->opcode = IB_WC_SEND; break; default: printf("Unexpected opcode %d " "in the CQE received for QPID = 0x%0x\n", CQE_OPCODE(&cqe), CQE_QPID(&cqe)); ret = -EINVAL; goto out; } } if (cqe_flushed) wc->status = IB_WC_WR_FLUSH_ERR; else { switch (CQE_STATUS(&cqe)) { case T4_ERR_SUCCESS: wc->status = IB_WC_SUCCESS; break; case T4_ERR_STAG: wc->status = IB_WC_LOC_ACCESS_ERR; break; case T4_ERR_PDID: wc->status = IB_WC_LOC_PROT_ERR; break; case T4_ERR_QPID: case T4_ERR_ACCESS: wc->status = IB_WC_LOC_ACCESS_ERR; break; case T4_ERR_WRAP: wc->status = IB_WC_GENERAL_ERR; break; case T4_ERR_BOUND: wc->status = IB_WC_LOC_LEN_ERR; break; case T4_ERR_INVALIDATE_SHARED_MR: case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND: wc->status = IB_WC_MW_BIND_ERR; break; case T4_ERR_CRC: case T4_ERR_MARKER: case T4_ERR_PDU_LEN_ERR: case T4_ERR_OUT_OF_RQE: case T4_ERR_DDP_VERSION: case T4_ERR_RDMA_VERSION: case T4_ERR_DDP_QUEUE_NUM: case T4_ERR_MSN: case T4_ERR_TBIT: case T4_ERR_MO: case T4_ERR_MSN_RANGE: case T4_ERR_IRD_OVERFLOW: case T4_ERR_OPCODE: case T4_ERR_INTERNAL_ERR: wc->status = IB_WC_FATAL_ERR; break; case T4_ERR_SWFLUSH: wc->status = IB_WC_WR_FLUSH_ERR; break; default: printf("Unexpected cqe_status 0x%x for QPID = 0x%0x\n", CQE_STATUS(&cqe), CQE_QPID(&cqe)); wc->status = IB_WC_FATAL_ERR; } } out: if (wq) spin_unlock(&qhp->lock); return ret; } int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) { struct c4iw_cq *chp; unsigned long flags; int npolled; int err = 0; chp = to_c4iw_cq(ibcq); spin_lock_irqsave(&chp->lock, flags); for (npolled = 0; npolled < num_entries; ++npolled) { do { err = c4iw_poll_cq_one(chp, wc + npolled); } while (err == -EAGAIN); if (err) break; } spin_unlock_irqrestore(&chp->lock, flags); return !err || err == -ENODATA ? npolled : err; } int c4iw_destroy_cq(struct ib_cq *ib_cq) { struct c4iw_cq *chp; struct c4iw_ucontext *ucontext; CTR2(KTR_IW_CXGBE, "%s ib_cq %p", __func__, ib_cq); chp = to_c4iw_cq(ib_cq); remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid); atomic_dec(&chp->refcnt); wait_event(chp->wait, !atomic_read(&chp->refcnt)); ucontext = ib_cq->uobject ? to_c4iw_ucontext(ib_cq->uobject->context) : NULL; destroy_cq(&chp->rhp->rdev, &chp->cq, ucontext ? &ucontext->uctx : &chp->cq.rdev->uctx); kfree(chp); return 0; } struct ib_cq * -c4iw_create_cq(struct ib_device *ibdev, struct ib_cq_init_attr *attr, +c4iw_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, struct ib_ucontext *ib_context, struct ib_udata *udata) { + int entries = attr->cqe; + int vector = attr->comp_vector; struct c4iw_dev *rhp; struct c4iw_cq *chp; struct c4iw_create_cq_resp uresp; struct c4iw_ucontext *ucontext = NULL; int ret; size_t memsize, hwentries; struct c4iw_mm_entry *mm, *mm2; - int entries = attr->cqe; CTR3(KTR_IW_CXGBE, "%s ib_dev %p entries %d", __func__, ibdev, entries); + if (attr->flags) + return ERR_PTR(-EINVAL); rhp = to_c4iw_dev(ibdev); chp = kzalloc(sizeof(*chp), GFP_KERNEL); if (!chp) return ERR_PTR(-ENOMEM); + if (ib_context) ucontext = to_c4iw_ucontext(ib_context); /* account for the status page. */ entries++; /* IQ needs one extra entry to differentiate full vs empty. */ entries++; /* * entries must be multiple of 16 for HW. */ entries = roundup(entries, 16); /* - * Make actual HW queue 2x to avoid cidx_inc overflows. + * Make actual HW queue 2x to avoid cdix_inc overflows. */ - hwentries = entries * 2; + hwentries = min(entries * 2, rhp->rdev.hw_queue.t4_max_iq_size); /* * Make HW queue at least 64 entries so GTS updates aren't too * frequent. */ if (hwentries < 64) hwentries = 64; memsize = hwentries * sizeof *chp->cq.queue; /* * memsize must be a multiple of the page size if its a user cq. */ - if (ucontext) { + if (ucontext) memsize = roundup(memsize, PAGE_SIZE); - hwentries = memsize / sizeof *chp->cq.queue; - while (hwentries > T4_MAX_IQ_SIZE) { - memsize -= PAGE_SIZE; - hwentries = memsize / sizeof *chp->cq.queue; - } - } chp->cq.size = hwentries; chp->cq.memsize = memsize; + chp->cq.vector = vector; ret = create_cq(&rhp->rdev, &chp->cq, ucontext ? &ucontext->uctx : &rhp->rdev.uctx); if (ret) goto err1; chp->rhp = rhp; chp->cq.size--; /* status page */ chp->ibcq.cqe = entries - 2; spin_lock_init(&chp->lock); spin_lock_init(&chp->comp_handler_lock); atomic_set(&chp->refcnt, 1); init_waitqueue_head(&chp->wait); ret = insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid); if (ret) goto err2; if (ucontext) { + ret = -ENOMEM; mm = kmalloc(sizeof *mm, GFP_KERNEL); if (!mm) goto err3; mm2 = kmalloc(sizeof *mm2, GFP_KERNEL); if (!mm2) goto err4; memset(&uresp, 0, sizeof(uresp)); uresp.qid_mask = rhp->rdev.cqmask; uresp.cqid = chp->cq.cqid; uresp.size = chp->cq.size; uresp.memsize = chp->cq.memsize; spin_lock(&ucontext->mmap_lock); uresp.key = ucontext->key; ucontext->key += PAGE_SIZE; uresp.gts_key = ucontext->key; ucontext->key += PAGE_SIZE; spin_unlock(&ucontext->mmap_lock); ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp) - sizeof(uresp.reserved)); if (ret) goto err5; mm->key = uresp.key; mm->addr = vtophys(chp->cq.queue); mm->len = chp->cq.memsize; insert_mmap(ucontext, mm); mm2->key = uresp.gts_key; - mm2->addr = chp->cq.ugts; + mm2->addr = chp->cq.bar2_pa; mm2->len = PAGE_SIZE; insert_mmap(ucontext, mm2); } CTR6(KTR_IW_CXGBE, "%s cqid 0x%0x chp %p size %u memsize %zu, dma_addr 0x%0llx", __func__, chp->cq.cqid, chp, chp->cq.size, chp->cq.memsize, (unsigned long long) chp->cq.dma_addr); return &chp->ibcq; err5: kfree(mm2); err4: kfree(mm); err3: remove_handle(rhp, &rhp->cqidr, chp->cq.cqid); err2: destroy_cq(&chp->rhp->rdev, &chp->cq, ucontext ? &ucontext->uctx : &rhp->rdev.uctx); err1: kfree(chp); return ERR_PTR(ret); } int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata) { return -ENOSYS; } int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { struct c4iw_cq *chp; - int ret; + int ret = 0; unsigned long flag; chp = to_c4iw_cq(ibcq); spin_lock_irqsave(&chp->lock, flag); - ret = t4_arm_cq(&chp->cq, - (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED); + t4_arm_cq(&chp->cq, + (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED); + if (flags & IB_CQ_REPORT_MISSED_EVENTS) + ret = t4_cq_notempty(&chp->cq); spin_unlock_irqrestore(&chp->lock, flag); - if (ret && !(flags & IB_CQ_REPORT_MISSED_EVENTS)) - ret = 0; return ret; } #endif Index: projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/device.c =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/device.c (revision 331317) +++ projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/device.c (revision 331318) @@ -1,361 +1,455 @@ /* * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include #include #include #include #include #ifdef TCP_OFFLOAD #include "iw_cxgbe.h" void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx) { struct list_head *pos, *nxt; struct c4iw_qid_list *entry; mutex_lock(&uctx->lock); list_for_each_safe(pos, nxt, &uctx->qpids) { entry = list_entry(pos, struct c4iw_qid_list, entry); list_del_init(&entry->entry); if (!(entry->qid & rdev->qpmask)) { c4iw_put_resource(&rdev->resource.qid_table, entry->qid); mutex_lock(&rdev->stats.lock); rdev->stats.qid.cur -= rdev->qpmask + 1; mutex_unlock(&rdev->stats.lock); } kfree(entry); } - list_for_each_safe(pos, nxt, &uctx->qpids) { + list_for_each_safe(pos, nxt, &uctx->cqids) { entry = list_entry(pos, struct c4iw_qid_list, entry); list_del_init(&entry->entry); kfree(entry); } mutex_unlock(&uctx->lock); } void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx) { INIT_LIST_HEAD(&uctx->qpids); INIT_LIST_HEAD(&uctx->cqids); mutex_init(&uctx->lock); } static int c4iw_rdev_open(struct c4iw_rdev *rdev) { struct adapter *sc = rdev->adap; struct sge_params *sp = &sc->params.sge; int rc; + unsigned short ucq_density = 1 << sp->iq_s_qpp; /* # of user CQs/page */ + unsigned short udb_density = 1 << sp->eq_s_qpp; /* # of user DB/page */ + c4iw_init_dev_ucontext(rdev, &rdev->uctx); - /* XXX: we can probably make this work */ - if (sp->eq_s_qpp > PAGE_SHIFT || sp->iq_s_qpp > PAGE_SHIFT) { - device_printf(sc->dev, - "doorbell density too high (eq %d, iq %d, pg %d).\n", - sp->eq_s_qpp, sp->eq_s_qpp, PAGE_SHIFT); + /* + * This implementation assumes udb_density == ucq_density! Eventually + * we might need to support this but for now fail the open. Also the + * cqid and qpid range must match for now. + */ + if (udb_density != ucq_density) { + device_printf(sc->dev, "unsupported udb/ucq densities %u/%u\n", + udb_density, ucq_density); rc = -EINVAL; goto err1; } + if (sc->vres.qp.start != sc->vres.cq.start || + sc->vres.qp.size != sc->vres.cq.size) { + device_printf(sc->dev, "%s: unsupported qp and cq id ranges " + "qp start %u size %u cq start %u size %u\n", __func__, + sc->vres.qp.start, sc->vres.qp.size, sc->vres.cq.start, + sc->vres.cq.size); + rc = -EINVAL; + goto err1; + } rdev->qpshift = PAGE_SHIFT - sp->eq_s_qpp; - rdev->qpmask = (1 << sp->eq_s_qpp) - 1; + rdev->qpmask = udb_density - 1; rdev->cqshift = PAGE_SHIFT - sp->iq_s_qpp; - rdev->cqmask = (1 << sp->iq_s_qpp) - 1; + rdev->cqmask = ucq_density - 1; + CTR5(KTR_IW_CXGBE, "%s dev %s stag start 0x%0x size 0x%0x num stags %d", + __func__, device_get_nameunit(sc->dev), sc->vres.stag.start, + sc->vres.stag.size, c4iw_num_stags(rdev)); + CTR5(KTR_IW_CXGBE, "%s pbl start 0x%0x size 0x%0x" + " rq start 0x%0x size 0x%0x", __func__, + sc->vres.pbl.start, sc->vres.pbl.size, + sc->vres.rq.start, sc->vres.rq.size); + CTR5(KTR_IW_CXGBE, "%s:qp qid start %u size %u cq qid start %u size %u", + __func__, sc->vres.qp.start, sc->vres.qp.size, + sc->vres.cq.start, sc->vres.cq.size); + /*TODO + CTR5(KTR_IW_CXGBE, "%s udb %pR db_reg %p gts_reg %p" + "qpmask 0x%x cqmask 0x%x", __func__, + db_reg,gts_reg,rdev->qpmask, rdev->cqmask); + */ + + if (c4iw_num_stags(rdev) == 0) { rc = -EINVAL; goto err1; } rdev->stats.pd.total = T4_MAX_NUM_PD; rdev->stats.stag.total = sc->vres.stag.size; rdev->stats.pbl.total = sc->vres.pbl.size; rdev->stats.rqt.total = sc->vres.rq.size; rdev->stats.qid.total = sc->vres.qp.size; rc = c4iw_init_resource(rdev, c4iw_num_stags(rdev), T4_MAX_NUM_PD); if (rc) { device_printf(sc->dev, "error %d initializing resources\n", rc); goto err1; } rc = c4iw_pblpool_create(rdev); if (rc) { device_printf(sc->dev, "error %d initializing pbl pool\n", rc); goto err2; } rc = c4iw_rqtpool_create(rdev); if (rc) { device_printf(sc->dev, "error %d initializing rqt pool\n", rc); goto err3; } + rdev->status_page = (struct t4_dev_status_page *) + __get_free_page(GFP_KERNEL); + if (!rdev->status_page) { + rc = -ENOMEM; + goto err4; + } + rdev->status_page->qp_start = sc->vres.qp.start; + rdev->status_page->qp_size = sc->vres.qp.size; + rdev->status_page->cq_start = sc->vres.cq.start; + rdev->status_page->cq_size = sc->vres.cq.size; + /* T5 and above devices don't need Doorbell recovery logic, + * so db_off is always set to '0'. + */ + rdev->status_page->db_off = 0; + + rdev->status_page->wc_supported = rdev->adap->iwt.wc_en; + + rdev->free_workq = create_singlethread_workqueue("iw_cxgb4_free"); + if (!rdev->free_workq) { + rc = -ENOMEM; + goto err5; + } return (0); +err5: + free_page((unsigned long)rdev->status_page); +err4: + c4iw_rqtpool_destroy(rdev); err3: c4iw_pblpool_destroy(rdev); err2: c4iw_destroy_resource(&rdev->resource); err1: return (rc); } static void c4iw_rdev_close(struct c4iw_rdev *rdev) { + free_page((unsigned long)rdev->status_page); c4iw_pblpool_destroy(rdev); c4iw_rqtpool_destroy(rdev); c4iw_destroy_resource(&rdev->resource); } static void c4iw_dealloc(struct c4iw_dev *iwsc) { c4iw_rdev_close(&iwsc->rdev); idr_destroy(&iwsc->cqidr); idr_destroy(&iwsc->qpidr); idr_destroy(&iwsc->mmidr); ib_dealloc_device(&iwsc->ibdev); } static struct c4iw_dev * c4iw_alloc(struct adapter *sc) { struct c4iw_dev *iwsc; int rc; iwsc = (struct c4iw_dev *)ib_alloc_device(sizeof(*iwsc)); if (iwsc == NULL) { device_printf(sc->dev, "Cannot allocate ib device.\n"); return (ERR_PTR(-ENOMEM)); } iwsc->rdev.adap = sc; + /* init various hw-queue params based on lld info */ + CTR3(KTR_IW_CXGBE, "%s: Ing. padding boundary is %d, " + "egrsstatuspagesize = %d", __func__, + sc->params.sge.pad_boundary, + sc->params.sge.spg_len); + + iwsc->rdev.hw_queue.t4_eq_status_entries = + sc->params.sge.spg_len / EQ_ESIZE; + iwsc->rdev.hw_queue.t4_max_eq_size = 65520; + iwsc->rdev.hw_queue.t4_max_iq_size = 65520; + iwsc->rdev.hw_queue.t4_max_rq_size = 8192 - + iwsc->rdev.hw_queue.t4_eq_status_entries - 1; + iwsc->rdev.hw_queue.t4_max_sq_size = + iwsc->rdev.hw_queue.t4_max_eq_size - + iwsc->rdev.hw_queue.t4_eq_status_entries - 1; + iwsc->rdev.hw_queue.t4_max_qp_depth = + iwsc->rdev.hw_queue.t4_max_rq_size; + iwsc->rdev.hw_queue.t4_max_cq_depth = + iwsc->rdev.hw_queue.t4_max_iq_size - 2; + iwsc->rdev.hw_queue.t4_stat_len = iwsc->rdev.adap->params.sge.spg_len; + + /* As T5 and above devices support BAR2 kernel doorbells & WC, we map + * all of BAR2, for both User and Kernel Doorbells-GTS. + */ + iwsc->rdev.bar2_kva = (void __iomem *)((u64)iwsc->rdev.adap->udbs_base); + iwsc->rdev.bar2_pa = vtophys(iwsc->rdev.adap->udbs_base); + iwsc->rdev.bar2_len = rman_get_size(iwsc->rdev.adap->udbs_res); + rc = c4iw_rdev_open(&iwsc->rdev); if (rc != 0) { device_printf(sc->dev, "Unable to open CXIO rdev (%d)\n", rc); ib_dealloc_device(&iwsc->ibdev); return (ERR_PTR(rc)); } idr_init(&iwsc->cqidr); idr_init(&iwsc->qpidr); idr_init(&iwsc->mmidr); spin_lock_init(&iwsc->lock); mutex_init(&iwsc->rdev.stats.lock); + iwsc->avail_ird = iwsc->rdev.adap->params.max_ird_adapter; return (iwsc); } static int c4iw_mod_load(void); static int c4iw_mod_unload(void); static int c4iw_activate(struct adapter *); static int c4iw_deactivate(struct adapter *); static struct uld_info c4iw_uld_info = { .uld_id = ULD_IWARP, .activate = c4iw_activate, .deactivate = c4iw_deactivate, }; static int c4iw_activate(struct adapter *sc) { struct c4iw_dev *iwsc; int rc; ASSERT_SYNCHRONIZED_OP(sc); + if (is_t4(sc)) { + device_printf(sc->dev, "No iWARP support for T4 devices, " + "please install T5 or above devices.\n"); + return (ENOSYS); + } + if (uld_active(sc, ULD_IWARP)) { KASSERT(0, ("%s: RDMA already eanbled on sc %p", __func__, sc)); return (0); } if (sc->rdmacaps == 0) { device_printf(sc->dev, "RDMA not supported or RDMA cap is not enabled.\n"); return (ENOSYS); } iwsc = c4iw_alloc(sc); if (IS_ERR(iwsc)) { rc = -PTR_ERR(iwsc); device_printf(sc->dev, "initialization failed: %d\n", rc); return (rc); } sc->iwarp_softc = iwsc; rc = -c4iw_register_device(iwsc); if (rc) { device_printf(sc->dev, "RDMA registration failed: %d\n", rc); c4iw_dealloc(iwsc); sc->iwarp_softc = NULL; } return (rc); } static int c4iw_deactivate(struct adapter *sc) { struct c4iw_dev *iwsc = sc->iwarp_softc; ASSERT_SYNCHRONIZED_OP(sc); c4iw_unregister_device(iwsc); c4iw_dealloc(iwsc); sc->iwarp_softc = NULL; return (0); } static void c4iw_activate_all(struct adapter *sc, void *arg __unused) { if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4iwact") != 0) return; /* Activate iWARP if any port on this adapter has IFCAP_TOE enabled. */ if (sc->offload_map && !uld_active(sc, ULD_IWARP)) (void) t4_activate_uld(sc, ULD_IWARP); end_synchronized_op(sc, 0); } static void c4iw_deactivate_all(struct adapter *sc, void *arg __unused) { if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4iwdea") != 0) return; if (uld_active(sc, ULD_IWARP)) (void) t4_deactivate_uld(sc, ULD_IWARP); end_synchronized_op(sc, 0); } static int c4iw_mod_load(void) { int rc; rc = -c4iw_cm_init(); if (rc != 0) return (rc); rc = t4_register_uld(&c4iw_uld_info); if (rc != 0) { c4iw_cm_term(); return (rc); } t4_iterate(c4iw_activate_all, NULL); return (rc); } static int c4iw_mod_unload(void) { t4_iterate(c4iw_deactivate_all, NULL); c4iw_cm_term(); if (t4_unregister_uld(&c4iw_uld_info) == EBUSY) return (EBUSY); return (0); } #endif /* * t4_tom won't load on kernels without TCP_OFFLOAD and this module's dependency * on t4_tom ensures that it won't either. So we don't directly check for * TCP_OFFLOAD here. */ static int c4iw_modevent(module_t mod, int cmd, void *arg) { int rc = 0; #ifdef TCP_OFFLOAD switch (cmd) { case MOD_LOAD: rc = c4iw_mod_load(); if (rc == 0) - printf("iw_cxgbe: Chelsio T4/T5/T6 RDMA driver loaded.\n"); + printf("iw_cxgbe: Chelsio T5/T6 RDMA driver loaded.\n"); break; case MOD_UNLOAD: rc = c4iw_mod_unload(); break; default: rc = EINVAL; } #else printf("t4_tom: compiled without TCP_OFFLOAD support.\n"); rc = EOPNOTSUPP; #endif return (rc); } static moduledata_t c4iw_mod_data = { "iw_cxgbe", c4iw_modevent, 0 }; MODULE_VERSION(iw_cxgbe, 1); MODULE_DEPEND(iw_cxgbe, t4nex, 1, 1, 1); MODULE_DEPEND(iw_cxgbe, t4_tom, 1, 1, 1); MODULE_DEPEND(iw_cxgbe, ibcore, 1, 1, 1); MODULE_DEPEND(iw_cxgbe, linuxkpi, 1, 1, 1); DECLARE_MODULE(iw_cxgbe, c4iw_mod_data, SI_SUB_EXEC, SI_ORDER_ANY); Index: projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h (revision 331317) +++ projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h (revision 331318) @@ -1,962 +1,999 @@ /* * Copyright (c) 2009-2013, 2016 Chelsio, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * * $FreeBSD$ */ #ifndef __IW_CXGB4_H__ #define __IW_CXGB4_H__ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #undef prefetch #include "common/common.h" #include "common/t4_msg.h" #include "common/t4_regs.h" #include "common/t4_tcb.h" #include "t4_l2t.h" #define DRV_NAME "iw_cxgbe" #define MOD DRV_NAME ":" #define KTR_IW_CXGBE KTR_SPARE3 extern int c4iw_debug; #define PDBG(fmt, args...) \ do { \ if (c4iw_debug) \ printf(MOD fmt, ## args); \ } while (0) #include "t4.h" static inline void *cplhdr(struct mbuf *m) { return mtod(m, void*); } #define PBL_OFF(rdev_p, a) ((a) - (rdev_p)->adap->vres.pbl.start) #define RQT_OFF(rdev_p, a) ((a) - (rdev_p)->adap->vres.rq.start) #define C4IW_ID_TABLE_F_RANDOM 1 /* Pseudo-randomize the id's returned */ #define C4IW_ID_TABLE_F_EMPTY 2 /* Table is initially empty */ struct c4iw_id_table { u32 flags; u32 start; /* logical minimal id */ u32 last; /* hint for find */ u32 max; spinlock_t lock; unsigned long *table; }; struct c4iw_resource { struct c4iw_id_table tpt_table; struct c4iw_id_table qid_table; struct c4iw_id_table pdid_table; }; struct c4iw_qid_list { struct list_head entry; u32 qid; }; struct c4iw_dev_ucontext { struct list_head qpids; struct list_head cqids; struct mutex lock; }; enum c4iw_rdev_flags { T4_FATAL_ERROR = (1<<0), + T4_STATUS_PAGE_DISABLED = (1<<1), }; struct c4iw_stat { u64 total; u64 cur; u64 max; u64 fail; }; struct c4iw_stats { struct mutex lock; struct c4iw_stat qid; struct c4iw_stat pd; struct c4iw_stat stag; struct c4iw_stat pbl; struct c4iw_stat rqt; }; +struct c4iw_hw_queue { + int t4_eq_status_entries; + int t4_max_eq_size; + int t4_max_iq_size; + int t4_max_rq_size; + int t4_max_sq_size; + int t4_max_qp_depth; + int t4_max_cq_depth; + int t4_stat_len; +}; + struct c4iw_rdev { struct adapter *adap; struct c4iw_resource resource; unsigned long qpshift; u32 qpmask; unsigned long cqshift; u32 cqmask; struct c4iw_dev_ucontext uctx; vmem_t *rqt_arena; vmem_t *pbl_arena; u32 flags; struct c4iw_stats stats; + struct c4iw_hw_queue hw_queue; + struct t4_dev_status_page *status_page; + unsigned long bar2_pa; + void __iomem *bar2_kva; + unsigned int bar2_len; + struct workqueue_struct *free_workq; }; static inline int c4iw_fatal_error(struct c4iw_rdev *rdev) { return rdev->flags & T4_FATAL_ERROR; } static inline int c4iw_num_stags(struct c4iw_rdev *rdev) { return (int)(rdev->adap->vres.stag.size >> 5); } #define C4IW_WR_TO (60*HZ) struct c4iw_wr_wait { int ret; struct completion completion; }; static inline void c4iw_init_wr_wait(struct c4iw_wr_wait *wr_waitp) { wr_waitp->ret = 0; init_completion(&wr_waitp->completion); } static inline void c4iw_wake_up(struct c4iw_wr_wait *wr_waitp, int ret) { wr_waitp->ret = ret; complete(&wr_waitp->completion); } static inline int c4iw_wait_for_reply(struct c4iw_rdev *rdev, struct c4iw_wr_wait *wr_waitp, - u32 hwtid, u32 qpid, const char *func) + u32 hwtid, u32 qpid, struct socket *so, const char *func) { struct adapter *sc = rdev->adap; unsigned to = C4IW_WR_TO; int ret; int timedout = 0; struct timeval t1, t2; if (c4iw_fatal_error(rdev)) { wr_waitp->ret = -EIO; goto out; } getmicrotime(&t1); do { + /* If waiting for reply in rdma_init()/rdma_fini() threads, then + * check if there are any connection errors. + */ + if (so && so->so_error) { + wr_waitp->ret = -ECONNRESET; + CTR5(KTR_IW_CXGBE, "%s - Connection ERROR %u for sock %p" + "tid %u qpid %u", func, + so->so_error, so, hwtid, qpid); + break; + } + ret = wait_for_completion_timeout(&wr_waitp->completion, to); if (!ret) { getmicrotime(&t2); timevalsub(&t2, &t1); printf("%s - Device %s not responding after %ld.%06ld " "seconds - tid %u qpid %u\n", func, device_get_nameunit(sc->dev), t2.tv_sec, t2.tv_usec, hwtid, qpid); if (c4iw_fatal_error(rdev)) { wr_waitp->ret = -EIO; break; } to = to << 2; timedout = 1; } } while (!ret); out: if (timedout) { getmicrotime(&t2); timevalsub(&t2, &t1); printf("%s - Device %s reply after %ld.%06ld seconds - " "tid %u qpid %u\n", func, device_get_nameunit(sc->dev), t2.tv_sec, t2.tv_usec, hwtid, qpid); } if (wr_waitp->ret) CTR4(KTR_IW_CXGBE, "%p: FW reply %d tid %u qpid %u", sc, wr_waitp->ret, hwtid, qpid); return (wr_waitp->ret); } struct c4iw_dev { struct ib_device ibdev; struct c4iw_rdev rdev; u32 device_cap_flags; struct idr cqidr; struct idr qpidr; struct idr mmidr; spinlock_t lock; struct dentry *debugfs_root; + u32 avail_ird; }; static inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev) { return container_of(ibdev, struct c4iw_dev, ibdev); } static inline struct c4iw_dev *rdev_to_c4iw_dev(struct c4iw_rdev *rdev) { return container_of(rdev, struct c4iw_dev, rdev); } static inline struct c4iw_cq *get_chp(struct c4iw_dev *rhp, u32 cqid) { return idr_find(&rhp->cqidr, cqid); } static inline struct c4iw_qp *get_qhp(struct c4iw_dev *rhp, u32 qpid) { return idr_find(&rhp->qpidr, qpid); } static inline struct c4iw_mr *get_mhp(struct c4iw_dev *rhp, u32 mmid) { return idr_find(&rhp->mmidr, mmid); } static inline int _insert_handle(struct c4iw_dev *rhp, struct idr *idr, void *handle, u32 id, int lock) { int ret; int newid; do { if (!idr_pre_get(idr, lock ? GFP_KERNEL : GFP_ATOMIC)) return -ENOMEM; if (lock) spin_lock_irq(&rhp->lock); ret = idr_get_new_above(idr, handle, id, &newid); BUG_ON(!ret && newid != id); if (lock) spin_unlock_irq(&rhp->lock); } while (ret == -EAGAIN); return ret; } static inline int insert_handle(struct c4iw_dev *rhp, struct idr *idr, void *handle, u32 id) { return _insert_handle(rhp, idr, handle, id, 1); } static inline int insert_handle_nolock(struct c4iw_dev *rhp, struct idr *idr, void *handle, u32 id) { return _insert_handle(rhp, idr, handle, id, 0); } static inline void _remove_handle(struct c4iw_dev *rhp, struct idr *idr, u32 id, int lock) { if (lock) spin_lock_irq(&rhp->lock); idr_remove(idr, id); if (lock) spin_unlock_irq(&rhp->lock); } static inline void remove_handle(struct c4iw_dev *rhp, struct idr *idr, u32 id) { _remove_handle(rhp, idr, id, 1); } static inline void remove_handle_nolock(struct c4iw_dev *rhp, struct idr *idr, u32 id) { _remove_handle(rhp, idr, id, 0); } +extern int c4iw_max_read_depth; + +static inline int cur_max_read_depth(struct c4iw_dev *dev) +{ + return min(dev->rdev.adap->params.max_ordird_qp, c4iw_max_read_depth); +} + struct c4iw_pd { struct ib_pd ibpd; u32 pdid; struct c4iw_dev *rhp; }; static inline struct c4iw_pd *to_c4iw_pd(struct ib_pd *ibpd) { return container_of(ibpd, struct c4iw_pd, ibpd); } struct tpt_attributes { u64 len; u64 va_fbo; enum fw_ri_mem_perms perms; u32 stag; u32 pdid; u32 qpid; u32 pbl_addr; u32 pbl_size; u32 state:1; u32 type:2; u32 rsvd:1; u32 remote_invaliate_disable:1; u32 zbva:1; u32 mw_bind_enable:1; u32 page_size:5; }; struct c4iw_mr { struct ib_mr ibmr; struct ib_umem *umem; struct c4iw_dev *rhp; u64 kva; struct tpt_attributes attr; + u64 *mpl; + dma_addr_t mpl_addr; + u32 max_mpl_len; + u32 mpl_len; }; static inline struct c4iw_mr *to_c4iw_mr(struct ib_mr *ibmr) { return container_of(ibmr, struct c4iw_mr, ibmr); } struct c4iw_mw { struct ib_mw ibmw; struct c4iw_dev *rhp; u64 kva; struct tpt_attributes attr; }; static inline struct c4iw_mw *to_c4iw_mw(struct ib_mw *ibmw) { return container_of(ibmw, struct c4iw_mw, ibmw); } -struct c4iw_fr_page_list { - struct ib_fast_reg_page_list ibpl; - DECLARE_PCI_UNMAP_ADDR(mapping); - dma_addr_t dma_addr; - struct c4iw_dev *dev; - int size; -}; - -static inline struct c4iw_fr_page_list *to_c4iw_fr_page_list( - struct ib_fast_reg_page_list *ibpl) -{ - return container_of(ibpl, struct c4iw_fr_page_list, ibpl); -} - struct c4iw_cq { struct ib_cq ibcq; struct c4iw_dev *rhp; struct t4_cq cq; spinlock_t lock; spinlock_t comp_handler_lock; atomic_t refcnt; wait_queue_head_t wait; }; static inline struct c4iw_cq *to_c4iw_cq(struct ib_cq *ibcq) { return container_of(ibcq, struct c4iw_cq, ibcq); } struct c4iw_mpa_attributes { u8 initiator; u8 recv_marker_enabled; u8 xmit_marker_enabled; u8 crc_enabled; u8 enhanced_rdma_conn; u8 version; u8 p2p_type; }; struct c4iw_qp_attributes { u32 scq; u32 rcq; u32 sq_num_entries; u32 rq_num_entries; u32 sq_max_sges; u32 sq_max_sges_rdma_write; u32 rq_max_sges; u32 state; u8 enable_rdma_read; u8 enable_rdma_write; u8 enable_bind; u8 enable_mmid0_fastreg; u32 max_ord; u32 max_ird; u32 pd; u32 next_state; char terminate_buffer[52]; u32 terminate_msg_len; u8 is_terminate_local; struct c4iw_mpa_attributes mpa_attr; struct c4iw_ep *llp_stream_handle; u8 layer_etype; u8 ecode; u16 sq_db_inc; u16 rq_db_inc; + u8 send_term; }; struct c4iw_qp { struct ib_qp ibqp; struct c4iw_dev *rhp; struct c4iw_ep *ep; struct c4iw_qp_attributes attr; struct t4_wq wq; spinlock_t lock; struct mutex mutex; - atomic_t refcnt; + struct kref kref; wait_queue_head_t wait; struct timer_list timer; int sq_sig_all; + struct work_struct free_work; + struct c4iw_ucontext *ucontext; }; static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp) { return container_of(ibqp, struct c4iw_qp, ibqp); } struct c4iw_ucontext { struct ib_ucontext ibucontext; struct c4iw_dev_ucontext uctx; u32 key; spinlock_t mmap_lock; struct list_head mmaps; + struct kref kref; }; static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c) { return container_of(c, struct c4iw_ucontext, ibucontext); } +void _c4iw_free_ucontext(struct kref *kref); + +static inline void c4iw_put_ucontext(struct c4iw_ucontext *ucontext) +{ + kref_put(&ucontext->kref, _c4iw_free_ucontext); +} +static inline void c4iw_get_ucontext(struct c4iw_ucontext *ucontext) +{ + kref_get(&ucontext->kref); +} + struct c4iw_mm_entry { struct list_head entry; u64 addr; u32 key; unsigned len; }; static inline struct c4iw_mm_entry *remove_mmap(struct c4iw_ucontext *ucontext, u32 key, unsigned len) { struct list_head *pos, *nxt; struct c4iw_mm_entry *mm; spin_lock(&ucontext->mmap_lock); list_for_each_safe(pos, nxt, &ucontext->mmaps) { mm = list_entry(pos, struct c4iw_mm_entry, entry); if (mm->key == key && mm->len == len) { list_del_init(&mm->entry); spin_unlock(&ucontext->mmap_lock); CTR4(KTR_IW_CXGBE, "%s key 0x%x addr 0x%llx len %d", __func__, key, (unsigned long long) mm->addr, mm->len); return mm; } } spin_unlock(&ucontext->mmap_lock); return NULL; } static inline void insert_mmap(struct c4iw_ucontext *ucontext, struct c4iw_mm_entry *mm) { spin_lock(&ucontext->mmap_lock); CTR4(KTR_IW_CXGBE, "%s key 0x%x addr 0x%llx len %d", __func__, mm->key, (unsigned long long) mm->addr, mm->len); list_add_tail(&mm->entry, &ucontext->mmaps); spin_unlock(&ucontext->mmap_lock); } enum c4iw_qp_attr_mask { C4IW_QP_ATTR_NEXT_STATE = 1 << 0, C4IW_QP_ATTR_SQ_DB = 1<<1, C4IW_QP_ATTR_RQ_DB = 1<<2, C4IW_QP_ATTR_ENABLE_RDMA_READ = 1 << 7, C4IW_QP_ATTR_ENABLE_RDMA_WRITE = 1 << 8, C4IW_QP_ATTR_ENABLE_RDMA_BIND = 1 << 9, C4IW_QP_ATTR_MAX_ORD = 1 << 11, C4IW_QP_ATTR_MAX_IRD = 1 << 12, C4IW_QP_ATTR_LLP_STREAM_HANDLE = 1 << 22, C4IW_QP_ATTR_STREAM_MSG_BUFFER = 1 << 23, C4IW_QP_ATTR_MPA_ATTR = 1 << 24, C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE = 1 << 25, C4IW_QP_ATTR_VALID_MODIFY = (C4IW_QP_ATTR_ENABLE_RDMA_READ | C4IW_QP_ATTR_ENABLE_RDMA_WRITE | C4IW_QP_ATTR_MAX_ORD | C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_STREAM_MSG_BUFFER | C4IW_QP_ATTR_MPA_ATTR | C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE) }; int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp, enum c4iw_qp_attr_mask mask, struct c4iw_qp_attributes *attrs, int internal); enum c4iw_qp_state { C4IW_QP_STATE_IDLE, C4IW_QP_STATE_RTS, C4IW_QP_STATE_ERROR, C4IW_QP_STATE_TERMINATE, C4IW_QP_STATE_CLOSING, C4IW_QP_STATE_TOT }; /* * IW_CXGBE event bits. * These bits are used for handling all events for a particular 'ep' serially. */ #define C4IW_EVENT_SOCKET 0x0001 #define C4IW_EVENT_TIMEOUT 0x0002 #define C4IW_EVENT_TERM 0x0004 static inline int c4iw_convert_state(enum ib_qp_state ib_state) { switch (ib_state) { case IB_QPS_RESET: case IB_QPS_INIT: return C4IW_QP_STATE_IDLE; case IB_QPS_RTS: return C4IW_QP_STATE_RTS; case IB_QPS_SQD: return C4IW_QP_STATE_CLOSING; case IB_QPS_SQE: return C4IW_QP_STATE_TERMINATE; case IB_QPS_ERR: return C4IW_QP_STATE_ERROR; default: return -1; } } static inline int to_ib_qp_state(int c4iw_qp_state) { switch (c4iw_qp_state) { case C4IW_QP_STATE_IDLE: return IB_QPS_INIT; case C4IW_QP_STATE_RTS: return IB_QPS_RTS; case C4IW_QP_STATE_CLOSING: return IB_QPS_SQD; case C4IW_QP_STATE_TERMINATE: return IB_QPS_SQE; case C4IW_QP_STATE_ERROR: return IB_QPS_ERR; } return IB_QPS_ERR; } #define C4IW_DRAIN_OPCODE FW_RI_SGE_EC_CR_RETURN static inline u32 c4iw_ib_to_tpt_access(int a) { return (a & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) | (a & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0) | (a & IB_ACCESS_LOCAL_WRITE ? FW_RI_MEM_ACCESS_LOCAL_WRITE : 0) | FW_RI_MEM_ACCESS_LOCAL_READ; } static inline u32 c4iw_ib_to_tpt_bind_access(int acc) { return (acc & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) | (acc & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0); } enum c4iw_mmid_state { C4IW_STAG_STATE_VALID, C4IW_STAG_STATE_INVALID }; #define C4IW_NODE_DESC "iw_cxgbe Chelsio Communications" #define MPA_KEY_REQ "MPA ID Req Frame" #define MPA_KEY_REP "MPA ID Rep Frame" #define MPA_MAX_PRIVATE_DATA 256 #define MPA_ENHANCED_RDMA_CONN 0x10 #define MPA_REJECT 0x20 #define MPA_CRC 0x40 #define MPA_MARKERS 0x80 #define MPA_FLAGS_MASK 0xE0 #define MPA_V2_PEER2PEER_MODEL 0x8000 #define MPA_V2_ZERO_LEN_FPDU_RTR 0x4000 #define MPA_V2_RDMA_WRITE_RTR 0x8000 #define MPA_V2_RDMA_READ_RTR 0x4000 #define MPA_V2_IRD_ORD_MASK 0x3FFF #define c4iw_put_ep(ep) { \ CTR4(KTR_IW_CXGBE, "put_ep (%s:%u) ep %p, refcnt %d", \ __func__, __LINE__, ep, atomic_read(&(ep)->kref.refcount)); \ WARN_ON(atomic_read(&(ep)->kref.refcount) < 1); \ kref_put(&((ep)->kref), _c4iw_free_ep); \ } #define c4iw_get_ep(ep) { \ CTR4(KTR_IW_CXGBE, "get_ep (%s:%u) ep %p, refcnt %d", \ __func__, __LINE__, ep, atomic_read(&(ep)->kref.refcount)); \ kref_get(&((ep)->kref)); \ } void _c4iw_free_ep(struct kref *kref); struct mpa_message { u8 key[16]; u8 flags; u8 revision; __be16 private_data_size; u8 private_data[0]; }; struct mpa_v2_conn_params { __be16 ird; __be16 ord; }; struct terminate_message { u8 layer_etype; u8 ecode; __be16 hdrct_rsvd; u8 len_hdrs[0]; }; #define TERM_MAX_LENGTH (sizeof(struct terminate_message) + 2 + 18 + 28) enum c4iw_layers_types { LAYER_RDMAP = 0x00, LAYER_DDP = 0x10, LAYER_MPA = 0x20, RDMAP_LOCAL_CATA = 0x00, RDMAP_REMOTE_PROT = 0x01, RDMAP_REMOTE_OP = 0x02, DDP_LOCAL_CATA = 0x00, DDP_TAGGED_ERR = 0x01, DDP_UNTAGGED_ERR = 0x02, DDP_LLP = 0x03 }; enum c4iw_rdma_ecodes { RDMAP_INV_STAG = 0x00, RDMAP_BASE_BOUNDS = 0x01, RDMAP_ACC_VIOL = 0x02, RDMAP_STAG_NOT_ASSOC = 0x03, RDMAP_TO_WRAP = 0x04, RDMAP_INV_VERS = 0x05, RDMAP_INV_OPCODE = 0x06, RDMAP_STREAM_CATA = 0x07, RDMAP_GLOBAL_CATA = 0x08, RDMAP_CANT_INV_STAG = 0x09, RDMAP_UNSPECIFIED = 0xff }; enum c4iw_ddp_ecodes { DDPT_INV_STAG = 0x00, DDPT_BASE_BOUNDS = 0x01, DDPT_STAG_NOT_ASSOC = 0x02, DDPT_TO_WRAP = 0x03, DDPT_INV_VERS = 0x04, DDPU_INV_QN = 0x01, DDPU_INV_MSN_NOBUF = 0x02, DDPU_INV_MSN_RANGE = 0x03, DDPU_INV_MO = 0x04, DDPU_MSG_TOOBIG = 0x05, DDPU_INV_VERS = 0x06 }; enum c4iw_mpa_ecodes { MPA_CRC_ERR = 0x02, MPA_MARKER_ERR = 0x03, MPA_LOCAL_CATA = 0x05, MPA_INSUFF_IRD = 0x06, MPA_NOMATCH_RTR = 0x07, }; enum c4iw_ep_state { IDLE = 0, LISTEN, CONNECTING, MPA_REQ_WAIT, MPA_REQ_SENT, MPA_REQ_RCVD, MPA_REP_SENT, FPDU_MODE, ABORTING, CLOSING, MORIBUND, DEAD, }; enum c4iw_ep_flags { PEER_ABORT_IN_PROGRESS = 0, ABORT_REQ_IN_PROGRESS = 1, RELEASE_RESOURCES = 2, CLOSE_SENT = 3, TIMEOUT = 4, - QP_REFERENCED = 5 + QP_REFERENCED = 5, + STOP_MPA_TIMER = 7, }; enum c4iw_ep_history { ACT_OPEN_REQ = 0, ACT_OFLD_CONN = 1, ACT_OPEN_RPL = 2, ACT_ESTAB = 3, PASS_ACCEPT_REQ = 4, PASS_ESTAB = 5, ABORT_UPCALL = 6, ESTAB_UPCALL = 7, CLOSE_UPCALL = 8, ULP_ACCEPT = 9, ULP_REJECT = 10, TIMEDOUT = 11, PEER_ABORT = 12, PEER_CLOSE = 13, CONNREQ_UPCALL = 14, ABORT_CONN = 15, DISCONN_UPCALL = 16, EP_DISC_CLOSE = 17, EP_DISC_ABORT = 18, CONN_RPL_UPCALL = 19, ACT_RETRY_NOMEM = 20, ACT_RETRY_INUSE = 21, CLOSE_CON_RPL = 22, EP_DISC_FAIL = 24, QP_REFED = 25, QP_DEREFED = 26, CM_ID_REFED = 27, CM_ID_DEREFED = 28 }; struct c4iw_ep_common { TAILQ_ENTRY(c4iw_ep_common) entry; /* Work queue attachment */ struct iw_cm_id *cm_id; struct c4iw_qp *qp; struct c4iw_dev *dev; enum c4iw_ep_state state; struct kref kref; struct mutex mutex; - struct sockaddr_in local_addr; - struct sockaddr_in remote_addr; + struct sockaddr_storage local_addr; + struct sockaddr_storage remote_addr; struct c4iw_wr_wait wr_wait; unsigned long flags; unsigned long history; int rpl_err; int rpl_done; struct thread *thread; struct socket *so; int ep_events; }; struct c4iw_listen_ep { struct c4iw_ep_common com; unsigned int stid; int backlog; + struct list_head listen_ep_list; /* list of all listener ep's bound + to one port address */ }; struct c4iw_ep { struct c4iw_ep_common com; - struct c4iw_ep *parent_ep; + struct c4iw_listen_ep *parent_ep; struct timer_list timer; unsigned int atid; u32 hwtid; u32 snd_seq; u32 rcv_seq; struct l2t_entry *l2t; struct dst_entry *dst; struct c4iw_mpa_attributes mpa_attr; u8 mpa_pkt[sizeof(struct mpa_message) + MPA_MAX_PRIVATE_DATA]; unsigned int mpa_pkt_len; u32 ird; u32 ord; u32 smac_idx; u32 tx_chan; u32 mtu; u16 mss; u16 emss; u16 plen; u16 rss_qid; u16 txq_idx; u16 ctrlq_idx; u8 tos; u8 retry_with_mpa_v1; u8 tried_with_mpa_v1; }; static inline struct c4iw_ep *to_ep(struct iw_cm_id *cm_id) { return cm_id->provider_data; } static inline struct c4iw_listen_ep *to_listen_ep(struct iw_cm_id *cm_id) { return cm_id->provider_data; } static inline int compute_wscale(int win) { int wscale = 0; while (wscale < 14 && (65535< __FBSDID("$FreeBSD$"); #include "opt_inet.h" #ifdef TCP_OFFLOAD #include #include #include #include #include #include "iw_cxgbe.h" +int use_dsgl = 1; #define T4_ULPTX_MIN_IO 32 #define C4IW_MAX_INLINE_SIZE 96 static int mr_exceeds_hw_limits(struct c4iw_dev *dev, u64 length) { - return ((is_t4(dev->rdev.adap) || - is_t5(dev->rdev.adap)) && - length >= 8*1024*1024*1024ULL); + return (is_t5(dev->rdev.adap) && length >= 8*1024*1024*1024ULL); } static int write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data) { struct adapter *sc = rdev->adap; struct ulp_mem_io *ulpmc; struct ulptx_idata *ulpsc; u8 wr_len, *to_dp, *from_dp; int copy_len, num_wqe, i, ret = 0; struct c4iw_wr_wait wr_wait; struct wrqe *wr; u32 cmd; cmd = cpu_to_be32(V_ULPTX_CMD(ULP_TX_MEM_WRITE)); - if (is_t4(sc)) - cmd |= cpu_to_be32(F_ULP_MEMIO_ORDER); - else - cmd |= cpu_to_be32(F_T5_ULP_MEMIO_IMM); + cmd |= cpu_to_be32(F_T5_ULP_MEMIO_IMM); + addr &= 0x7FFFFFF; CTR3(KTR_IW_CXGBE, "%s addr 0x%x len %u", __func__, addr, len); num_wqe = DIV_ROUND_UP(len, C4IW_MAX_INLINE_SIZE); c4iw_init_wr_wait(&wr_wait); for (i = 0; i < num_wqe; i++) { copy_len = min(len, C4IW_MAX_INLINE_SIZE); wr_len = roundup(sizeof *ulpmc + sizeof *ulpsc + roundup(copy_len, T4_ULPTX_MIN_IO), 16); wr = alloc_wrqe(wr_len, &sc->sge.mgmtq); if (wr == NULL) return (0); ulpmc = wrtod(wr); memset(ulpmc, 0, wr_len); INIT_ULPTX_WR(ulpmc, wr_len, 0, 0); if (i == (num_wqe-1)) { ulpmc->wr.wr_hi = cpu_to_be32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL); ulpmc->wr.wr_lo = (__force __be64)(unsigned long) &wr_wait; } else ulpmc->wr.wr_hi = cpu_to_be32(V_FW_WR_OP(FW_ULPTX_WR)); ulpmc->wr.wr_mid = cpu_to_be32( V_FW_WR_LEN16(DIV_ROUND_UP(wr_len, 16))); ulpmc->cmd = cmd; ulpmc->dlen = cpu_to_be32(V_ULP_MEMIO_DATA_LEN( DIV_ROUND_UP(copy_len, T4_ULPTX_MIN_IO))); ulpmc->len16 = cpu_to_be32(DIV_ROUND_UP(wr_len-sizeof(ulpmc->wr), 16)); ulpmc->lock_addr = cpu_to_be32(V_ULP_MEMIO_ADDR(addr + i * 3)); ulpsc = (struct ulptx_idata *)(ulpmc + 1); ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_IMM)); ulpsc->len = cpu_to_be32(roundup(copy_len, T4_ULPTX_MIN_IO)); to_dp = (u8 *)(ulpsc + 1); from_dp = (u8 *)data + i * C4IW_MAX_INLINE_SIZE; if (data) memcpy(to_dp, from_dp, copy_len); else memset(to_dp, 0, copy_len); if (copy_len % T4_ULPTX_MIN_IO) memset(to_dp + copy_len, 0, T4_ULPTX_MIN_IO - (copy_len % T4_ULPTX_MIN_IO)); t4_wrq_tx(sc, wr); len -= C4IW_MAX_INLINE_SIZE; } - ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__); + ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, NULL, __func__); return ret; } /* * Build and write a TPT entry. * IN: stag key, pdid, perm, bind_enabled, zbva, to, len, page_size, * pbl_size and pbl_addr * OUT: stag index */ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, u32 *stag, u8 stag_state, u32 pdid, enum fw_ri_stag_type type, enum fw_ri_mem_perms perm, int bind_enabled, u32 zbva, u64 to, u64 len, u8 page_size, u32 pbl_size, u32 pbl_addr) { int err; struct fw_ri_tpte tpt; u32 stag_idx; static atomic_t key; if (c4iw_fatal_error(rdev)) return -EIO; stag_state = stag_state > 0; stag_idx = (*stag) >> 8; if ((!reset_tpt_entry) && (*stag == T4_STAG_UNSET)) { stag_idx = c4iw_get_resource(&rdev->resource.tpt_table); if (!stag_idx) { mutex_lock(&rdev->stats.lock); rdev->stats.stag.fail++; mutex_unlock(&rdev->stats.lock); return -ENOMEM; } mutex_lock(&rdev->stats.lock); rdev->stats.stag.cur += 32; if (rdev->stats.stag.cur > rdev->stats.stag.max) rdev->stats.stag.max = rdev->stats.stag.cur; mutex_unlock(&rdev->stats.lock); *stag = (stag_idx << 8) | (atomic_inc_return(&key) & 0xff); } CTR5(KTR_IW_CXGBE, "%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x", __func__, stag_state, type, pdid, stag_idx); /* write TPT entry */ if (reset_tpt_entry) memset(&tpt, 0, sizeof(tpt)); else { tpt.valid_to_pdid = cpu_to_be32(F_FW_RI_TPTE_VALID | V_FW_RI_TPTE_STAGKEY((*stag & M_FW_RI_TPTE_STAGKEY)) | V_FW_RI_TPTE_STAGSTATE(stag_state) | V_FW_RI_TPTE_STAGTYPE(type) | V_FW_RI_TPTE_PDID(pdid)); tpt.locread_to_qpid = cpu_to_be32(V_FW_RI_TPTE_PERM(perm) | (bind_enabled ? F_FW_RI_TPTE_MWBINDEN : 0) | V_FW_RI_TPTE_ADDRTYPE((zbva ? FW_RI_ZERO_BASED_TO : FW_RI_VA_BASED_TO))| V_FW_RI_TPTE_PS(page_size)); tpt.nosnoop_pbladdr = !pbl_size ? 0 : cpu_to_be32( V_FW_RI_TPTE_PBLADDR(PBL_OFF(rdev, pbl_addr)>>3)); tpt.len_lo = cpu_to_be32((u32)(len & 0xffffffffUL)); tpt.va_hi = cpu_to_be32((u32)(to >> 32)); tpt.va_lo_fbo = cpu_to_be32((u32)(to & 0xffffffffUL)); tpt.dca_mwbcnt_pstag = cpu_to_be32(0); tpt.len_hi = cpu_to_be32((u32)(len >> 32)); } err = write_adapter_mem(rdev, stag_idx + (rdev->adap->vres.stag.start >> 5), sizeof(tpt), &tpt); if (reset_tpt_entry) { c4iw_put_resource(&rdev->resource.tpt_table, stag_idx); mutex_lock(&rdev->stats.lock); rdev->stats.stag.cur -= 32; mutex_unlock(&rdev->stats.lock); } return err; } static int write_pbl(struct c4iw_rdev *rdev, __be64 *pbl, u32 pbl_addr, u32 pbl_size) { int err; CTR4(KTR_IW_CXGBE, "%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d", __func__, pbl_addr, rdev->adap->vres.pbl.start, pbl_size); err = write_adapter_mem(rdev, pbl_addr >> 5, pbl_size << 3, pbl); return err; } static int dereg_mem(struct c4iw_rdev *rdev, u32 stag, u32 pbl_size, u32 pbl_addr) { return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, pbl_size, pbl_addr); } static int allocate_window(struct c4iw_rdev *rdev, u32 * stag, u32 pdid) { *stag = T4_STAG_UNSET; return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_MW, 0, 0, 0, 0UL, 0, 0, 0, 0); } static int deallocate_window(struct c4iw_rdev *rdev, u32 stag) { return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 0, 0); } static int allocate_stag(struct c4iw_rdev *rdev, u32 *stag, u32 pdid, u32 pbl_size, u32 pbl_addr) { *stag = T4_STAG_UNSET; return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_NSMR, 0, 0, 0, 0UL, 0, 0, pbl_size, pbl_addr); } static int finish_mem_reg(struct c4iw_mr *mhp, u32 stag) { u32 mmid; mhp->attr.state = 1; mhp->attr.stag = stag; mmid = stag >> 8; mhp->ibmr.rkey = mhp->ibmr.lkey = stag; CTR3(KTR_IW_CXGBE, "%s mmid 0x%x mhp %p", __func__, mmid, mhp); return insert_handle(mhp->rhp, &mhp->rhp->mmidr, mhp, mmid); } static int register_mem(struct c4iw_dev *rhp, struct c4iw_pd *php, struct c4iw_mr *mhp, int shift) { u32 stag = T4_STAG_UNSET; int ret; ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid, FW_RI_STAG_NSMR, mhp->attr.len ? mhp->attr.perms : 0, mhp->attr.mw_bind_enable, mhp->attr.zbva, mhp->attr.va_fbo, mhp->attr.len ? mhp->attr.len : -1, shift - 12, mhp->attr.pbl_size, mhp->attr.pbl_addr); if (ret) return ret; ret = finish_mem_reg(mhp, stag); if (ret) dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, mhp->attr.pbl_addr); return ret; } -static int reregister_mem(struct c4iw_dev *rhp, struct c4iw_pd *php, - struct c4iw_mr *mhp, int shift, int npages) -{ - u32 stag; - int ret; - - if (npages > mhp->attr.pbl_size) - return -ENOMEM; - - stag = mhp->attr.stag; - ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid, - FW_RI_STAG_NSMR, mhp->attr.perms, - mhp->attr.mw_bind_enable, mhp->attr.zbva, - mhp->attr.va_fbo, mhp->attr.len, shift - 12, - mhp->attr.pbl_size, mhp->attr.pbl_addr); - if (ret) - return ret; - - ret = finish_mem_reg(mhp, stag); - if (ret) - dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, - mhp->attr.pbl_addr); - - return ret; -} - static int alloc_pbl(struct c4iw_mr *mhp, int npages) { mhp->attr.pbl_addr = c4iw_pblpool_alloc(&mhp->rhp->rdev, npages << 3); if (!mhp->attr.pbl_addr) return -ENOMEM; mhp->attr.pbl_size = npages; return 0; } -static int build_phys_page_list(struct ib_phys_buf *buffer_list, - int num_phys_buf, u64 *iova_start, - u64 *total_size, int *npages, - int *shift, __be64 **page_list) -{ - u64 mask; - int i, j, n; - - mask = 0; - *total_size = 0; - for (i = 0; i < num_phys_buf; ++i) { - if (i != 0 && buffer_list[i].addr & ~PAGE_MASK) - return -EINVAL; - if (i != 0 && i != num_phys_buf - 1 && - (buffer_list[i].size & ~PAGE_MASK)) - return -EINVAL; - *total_size += buffer_list[i].size; - if (i > 0) - mask |= buffer_list[i].addr; - else - mask |= buffer_list[i].addr & PAGE_MASK; - if (i != num_phys_buf - 1) - mask |= buffer_list[i].addr + buffer_list[i].size; - else - mask |= (buffer_list[i].addr + buffer_list[i].size + - PAGE_SIZE - 1) & PAGE_MASK; - } - - /* Find largest page shift we can use to cover buffers */ - for (*shift = PAGE_SHIFT; *shift < PAGE_SHIFT + M_FW_RI_TPTE_PS; - ++(*shift)) - if ((1ULL << *shift) & mask) - break; - - buffer_list[0].size += buffer_list[0].addr & ((1ULL << *shift) - 1); - buffer_list[0].addr &= ~0ull << *shift; - - *npages = 0; - for (i = 0; i < num_phys_buf; ++i) - *npages += (buffer_list[i].size + - (1ULL << *shift) - 1) >> *shift; - - if (!*npages) - return -EINVAL; - - *page_list = kmalloc(sizeof(u64) * *npages, GFP_KERNEL); - if (!*page_list) - return -ENOMEM; - - n = 0; - for (i = 0; i < num_phys_buf; ++i) - for (j = 0; - j < (buffer_list[i].size + (1ULL << *shift) - 1) >> *shift; - ++j) - (*page_list)[n++] = cpu_to_be64(buffer_list[i].addr + - ((u64) j << *shift)); - - CTR6(KTR_IW_CXGBE, - "%s va 0x%llx mask 0x%llx shift %d len %lld pbl_size %d", __func__, - (unsigned long long)*iova_start, (unsigned long long)mask, *shift, - (unsigned long long)*total_size, *npages); - - return 0; - -} - -int c4iw_reregister_phys_mem(struct ib_mr *mr, int mr_rereg_mask, - struct ib_pd *pd, struct ib_phys_buf *buffer_list, - int num_phys_buf, int acc, u64 *iova_start) -{ - - struct c4iw_mr mh, *mhp; - struct c4iw_pd *php; - struct c4iw_dev *rhp; - __be64 *page_list = NULL; - int shift = 0; - u64 total_size = 0; - int npages = 0; - int ret; - - CTR3(KTR_IW_CXGBE, "%s ib_mr %p ib_pd %p", __func__, mr, pd); - - /* There can be no memory windows */ - if (atomic_read(&mr->usecnt)) - return -EINVAL; - - mhp = to_c4iw_mr(mr); - rhp = mhp->rhp; - php = to_c4iw_pd(mr->pd); - - /* make sure we are on the same adapter */ - if (rhp != php->rhp) - return -EINVAL; - - memcpy(&mh, mhp, sizeof *mhp); - - if (mr_rereg_mask & IB_MR_REREG_PD) - php = to_c4iw_pd(pd); - if (mr_rereg_mask & IB_MR_REREG_ACCESS) { - mh.attr.perms = c4iw_ib_to_tpt_access(acc); - mh.attr.mw_bind_enable = (acc & IB_ACCESS_MW_BIND) == - IB_ACCESS_MW_BIND; - } - if (mr_rereg_mask & IB_MR_REREG_TRANS) { - ret = build_phys_page_list(buffer_list, num_phys_buf, - iova_start, - &total_size, &npages, - &shift, &page_list); - if (ret) - return ret; - } - if (mr_exceeds_hw_limits(rhp, total_size)) { - kfree(page_list); - return -EINVAL; - } - ret = reregister_mem(rhp, php, &mh, shift, npages); - kfree(page_list); - if (ret) - return ret; - if (mr_rereg_mask & IB_MR_REREG_PD) - mhp->attr.pdid = php->pdid; - if (mr_rereg_mask & IB_MR_REREG_ACCESS) - mhp->attr.perms = c4iw_ib_to_tpt_access(acc); - if (mr_rereg_mask & IB_MR_REREG_TRANS) { - mhp->attr.zbva = 0; - mhp->attr.va_fbo = *iova_start; - mhp->attr.page_size = shift - 12; - mhp->attr.len = total_size; - mhp->attr.pbl_size = npages; - } - - return 0; -} - -struct ib_mr *c4iw_register_phys_mem(struct ib_pd *pd, - struct ib_phys_buf *buffer_list, - int num_phys_buf, int acc, u64 *iova_start) -{ - __be64 *page_list; - int shift; - u64 total_size; - int npages; - struct c4iw_dev *rhp; - struct c4iw_pd *php; - struct c4iw_mr *mhp; - int ret; - - CTR2(KTR_IW_CXGBE, "%s ib_pd %p", __func__, pd); - php = to_c4iw_pd(pd); - rhp = php->rhp; - - mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); - if (!mhp) - return ERR_PTR(-ENOMEM); - - mhp->rhp = rhp; - - /* First check that we have enough alignment */ - if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) { - ret = -EINVAL; - goto err; - } - - if (num_phys_buf > 1 && - ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK)) { - ret = -EINVAL; - goto err; - } - - ret = build_phys_page_list(buffer_list, num_phys_buf, iova_start, - &total_size, &npages, &shift, - &page_list); - if (ret) - goto err; - - if (mr_exceeds_hw_limits(rhp, total_size)) { - kfree(page_list); - ret = -EINVAL; - goto err; - } - ret = alloc_pbl(mhp, npages); - if (ret) { - kfree(page_list); - goto err; - } - - ret = write_pbl(&mhp->rhp->rdev, page_list, mhp->attr.pbl_addr, - npages); - kfree(page_list); - if (ret) - goto err_pbl; - - mhp->attr.pdid = php->pdid; - mhp->attr.zbva = 0; - - mhp->attr.perms = c4iw_ib_to_tpt_access(acc); - mhp->attr.va_fbo = *iova_start; - mhp->attr.page_size = shift - 12; - - mhp->attr.len = total_size; - mhp->attr.pbl_size = npages; - ret = register_mem(rhp, php, mhp, shift); - if (ret) - goto err_pbl; - - return &mhp->ibmr; - -err_pbl: - c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, - mhp->attr.pbl_size << 3); - -err: - kfree(mhp); - return ERR_PTR(ret); - -} - struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc) { struct c4iw_dev *rhp; struct c4iw_pd *php; struct c4iw_mr *mhp; int ret; u32 stag = T4_STAG_UNSET; CTR2(KTR_IW_CXGBE, "%s ib_pd %p", __func__, pd); php = to_c4iw_pd(pd); rhp = php->rhp; mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); if (!mhp) return ERR_PTR(-ENOMEM); mhp->rhp = rhp; mhp->attr.pdid = php->pdid; mhp->attr.perms = c4iw_ib_to_tpt_access(acc); mhp->attr.mw_bind_enable = (acc&IB_ACCESS_MW_BIND) == IB_ACCESS_MW_BIND; mhp->attr.zbva = 0; mhp->attr.va_fbo = 0; mhp->attr.page_size = 0; - mhp->attr.len = ~0UL; + mhp->attr.len = ~0ULL; mhp->attr.pbl_size = 0; ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, php->pdid, FW_RI_STAG_NSMR, mhp->attr.perms, - mhp->attr.mw_bind_enable, 0, 0, ~0UL, 0, 0, 0); + mhp->attr.mw_bind_enable, 0, 0, ~0ULL, 0, 0, 0); if (ret) goto err1; ret = finish_mem_reg(mhp, stag); if (ret) goto err2; return &mhp->ibmr; err2: dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, mhp->attr.pbl_addr); err1: kfree(mhp); return ERR_PTR(ret); } struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, - u64 virt, int acc, struct ib_udata *udata, int mr_id) + u64 virt, int acc, struct ib_udata *udata) { __be64 *pages; int shift, n, len; int i, k, entry; int err = 0; struct scatterlist *sg; struct c4iw_dev *rhp; struct c4iw_pd *php; struct c4iw_mr *mhp; CTR2(KTR_IW_CXGBE, "%s ib_pd %p", __func__, pd); if (length == ~0ULL) return ERR_PTR(-EINVAL); if ((length + start) < start) return ERR_PTR(-EINVAL); php = to_c4iw_pd(pd); rhp = php->rhp; if (mr_exceeds_hw_limits(rhp, length)) return ERR_PTR(-EINVAL); mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); if (!mhp) return ERR_PTR(-ENOMEM); mhp->rhp = rhp; mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0); if (IS_ERR(mhp->umem)) { err = PTR_ERR(mhp->umem); kfree(mhp); return ERR_PTR(err); } shift = ffs(mhp->umem->page_size) - 1; n = mhp->umem->nmap; err = alloc_pbl(mhp, n); if (err) goto err; pages = (__be64 *) __get_free_page(GFP_KERNEL); if (!pages) { err = -ENOMEM; goto err_pbl; } i = n = 0; for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) { len = sg_dma_len(sg) >> shift; for (k = 0; k < len; ++k) { pages[i++] = cpu_to_be64(sg_dma_address(sg) + mhp->umem->page_size * k); if (i == PAGE_SIZE / sizeof *pages) { err = write_pbl(&mhp->rhp->rdev, pages, mhp->attr.pbl_addr + (n << 3), i); if (err) goto pbl_done; n += i; i = 0; } } } if (i) err = write_pbl(&mhp->rhp->rdev, pages, mhp->attr.pbl_addr + (n << 3), i); pbl_done: free_page((unsigned long) pages); if (err) goto err_pbl; mhp->attr.pdid = php->pdid; mhp->attr.zbva = 0; mhp->attr.perms = c4iw_ib_to_tpt_access(acc); mhp->attr.va_fbo = virt; mhp->attr.page_size = shift - 12; mhp->attr.len = length; err = register_mem(rhp, php, mhp, shift); if (err) goto err_pbl; return &mhp->ibmr; err_pbl: c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, mhp->attr.pbl_size << 3); err: ib_umem_release(mhp->umem); kfree(mhp); return ERR_PTR(err); } -struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type) +struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, + struct ib_udata *udata) { struct c4iw_dev *rhp; struct c4iw_pd *php; struct c4iw_mw *mhp; u32 mmid; u32 stag = 0; int ret; + if (type != IB_MW_TYPE_1) + return ERR_PTR(-EINVAL); + php = to_c4iw_pd(pd); rhp = php->rhp; mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); if (!mhp) return ERR_PTR(-ENOMEM); ret = allocate_window(&rhp->rdev, &stag, php->pdid); if (ret) { kfree(mhp); return ERR_PTR(ret); } mhp->rhp = rhp; mhp->attr.pdid = php->pdid; mhp->attr.type = FW_RI_STAG_MW; mhp->attr.stag = stag; mmid = (stag) >> 8; mhp->ibmw.rkey = stag; if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { deallocate_window(&rhp->rdev, mhp->attr.stag); kfree(mhp); return ERR_PTR(-ENOMEM); } CTR4(KTR_IW_CXGBE, "%s mmid 0x%x mhp %p stag 0x%x", __func__, mmid, mhp, stag); return &(mhp->ibmw); } int c4iw_dealloc_mw(struct ib_mw *mw) { struct c4iw_dev *rhp; struct c4iw_mw *mhp; u32 mmid; mhp = to_c4iw_mw(mw); rhp = mhp->rhp; mmid = (mw->rkey) >> 8; remove_handle(rhp, &rhp->mmidr, mmid); deallocate_window(&rhp->rdev, mhp->attr.stag); kfree(mhp); CTR4(KTR_IW_CXGBE, "%s ib_mw %p mmid 0x%x ptr %p", __func__, mw, mmid, mhp); return 0; } -struct ib_mr *c4iw_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth) +struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, + enum ib_mr_type mr_type, + u32 max_num_sg) { struct c4iw_dev *rhp; struct c4iw_pd *php; struct c4iw_mr *mhp; u32 mmid; u32 stag = 0; int ret = 0; + int length = roundup(max_num_sg * sizeof(u64), 32); php = to_c4iw_pd(pd); rhp = php->rhp; + + if (mr_type != IB_MR_TYPE_MEM_REG || + max_num_sg > t4_max_fr_depth( + rhp->rdev.adap->params.ulptx_memwrite_dsgl && use_dsgl)) + return ERR_PTR(-EINVAL); + mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); if (!mhp) { ret = -ENOMEM; goto err; } + mhp->mpl = dma_alloc_coherent(rhp->ibdev.dma_device, + length, &mhp->mpl_addr, GFP_KERNEL); + if (!mhp->mpl) { + ret = -ENOMEM; + goto err_mpl; + } + mhp->max_mpl_len = length; + mhp->rhp = rhp; - ret = alloc_pbl(mhp, pbl_depth); + ret = alloc_pbl(mhp, max_num_sg); if (ret) goto err1; - mhp->attr.pbl_size = pbl_depth; + mhp->attr.pbl_size = max_num_sg; ret = allocate_stag(&rhp->rdev, &stag, php->pdid, - mhp->attr.pbl_size, mhp->attr.pbl_addr); + mhp->attr.pbl_size, mhp->attr.pbl_addr); if (ret) goto err2; mhp->attr.pdid = php->pdid; mhp->attr.type = FW_RI_STAG_NSMR; mhp->attr.stag = stag; - mhp->attr.state = 1; + mhp->attr.state = 0; mmid = (stag) >> 8; mhp->ibmr.rkey = mhp->ibmr.lkey = stag; if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { ret = -ENOMEM; goto err3; } - CTR4(KTR_IW_CXGBE, "%s mmid 0x%x mhp %p stag 0x%x", __func__, mmid, mhp, - stag); + PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag); return &(mhp->ibmr); err3: dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size, mhp->attr.pbl_addr); err2: c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, mhp->attr.pbl_size << 3); err1: + dma_free_coherent(rhp->ibdev.dma_device, + mhp->max_mpl_len, mhp->mpl, mhp->mpl_addr); +err_mpl: kfree(mhp); err: return ERR_PTR(ret); } - -struct ib_fast_reg_page_list *c4iw_alloc_fastreg_pbl(struct ib_device *device, - int page_list_len) +static int c4iw_set_page(struct ib_mr *ibmr, u64 addr) { - struct c4iw_fr_page_list *c4pl; - struct c4iw_dev *dev = to_c4iw_dev(device); - bus_addr_t dma_addr; - int size = sizeof *c4pl + page_list_len * sizeof(u64); + struct c4iw_mr *mhp = to_c4iw_mr(ibmr); - c4pl = contigmalloc(size, - M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0); - if (c4pl) - dma_addr = vtophys(c4pl); - else - return ERR_PTR(-ENOMEM); + if (unlikely(mhp->mpl_len == mhp->max_mpl_len)) + return -ENOMEM; - pci_unmap_addr_set(c4pl, mapping, dma_addr); - c4pl->dma_addr = dma_addr; - c4pl->dev = dev; - c4pl->size = size; - c4pl->ibpl.page_list = (u64 *)(c4pl + 1); - c4pl->ibpl.max_page_list_len = page_list_len; + mhp->mpl[mhp->mpl_len++] = addr; - return &c4pl->ibpl; + return 0; } -void c4iw_free_fastreg_pbl(struct ib_fast_reg_page_list *ibpl) +int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset) { - struct c4iw_fr_page_list *c4pl = to_c4iw_fr_page_list(ibpl); - contigfree(c4pl, c4pl->size, M_DEVBUF); + struct c4iw_mr *mhp = to_c4iw_mr(ibmr); + + mhp->mpl_len = 0; + + return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, c4iw_set_page); } + int c4iw_dereg_mr(struct ib_mr *ib_mr) { struct c4iw_dev *rhp; struct c4iw_mr *mhp; u32 mmid; CTR2(KTR_IW_CXGBE, "%s ib_mr %p", __func__, ib_mr); - /* There can be no memory windows */ - if (atomic_read(&ib_mr->usecnt)) - return -EINVAL; mhp = to_c4iw_mr(ib_mr); rhp = mhp->rhp; mmid = mhp->attr.stag >> 8; remove_handle(rhp, &rhp->mmidr, mmid); dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, mhp->attr.pbl_addr); if (mhp->attr.pbl_size) c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, mhp->attr.pbl_size << 3); if (mhp->kva) kfree((void *) (unsigned long) mhp->kva); if (mhp->umem) ib_umem_release(mhp->umem); CTR3(KTR_IW_CXGBE, "%s mmid 0x%x ptr %p", __func__, mmid, mhp); kfree(mhp); return 0; +} + +void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey) +{ + struct c4iw_mr *mhp; + unsigned long flags; + + spin_lock_irqsave(&rhp->lock, flags); + mhp = get_mhp(rhp, rkey >> 8); + if (mhp) + mhp->attr.state = 0; + spin_unlock_irqrestore(&rhp->lock, flags); } #endif Index: projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/provider.c =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/provider.c (revision 331317) +++ projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/provider.c (revision 331318) @@ -1,526 +1,533 @@ /* * Copyright (c) 2009-2013, 2016 Chelsio, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #define LINUXKPI_PARAM_PREFIX iw_cxgbe_ #include "opt_inet.h" #ifdef TCP_OFFLOAD #include #include #include #include #include "iw_cxgbe.h" #include "user.h" - +extern int use_dsgl; static int fastreg_support = 1; module_param(fastreg_support, int, 0644); MODULE_PARM_DESC(fastreg_support, "Advertise fastreg support (default = 1)"); static int c4iw_modify_port(struct ib_device *ibdev, u8 port, int port_modify_mask, struct ib_port_modify *props) { return -ENOSYS; } static struct ib_ah *c4iw_ah_create(struct ib_pd *pd, struct ib_ah_attr *ah_attr) { return ERR_PTR(-ENOSYS); } static int c4iw_ah_destroy(struct ib_ah *ah) { return -ENOSYS; } static int c4iw_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) { return -ENOSYS; } static int c4iw_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) { return -ENOSYS; } static int c4iw_process_mad(struct ib_device *ibdev, int mad_flags, - u8 port_num, struct ib_wc *in_wc, - struct ib_grh *in_grh, struct ib_mad *in_mad, - struct ib_mad *out_mad) + u8 port_num, const struct ib_wc *in_wc, + const struct ib_grh *in_grh, + const struct ib_mad_hdr *in_mad, + size_t in_mad_size, + struct ib_mad_hdr *out_mad, + size_t *out_mad_size, + u16 *out_mad_pkey_index) + { return -ENOSYS; } -static int c4iw_dealloc_ucontext(struct ib_ucontext *context) +void _c4iw_free_ucontext(struct kref *kref) { - struct c4iw_dev *rhp = to_c4iw_dev(context->device); - struct c4iw_ucontext *ucontext = to_c4iw_ucontext(context); + struct c4iw_ucontext *ucontext; + struct c4iw_dev *rhp; struct c4iw_mm_entry *mm, *tmp; - CTR2(KTR_IW_CXGBE, "%s context %p", __func__, context); + ucontext = container_of(kref, struct c4iw_ucontext, kref); + rhp = to_c4iw_dev(ucontext->ibucontext.device); + + CTR2(KTR_IW_CXGBE, "%s ucontext %p", __func__, ucontext); list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry) kfree(mm); c4iw_release_dev_ucontext(&rhp->rdev, &ucontext->uctx); kfree(ucontext); +} + +static int c4iw_dealloc_ucontext(struct ib_ucontext *context) +{ + struct c4iw_ucontext *ucontext = to_c4iw_ucontext(context); + + CTR2(KTR_IW_CXGBE, "%s context %p", __func__, context); + c4iw_put_ucontext(ucontext); return 0; } static struct ib_ucontext *c4iw_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata) { struct c4iw_ucontext *context; struct c4iw_dev *rhp = to_c4iw_dev(ibdev); + static int warned; + struct c4iw_alloc_ucontext_resp uresp; + int ret = 0; + struct c4iw_mm_entry *mm = NULL; - CTR2(KTR_IW_CXGBE, "%s ibdev %p", __func__, ibdev); + PDBG("%s ibdev %p\n", __func__, ibdev); context = kzalloc(sizeof(*context), GFP_KERNEL); - if (!context) - return ERR_PTR(-ENOMEM); + if (!context) { + ret = -ENOMEM; + goto err; + } + c4iw_init_dev_ucontext(&rhp->rdev, &context->uctx); INIT_LIST_HEAD(&context->mmaps); spin_lock_init(&context->mmap_lock); + kref_init(&context->kref); + + if (udata->outlen < sizeof(uresp) - sizeof(uresp.reserved)) { + if (!warned++) + log(LOG_ERR, "%s Warning - downlevel libcxgb4 " + "(non-fatal), device status page disabled.\n", + __func__); + rhp->rdev.flags |= T4_STATUS_PAGE_DISABLED; + } else { + + mm = kmalloc(sizeof *mm, GFP_KERNEL); + if (!mm) + goto err_free; + + uresp.status_page_size = PAGE_SIZE; + + spin_lock(&context->mmap_lock); + uresp.status_page_key = context->key; + context->key += PAGE_SIZE; + spin_unlock(&context->mmap_lock); + + ret = ib_copy_to_udata(udata, &uresp, + sizeof(uresp) - sizeof(uresp.reserved)); + if (ret) + goto err_mm; + + mm->key = uresp.status_page_key; + mm->addr = vtophys(rhp->rdev.status_page); + mm->len = PAGE_SIZE; + insert_mmap(context, mm); + } return &context->ibucontext; +err_mm: + kfree(mm); +err_free: + kfree(context); +err: + return ERR_PTR(ret); } -#ifdef DOT5 -static inline pgprot_t t4_pgprot_wc(pgprot_t prot) -{ - return pgprot_writecombine(prot); -} -#endif - static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) { int len = vma->vm_end - vma->vm_start; u32 key = vma->vm_pgoff << PAGE_SHIFT; struct c4iw_rdev *rdev; int ret = 0; struct c4iw_mm_entry *mm; struct c4iw_ucontext *ucontext; - u64 addr, paddr; + u64 addr = 0; - u64 va_regs_res = 0, va_udbs_res = 0; - u64 len_regs_res = 0, len_udbs_res = 0; + CTR4(KTR_IW_CXGBE, "%s:1 ctx %p vma %p, vm_start %u", __func__, + context, vma, vma->vm_start); - CTR3(KTR_IW_CXGBE, "%s:1 ctx %p vma %p", __func__, context, vma); - CTR4(KTR_IW_CXGBE, "%s:1a pgoff 0x%lx key 0x%x len %d", __func__, vma->vm_pgoff, key, len); if (vma->vm_start & (PAGE_SIZE-1)) { CTR3(KTR_IW_CXGBE, "%s:2 unaligned vm_start %u vma %p", __func__, vma->vm_start, vma); return -EINVAL; } rdev = &(to_c4iw_dev(context->device)->rdev); ucontext = to_c4iw_ucontext(context); mm = remove_mmap(ucontext, key, len); if (!mm) { CTR4(KTR_IW_CXGBE, "%s:3 ucontext %p key %u len %u", __func__, ucontext, key, len); return -EINVAL; } addr = mm->addr; kfree(mm); - va_regs_res = (u64)rman_get_virtual(rdev->adap->regs_res); - len_regs_res = (u64)rman_get_size(rdev->adap->regs_res); - va_udbs_res = (u64)rman_get_virtual(rdev->adap->udbs_res); - len_udbs_res = (u64)rman_get_size(rdev->adap->udbs_res); + /* user DB-GTS registers if addr in udbs_res range, + * else WQ or CQ memory. + * */ + if (rdev->adap->iwt.wc_en && addr >= rdev->bar2_pa && + addr < rdev->bar2_pa + rdev->bar2_len) + vma->vm_page_prot = t4_pgprot_wc(vma->vm_page_prot); - CTR6(KTR_IW_CXGBE, - "%s:4 addr %p, masync region %p:%p, udb region %p:%p", __func__, - addr, va_regs_res, va_regs_res+len_regs_res, va_udbs_res, - va_udbs_res+len_udbs_res); - - if (addr >= va_regs_res && addr < va_regs_res + len_regs_res) { - CTR4(KTR_IW_CXGBE, "%s:5 MA_SYNC addr %p region %p, reglen %u", - __func__, addr, va_regs_res, len_regs_res); - /* - * MA_SYNC register... - */ - paddr = vtophys(addr); - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - ret = io_remap_pfn_range(vma, vma->vm_start, - paddr >> PAGE_SHIFT, - len, vma->vm_page_prot); - } else { - - if (addr >= va_udbs_res && addr < va_udbs_res + len_udbs_res) { - /* - * Map user DB or OCQP memory... - */ - paddr = vtophys(addr); - CTR4(KTR_IW_CXGBE, - "%s:6 USER DB-GTS addr %p region %p, reglen %u", - __func__, addr, va_udbs_res, len_udbs_res); -#ifdef DOT5 - if (!is_t4(rdev->lldi.adapter_type) && map_udb_as_wc) - vma->vm_page_prot = t4_pgprot_wc(vma->vm_page_prot); - else -#endif - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - ret = io_remap_pfn_range(vma, vma->vm_start, - paddr >> PAGE_SHIFT, - len, vma->vm_page_prot); - } else { - /* - * Map WQ or CQ contig dma memory... - */ - CTR4(KTR_IW_CXGBE, - "%s:7 WQ/CQ addr %p vm_start %u vma %p", __func__, - addr, vma->vm_start, vma); - ret = io_remap_pfn_range(vma, vma->vm_start, - addr >> PAGE_SHIFT, - len, vma->vm_page_prot); - } - } - CTR4(KTR_IW_CXGBE, "%s:8 ctx %p vma %p ret %u", __func__, context, vma, + ret = io_remap_pfn_range(vma, vma->vm_start, addr >> PAGE_SHIFT, + len, vma->vm_page_prot); + CTR4(KTR_IW_CXGBE, "%s:4 ctx %p vma %p ret %u", __func__, context, vma, ret); return ret; } static int c4iw_deallocate_pd(struct ib_pd *pd) { struct c4iw_pd *php = to_c4iw_pd(pd); struct c4iw_dev *rhp = php->rhp; CTR3(KTR_IW_CXGBE, "%s: pd %p, pdid 0x%x", __func__, pd, php->pdid); c4iw_put_resource(&rhp->rdev.resource.pdid_table, php->pdid); mutex_lock(&rhp->rdev.stats.lock); rhp->rdev.stats.pd.cur--; mutex_unlock(&rhp->rdev.stats.lock); kfree(php); return (0); } static struct ib_pd * c4iw_allocate_pd(struct ib_device *ibdev, struct ib_ucontext *context, struct ib_udata *udata) { struct c4iw_pd *php; u32 pdid; struct c4iw_dev *rhp; CTR4(KTR_IW_CXGBE, "%s: ibdev %p, context %p, data %p", __func__, ibdev, context, udata); rhp = (struct c4iw_dev *) ibdev; pdid = c4iw_get_resource(&rhp->rdev.resource.pdid_table); if (!pdid) return ERR_PTR(-EINVAL); php = kzalloc(sizeof(*php), GFP_KERNEL); if (!php) { c4iw_put_resource(&rhp->rdev.resource.pdid_table, pdid); return ERR_PTR(-ENOMEM); } php->pdid = pdid; php->rhp = rhp; if (context) { if (ib_copy_to_udata(udata, &php->pdid, sizeof(u32))) { c4iw_deallocate_pd(&php->ibpd); return ERR_PTR(-EFAULT); } } mutex_lock(&rhp->rdev.stats.lock); rhp->rdev.stats.pd.cur++; if (rhp->rdev.stats.pd.cur > rhp->rdev.stats.pd.max) rhp->rdev.stats.pd.max = rhp->rdev.stats.pd.cur; mutex_unlock(&rhp->rdev.stats.lock); CTR6(KTR_IW_CXGBE, "%s: ibdev %p, context %p, data %p, pddid 0x%x, pd %p", __func__, ibdev, context, udata, pdid, php); return (&php->ibpd); } static int c4iw_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) { CTR5(KTR_IW_CXGBE, "%s ibdev %p, port %d, index %d, pkey %p", __func__, ibdev, port, index, pkey); *pkey = 0; return (0); } static int c4iw_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid) { struct c4iw_dev *dev; struct port_info *pi; struct adapter *sc; CTR5(KTR_IW_CXGBE, "%s ibdev %p, port %d, index %d, gid %p", __func__, ibdev, port, index, gid); memset(&gid->raw[0], 0, sizeof(gid->raw)); dev = to_c4iw_dev(ibdev); sc = dev->rdev.adap; if (port == 0 || port > sc->params.nports) return (-EINVAL); pi = sc->port[port - 1]; memcpy(&gid->raw[0], pi->vi[0].hw_addr, ETHER_ADDR_LEN); return (0); } static int -c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *props) +c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *props, + struct ib_udata *uhw) { struct c4iw_dev *dev = to_c4iw_dev(ibdev); struct adapter *sc = dev->rdev.adap; - const int spg_ndesc = sc->params.sge.spg_len / EQ_ESIZE; CTR3(KTR_IW_CXGBE, "%s ibdev %p, props %p", __func__, ibdev, props); + if (uhw->inlen || uhw->outlen) + return -EINVAL; + memset(props, 0, sizeof *props); memcpy(&props->sys_image_guid, sc->port[0]->vi[0].hw_addr, ETHER_ADDR_LEN); props->hw_ver = sc->params.chipid; props->fw_ver = sc->params.fw_vers; props->device_cap_flags = dev->device_cap_flags; props->page_size_cap = T4_PAGESIZE_MASK; props->vendor_id = pci_get_vendor(sc->dev); props->vendor_part_id = pci_get_device(sc->dev); props->max_mr_size = T4_MAX_MR_SIZE; props->max_qp = sc->vres.qp.size / 2; - props->max_qp_wr = T4_MAX_QP_DEPTH(spg_ndesc); + props->max_qp_wr = dev->rdev.hw_queue.t4_max_qp_depth; props->max_sge = T4_MAX_RECV_SGE; props->max_sge_rd = 1; props->max_res_rd_atom = sc->params.max_ird_adapter; props->max_qp_rd_atom = min(sc->params.max_ordird_qp, c4iw_max_read_depth); props->max_qp_init_rd_atom = props->max_qp_rd_atom; props->max_cq = sc->vres.qp.size; - props->max_cqe = T4_MAX_CQ_DEPTH; + props->max_cqe = dev->rdev.hw_queue.t4_max_cq_depth; props->max_mr = c4iw_num_stags(&dev->rdev); props->max_pd = T4_MAX_NUM_PD; props->local_ca_ack_delay = 0; - props->max_fast_reg_page_list_len = T4_MAX_FR_DEPTH; + props->max_fast_reg_page_list_len = + t4_max_fr_depth(sc->params.ulptx_memwrite_dsgl && use_dsgl); return (0); } /* * Returns -errno on failure. */ static int c4iw_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) { struct c4iw_dev *dev; struct adapter *sc; struct port_info *pi; struct ifnet *ifp; CTR4(KTR_IW_CXGBE, "%s ibdev %p, port %d, props %p", __func__, ibdev, port, props); dev = to_c4iw_dev(ibdev); sc = dev->rdev.adap; if (port > sc->params.nports) return (-EINVAL); pi = sc->port[port - 1]; ifp = pi->vi[0].ifp; memset(props, 0, sizeof(struct ib_port_attr)); props->max_mtu = IB_MTU_4096; if (ifp->if_mtu >= 4096) props->active_mtu = IB_MTU_4096; else if (ifp->if_mtu >= 2048) props->active_mtu = IB_MTU_2048; else if (ifp->if_mtu >= 1024) props->active_mtu = IB_MTU_1024; else if (ifp->if_mtu >= 512) props->active_mtu = IB_MTU_512; else props->active_mtu = IB_MTU_256; props->state = pi->link_cfg.link_ok ? IB_PORT_ACTIVE : IB_PORT_DOWN; props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_SNMP_TUNNEL_SUP | IB_PORT_REINIT_SUP | IB_PORT_DEVICE_MGMT_SUP | IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP; props->gid_tbl_len = 1; props->pkey_tbl_len = 1; props->active_width = 2; props->active_speed = 2; props->max_msg_sz = -1; return 0; } static int c4iw_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; int err; immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; err = ib_query_port(ibdev, port_num, &attr); if (err) return err; immutable->pkey_tbl_len = attr.pkey_tbl_len; immutable->gid_tbl_len = attr.gid_tbl_len; return 0; } /* * Returns -errno on error. */ int c4iw_register_device(struct c4iw_dev *dev) { struct adapter *sc = dev->rdev.adap; struct ib_device *ibdev = &dev->ibdev; struct iw_cm_verbs *iwcm; int ret; CTR3(KTR_IW_CXGBE, "%s c4iw_dev %p, adapter %p", __func__, dev, sc); BUG_ON(!sc->port[0]); strlcpy(ibdev->name, device_get_nameunit(sc->dev), sizeof(ibdev->name)); memset(&ibdev->node_guid, 0, sizeof(ibdev->node_guid)); memcpy(&ibdev->node_guid, sc->port[0]->vi[0].hw_addr, ETHER_ADDR_LEN); ibdev->owner = THIS_MODULE; dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW; if (fastreg_support) dev->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; ibdev->local_dma_lkey = 0; ibdev->uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | (1ull << IB_USER_VERBS_CMD_REG_MR) | (1ull << IB_USER_VERBS_CMD_DEREG_MR) | (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | (1ull << IB_USER_VERBS_CMD_CREATE_QP) | (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | (1ull << IB_USER_VERBS_CMD_QUERY_QP) | (1ull << IB_USER_VERBS_CMD_POLL_CQ) | (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | (1ull << IB_USER_VERBS_CMD_POST_SEND) | (1ull << IB_USER_VERBS_CMD_POST_RECV); ibdev->node_type = RDMA_NODE_RNIC; strlcpy(ibdev->node_desc, C4IW_NODE_DESC, sizeof(ibdev->node_desc)); ibdev->phys_port_cnt = sc->params.nports; ibdev->num_comp_vectors = 1; ibdev->dma_device = NULL; ibdev->query_device = c4iw_query_device; ibdev->query_port = c4iw_query_port; ibdev->modify_port = c4iw_modify_port; ibdev->query_pkey = c4iw_query_pkey; ibdev->query_gid = c4iw_query_gid; ibdev->alloc_ucontext = c4iw_alloc_ucontext; ibdev->dealloc_ucontext = c4iw_dealloc_ucontext; ibdev->mmap = c4iw_mmap; ibdev->alloc_pd = c4iw_allocate_pd; ibdev->dealloc_pd = c4iw_deallocate_pd; ibdev->create_ah = c4iw_ah_create; ibdev->destroy_ah = c4iw_ah_destroy; ibdev->create_qp = c4iw_create_qp; ibdev->modify_qp = c4iw_ib_modify_qp; ibdev->query_qp = c4iw_ib_query_qp; ibdev->destroy_qp = c4iw_destroy_qp; ibdev->create_cq = c4iw_create_cq; ibdev->destroy_cq = c4iw_destroy_cq; ibdev->resize_cq = c4iw_resize_cq; ibdev->poll_cq = c4iw_poll_cq; ibdev->get_dma_mr = c4iw_get_dma_mr; - ibdev->reg_phys_mr = c4iw_register_phys_mem; - ibdev->rereg_phys_mr = c4iw_reregister_phys_mem; ibdev->reg_user_mr = c4iw_reg_user_mr; ibdev->dereg_mr = c4iw_dereg_mr; ibdev->alloc_mw = c4iw_alloc_mw; - ibdev->bind_mw = c4iw_bind_mw; ibdev->dealloc_mw = c4iw_dealloc_mw; - ibdev->alloc_fast_reg_mr = c4iw_alloc_fast_reg_mr; - ibdev->alloc_fast_reg_page_list = c4iw_alloc_fastreg_pbl; - ibdev->free_fast_reg_page_list = c4iw_free_fastreg_pbl; + ibdev->alloc_mr = c4iw_alloc_mr; + ibdev->map_mr_sg = c4iw_map_mr_sg; ibdev->attach_mcast = c4iw_multicast_attach; ibdev->detach_mcast = c4iw_multicast_detach; ibdev->process_mad = c4iw_process_mad; ibdev->req_notify_cq = c4iw_arm_cq; ibdev->post_send = c4iw_post_send; ibdev->post_recv = c4iw_post_receive; ibdev->uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION; ibdev->get_port_immutable = c4iw_port_immutable; iwcm = kmalloc(sizeof(*iwcm), GFP_KERNEL); if (iwcm == NULL) return (-ENOMEM); iwcm->connect = c4iw_connect; iwcm->accept = c4iw_accept_cr; iwcm->reject = c4iw_reject_cr; - iwcm->create_listen_ep = c4iw_create_listen_ep; - iwcm->destroy_listen_ep = c4iw_destroy_listen_ep; - iwcm->newconn = process_newconn; + iwcm->create_listen = c4iw_create_listen; + iwcm->destroy_listen = c4iw_destroy_listen; iwcm->add_ref = c4iw_qp_add_ref; iwcm->rem_ref = c4iw_qp_rem_ref; iwcm->get_qp = c4iw_get_qp; ibdev->iwcm = iwcm; ret = ib_register_device(&dev->ibdev, NULL); if (ret) kfree(iwcm); return (ret); } void c4iw_unregister_device(struct c4iw_dev *dev) { CTR3(KTR_IW_CXGBE, "%s c4iw_dev %p, adapter %p", __func__, dev, dev->rdev.adap); ib_unregister_device(&dev->ibdev); kfree(dev->ibdev.iwcm); return; } #endif Index: projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/qp.c =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/qp.c (revision 331317) +++ projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/qp.c (revision 331318) @@ -1,1762 +1,1949 @@ /* * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #ifdef TCP_OFFLOAD #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct sge_iq; struct rss_header; struct cpl_set_tcb_rpl; #include #include "offload.h" #include "tom/t4_tom.h" #include "iw_cxgbe.h" #include "user.h" - +extern int use_dsgl; static int creds(struct toepcb *toep, struct inpcb *inp, size_t wrsize); +static int max_fr_immd = T4_MAX_FR_IMMD;//SYSCTL parameter later... - -static void set_state(struct c4iw_qp *qhp, enum c4iw_qp_state state) +static int alloc_ird(struct c4iw_dev *dev, u32 ird) { - unsigned long flag; - spin_lock_irqsave(&qhp->lock, flag); - qhp->attr.state = state; - spin_unlock_irqrestore(&qhp->lock, flag); -} + int ret = 0; -static void dealloc_host_sq(struct c4iw_rdev *rdev, struct t4_sq *sq) -{ + spin_lock_irq(&dev->lock); + if (ird <= dev->avail_ird) + dev->avail_ird -= ird; + else + ret = -ENOMEM; + spin_unlock_irq(&dev->lock); - contigfree(sq->queue, sq->memsize, M_DEVBUF); + if (ret) + log(LOG_WARNING, "%s: device IRD resources exhausted\n", + device_get_nameunit(dev->rdev.adap->dev)); + + return ret; } -static void dealloc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq) +static void free_ird(struct c4iw_dev *dev, int ird) { - - dealloc_host_sq(rdev, sq); + spin_lock_irq(&dev->lock); + dev->avail_ird += ird; + spin_unlock_irq(&dev->lock); } -static int alloc_host_sq(struct c4iw_rdev *rdev, struct t4_sq *sq) +static void set_state(struct c4iw_qp *qhp, enum c4iw_qp_state state) { - sq->queue = contigmalloc(sq->memsize, M_DEVBUF, M_NOWAIT, 0ul, ~0ul, - 4096, 0); - - if (sq->queue) - sq->dma_addr = vtophys(sq->queue); - else - return -ENOMEM; - sq->phys_addr = vtophys(sq->queue); - pci_unmap_addr_set(sq, mapping, sq->dma_addr); - CTR4(KTR_IW_CXGBE, "%s sq %p dma_addr %p phys_addr %p", __func__, - sq->queue, sq->dma_addr, sq->phys_addr); - return 0; + unsigned long flag; + spin_lock_irqsave(&qhp->lock, flag); + qhp->attr.state = state; + spin_unlock_irqrestore(&qhp->lock, flag); } static int destroy_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, struct c4iw_dev_ucontext *uctx) { + struct c4iw_dev *rhp = rdev_to_c4iw_dev(rdev); /* * uP clears EQ contexts when the connection exits rdma mode, * so no need to post a RESET WR for these EQs. */ - contigfree(wq->rq.queue, wq->rq.memsize, M_DEVBUF); - dealloc_sq(rdev, &wq->sq); + dma_free_coherent(rhp->ibdev.dma_device, + wq->rq.memsize, wq->rq.queue, + dma_unmap_addr(&wq->rq, mapping)); + dma_free_coherent(rhp->ibdev.dma_device, + wq->sq.memsize, wq->sq.queue, + dma_unmap_addr(&wq->sq, mapping)); c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size); kfree(wq->rq.sw_rq); kfree(wq->sq.sw_sq); c4iw_put_qpid(rdev, wq->rq.qid, uctx); c4iw_put_qpid(rdev, wq->sq.qid, uctx); return 0; } static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, struct t4_cq *rcq, struct t4_cq *scq, struct c4iw_dev_ucontext *uctx) { struct adapter *sc = rdev->adap; + struct c4iw_dev *rhp = rdev_to_c4iw_dev(rdev); int user = (uctx != &rdev->uctx); struct fw_ri_res_wr *res_wr; struct fw_ri_res *res; int wr_len; struct c4iw_wr_wait wr_wait; - int ret; + int ret = 0; int eqsize; struct wrqe *wr; - const int spg_ndesc = sc->params.sge.spg_len / EQ_ESIZE; + u64 sq_bar2_qoffset = 0, rq_bar2_qoffset = 0; wq->sq.qid = c4iw_get_qpid(rdev, uctx); if (!wq->sq.qid) return -ENOMEM; wq->rq.qid = c4iw_get_qpid(rdev, uctx); - if (!wq->rq.qid) - goto err1; + if (!wq->rq.qid) { + ret = -ENOMEM; + goto free_sq_qid; + } if (!user) { wq->sq.sw_sq = kzalloc(wq->sq.size * sizeof *wq->sq.sw_sq, GFP_KERNEL); - if (!wq->sq.sw_sq) - goto err2; + if (!wq->sq.sw_sq) { + ret = -ENOMEM; + goto free_rq_qid; + } wq->rq.sw_rq = kzalloc(wq->rq.size * sizeof *wq->rq.sw_rq, GFP_KERNEL); - if (!wq->rq.sw_rq) - goto err3; + if (!wq->rq.sw_rq) { + ret = -ENOMEM; + goto free_sw_sq; + } } - /* RQT must be a power of 2. */ - wq->rq.rqt_size = roundup_pow_of_two(wq->rq.size); + /* + * RQT must be a power of 2 and at least 16 deep. + */ + wq->rq.rqt_size = roundup_pow_of_two(max_t(u16, wq->rq.size, 16)); wq->rq.rqt_hwaddr = c4iw_rqtpool_alloc(rdev, wq->rq.rqt_size); - if (!wq->rq.rqt_hwaddr) - goto err4; + if (!wq->rq.rqt_hwaddr) { + ret = -ENOMEM; + goto free_sw_rq; + } - if (alloc_host_sq(rdev, &wq->sq)) - goto err5; - + /*QP memory, allocate DMAable memory for Send & Receive Queues */ + wq->sq.queue = dma_alloc_coherent(rhp->ibdev.dma_device, wq->sq.memsize, + &(wq->sq.dma_addr), GFP_KERNEL); + if (!wq->sq.queue) { + ret = -ENOMEM; + goto free_hwaddr; + } + wq->sq.phys_addr = vtophys(wq->sq.queue); + dma_unmap_addr_set(&wq->sq, mapping, wq->sq.dma_addr); memset(wq->sq.queue, 0, wq->sq.memsize); - pci_unmap_addr_set(&wq->sq, mapping, wq->sq.dma_addr); - wq->rq.queue = contigmalloc(wq->rq.memsize, - M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0); - if (wq->rq.queue) - wq->rq.dma_addr = vtophys(wq->rq.queue); - else - goto err6; - CTR5(KTR_IW_CXGBE, - "%s sq base va 0x%p pa 0x%llx rq base va 0x%p pa 0x%llx", __func__, - wq->sq.queue, (unsigned long long)vtophys(wq->sq.queue), - wq->rq.queue, (unsigned long long)vtophys(wq->rq.queue)); + wq->rq.queue = dma_alloc_coherent(rhp->ibdev.dma_device, + wq->rq.memsize, &(wq->rq.dma_addr), GFP_KERNEL); + if (!wq->rq.queue) { + ret = -ENOMEM; + goto free_sq_dma; + } + wq->rq.phys_addr = vtophys(wq->rq.queue); + dma_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr); memset(wq->rq.queue, 0, wq->rq.memsize); - pci_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr); - wq->db = (void *)((unsigned long)rman_get_virtual(sc->regs_res) + - sc->sge_kdoorbell_reg); - wq->gts = (void *)((unsigned long)rman_get_virtual(rdev->adap->regs_res) - + sc->sge_gts_reg); + CTR5(KTR_IW_CXGBE, + "%s QP sq base va 0x%p pa 0x%llx rq base va 0x%p pa 0x%llx", + __func__, + wq->sq.queue, (unsigned long long)wq->sq.phys_addr, + wq->rq.queue, (unsigned long long)wq->rq.phys_addr); + + /* Doorbell/WC regions, determine the BAR2 queue offset and qid. */ + t4_bar2_sge_qregs(rdev->adap, wq->sq.qid, T4_BAR2_QTYPE_EGRESS, user, + &sq_bar2_qoffset, &wq->sq.bar2_qid); + t4_bar2_sge_qregs(rdev->adap, wq->rq.qid, T4_BAR2_QTYPE_EGRESS, user, + &rq_bar2_qoffset, &wq->rq.bar2_qid); + if (user) { - wq->sq.udb = (u64)((char*)rman_get_virtual(rdev->adap->udbs_res) + - (wq->sq.qid << rdev->qpshift)); - wq->sq.udb &= PAGE_MASK; - wq->rq.udb = (u64)((char*)rman_get_virtual(rdev->adap->udbs_res) + - (wq->rq.qid << rdev->qpshift)); - wq->rq.udb &= PAGE_MASK; + /* Compute BAR2 DB/WC physical address(page-aligned) for + * Userspace mapping. + */ + wq->sq.bar2_pa = (rdev->bar2_pa + sq_bar2_qoffset) & PAGE_MASK; + wq->rq.bar2_pa = (rdev->bar2_pa + rq_bar2_qoffset) & PAGE_MASK; + CTR3(KTR_IW_CXGBE, + "%s BAR2 DB/WC sq base pa 0x%llx rq base pa 0x%llx", + __func__, (unsigned long long)wq->sq.bar2_pa, + (unsigned long long)wq->rq.bar2_pa); + } else { + /* Compute BAR2 DB/WC virtual address to access in kernel. */ + wq->sq.bar2_va = (void __iomem *)((u64)rdev->bar2_kva + + sq_bar2_qoffset); + wq->rq.bar2_va = (void __iomem *)((u64)rdev->bar2_kva + + rq_bar2_qoffset); + CTR3(KTR_IW_CXGBE, "%s BAR2 DB/WC sq base va %p rq base va %p", + __func__, (unsigned long long)wq->sq.bar2_va, + (unsigned long long)wq->rq.bar2_va); } + wq->rdev = rdev; wq->rq.msn = 1; /* build fw_ri_res_wr */ wr_len = sizeof *res_wr + 2 * sizeof *res; wr = alloc_wrqe(wr_len, &sc->sge.mgmtq); - if (wr == NULL) - return (0); + if (wr == NULL) { + ret = -ENOMEM; + goto free_rq_dma; + } res_wr = wrtod(wr); memset(res_wr, 0, wr_len); res_wr->op_nres = cpu_to_be32( V_FW_WR_OP(FW_RI_RES_WR) | V_FW_RI_RES_WR_NRES(2) | F_FW_WR_COMPL); res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16)); res_wr->cookie = (unsigned long) &wr_wait; res = res_wr->res; res->u.sqrq.restype = FW_RI_RES_TYPE_SQ; res->u.sqrq.op = FW_RI_RES_OP_WRITE; /* eqsize is the number of 64B entries plus the status page size. */ - eqsize = wq->sq.size * T4_SQ_NUM_SLOTS + spg_ndesc; + eqsize = wq->sq.size * T4_SQ_NUM_SLOTS + + rdev->hw_queue.t4_eq_status_entries; res->u.sqrq.fetchszm_to_iqid = cpu_to_be32( V_FW_RI_RES_WR_HOSTFCMODE(0) | /* no host cidx updates */ V_FW_RI_RES_WR_CPRIO(0) | /* don't keep in chip cache */ V_FW_RI_RES_WR_PCIECHN(0) | /* set by uP at ri_init time */ V_FW_RI_RES_WR_IQID(scq->cqid)); res->u.sqrq.dcaen_to_eqsize = cpu_to_be32( V_FW_RI_RES_WR_DCAEN(0) | V_FW_RI_RES_WR_DCACPU(0) | V_FW_RI_RES_WR_FBMIN(2) | - V_FW_RI_RES_WR_FBMAX(2) | + V_FW_RI_RES_WR_FBMAX(3) | V_FW_RI_RES_WR_CIDXFTHRESHO(0) | V_FW_RI_RES_WR_CIDXFTHRESH(0) | V_FW_RI_RES_WR_EQSIZE(eqsize)); res->u.sqrq.eqid = cpu_to_be32(wq->sq.qid); res->u.sqrq.eqaddr = cpu_to_be64(wq->sq.dma_addr); res++; res->u.sqrq.restype = FW_RI_RES_TYPE_RQ; res->u.sqrq.op = FW_RI_RES_OP_WRITE; /* eqsize is the number of 64B entries plus the status page size. */ - eqsize = wq->rq.size * T4_RQ_NUM_SLOTS + spg_ndesc; + eqsize = wq->rq.size * T4_RQ_NUM_SLOTS + + rdev->hw_queue.t4_eq_status_entries; res->u.sqrq.fetchszm_to_iqid = cpu_to_be32( V_FW_RI_RES_WR_HOSTFCMODE(0) | /* no host cidx updates */ V_FW_RI_RES_WR_CPRIO(0) | /* don't keep in chip cache */ V_FW_RI_RES_WR_PCIECHN(0) | /* set by uP at ri_init time */ V_FW_RI_RES_WR_IQID(rcq->cqid)); res->u.sqrq.dcaen_to_eqsize = cpu_to_be32( V_FW_RI_RES_WR_DCAEN(0) | V_FW_RI_RES_WR_DCACPU(0) | V_FW_RI_RES_WR_FBMIN(2) | - V_FW_RI_RES_WR_FBMAX(2) | + V_FW_RI_RES_WR_FBMAX(3) | V_FW_RI_RES_WR_CIDXFTHRESHO(0) | V_FW_RI_RES_WR_CIDXFTHRESH(0) | V_FW_RI_RES_WR_EQSIZE(eqsize)); res->u.sqrq.eqid = cpu_to_be32(wq->rq.qid); res->u.sqrq.eqaddr = cpu_to_be64(wq->rq.dma_addr); c4iw_init_wr_wait(&wr_wait); t4_wrq_tx(sc, wr); - ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, wq->sq.qid, __func__); + ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, wq->sq.qid, + NULL, __func__); if (ret) - goto err7; + goto free_rq_dma; - CTR6(KTR_IW_CXGBE, + CTR5(KTR_IW_CXGBE, "%s sqid 0x%x rqid 0x%x kdb 0x%p squdb 0x%llx rqudb 0x%llx", - __func__, wq->sq.qid, wq->rq.qid, wq->db, - (unsigned long long)wq->sq.udb, (unsigned long long)wq->rq.udb); + __func__, wq->sq.qid, wq->rq.qid, + (unsigned long long)wq->sq.bar2_va, + (unsigned long long)wq->rq.bar2_va); return 0; -err7: - contigfree(wq->rq.queue, wq->rq.memsize, M_DEVBUF); -err6: - dealloc_sq(rdev, &wq->sq); -err5: +free_rq_dma: + dma_free_coherent(rhp->ibdev.dma_device, + wq->rq.memsize, wq->rq.queue, + dma_unmap_addr(&wq->rq, mapping)); +free_sq_dma: + dma_free_coherent(rhp->ibdev.dma_device, + wq->sq.memsize, wq->sq.queue, + dma_unmap_addr(&wq->sq, mapping)); +free_hwaddr: c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size); -err4: +free_sw_rq: kfree(wq->rq.sw_rq); -err3: +free_sw_sq: kfree(wq->sq.sw_sq); -err2: +free_rq_qid: c4iw_put_qpid(rdev, wq->rq.qid, uctx); -err1: +free_sq_qid: c4iw_put_qpid(rdev, wq->sq.qid, uctx); - return -ENOMEM; + return ret; } static int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp, struct ib_send_wr *wr, int max, u32 *plenp) { u8 *dstp, *srcp; u32 plen = 0; int i; int rem, len; dstp = (u8 *)immdp->data; for (i = 0; i < wr->num_sge; i++) { if ((plen + wr->sg_list[i].length) > max) return -EMSGSIZE; srcp = (u8 *)(unsigned long)wr->sg_list[i].addr; plen += wr->sg_list[i].length; rem = wr->sg_list[i].length; while (rem) { if (dstp == (u8 *)&sq->queue[sq->size]) dstp = (u8 *)sq->queue; if (rem <= (u8 *)&sq->queue[sq->size] - dstp) len = rem; else len = (u8 *)&sq->queue[sq->size] - dstp; memcpy(dstp, srcp, len); dstp += len; srcp += len; rem -= len; } } len = roundup(plen + sizeof *immdp, 16) - (plen + sizeof *immdp); if (len) memset(dstp, 0, len); immdp->op = FW_RI_DATA_IMMD; immdp->r1 = 0; immdp->r2 = 0; immdp->immdlen = cpu_to_be32(plen); *plenp = plen; return 0; } static int build_isgl(__be64 *queue_start, __be64 *queue_end, struct fw_ri_isgl *isglp, struct ib_sge *sg_list, int num_sge, u32 *plenp) { int i; u32 plen = 0; __be64 *flitp = (__be64 *)isglp->sge; for (i = 0; i < num_sge; i++) { if ((plen + sg_list[i].length) < plen) return -EMSGSIZE; plen += sg_list[i].length; *flitp = cpu_to_be64(((u64)sg_list[i].lkey << 32) | sg_list[i].length); if (++flitp == queue_end) flitp = queue_start; *flitp = cpu_to_be64(sg_list[i].addr); if (++flitp == queue_end) flitp = queue_start; } *flitp = (__force __be64)0; isglp->op = FW_RI_DATA_ISGL; isglp->r1 = 0; isglp->nsge = cpu_to_be16(num_sge); isglp->r2 = 0; if (plenp) *plenp = plen; return 0; } static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) { u32 plen; int size; int ret; if (wr->num_sge > T4_MAX_SEND_SGE) return -EINVAL; switch (wr->opcode) { case IB_WR_SEND: if (wr->send_flags & IB_SEND_SOLICITED) wqe->send.sendop_pkd = cpu_to_be32( V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_SE)); else wqe->send.sendop_pkd = cpu_to_be32( V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND)); wqe->send.stag_inv = 0; break; case IB_WR_SEND_WITH_INV: if (wr->send_flags & IB_SEND_SOLICITED) wqe->send.sendop_pkd = cpu_to_be32( V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_SE_INV)); else wqe->send.sendop_pkd = cpu_to_be32( V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_INV)); wqe->send.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey); break; default: return -EINVAL; } + wqe->send.r3 = 0; + wqe->send.r4 = 0; plen = 0; if (wr->num_sge) { if (wr->send_flags & IB_SEND_INLINE) { ret = build_immd(sq, wqe->send.u.immd_src, wr, T4_MAX_SEND_INLINE, &plen); if (ret) return ret; size = sizeof wqe->send + sizeof(struct fw_ri_immd) + plen; } else { ret = build_isgl((__be64 *)sq->queue, (__be64 *)&sq->queue[sq->size], wqe->send.u.isgl_src, wr->sg_list, wr->num_sge, &plen); if (ret) return ret; size = sizeof wqe->send + sizeof(struct fw_ri_isgl) + wr->num_sge * sizeof(struct fw_ri_sge); } } else { wqe->send.u.immd_src[0].op = FW_RI_DATA_IMMD; wqe->send.u.immd_src[0].r1 = 0; wqe->send.u.immd_src[0].r2 = 0; wqe->send.u.immd_src[0].immdlen = 0; size = sizeof wqe->send + sizeof(struct fw_ri_immd); plen = 0; } *len16 = DIV_ROUND_UP(size, 16); wqe->send.plen = cpu_to_be32(plen); return 0; } static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) { u32 plen; int size; int ret; if (wr->num_sge > T4_MAX_SEND_SGE) return -EINVAL; wqe->write.r2 = 0; - wqe->write.stag_sink = cpu_to_be32(wr->wr.rdma.rkey); - wqe->write.to_sink = cpu_to_be64(wr->wr.rdma.remote_addr); + wqe->write.stag_sink = cpu_to_be32(rdma_wr(wr)->rkey); + wqe->write.to_sink = cpu_to_be64(rdma_wr(wr)->remote_addr); if (wr->num_sge) { if (wr->send_flags & IB_SEND_INLINE) { ret = build_immd(sq, wqe->write.u.immd_src, wr, T4_MAX_WRITE_INLINE, &plen); if (ret) return ret; size = sizeof wqe->write + sizeof(struct fw_ri_immd) + plen; } else { ret = build_isgl((__be64 *)sq->queue, (__be64 *)&sq->queue[sq->size], wqe->write.u.isgl_src, wr->sg_list, wr->num_sge, &plen); if (ret) return ret; size = sizeof wqe->write + sizeof(struct fw_ri_isgl) + wr->num_sge * sizeof(struct fw_ri_sge); } } else { wqe->write.u.immd_src[0].op = FW_RI_DATA_IMMD; wqe->write.u.immd_src[0].r1 = 0; wqe->write.u.immd_src[0].r2 = 0; wqe->write.u.immd_src[0].immdlen = 0; size = sizeof wqe->write + sizeof(struct fw_ri_immd); plen = 0; } *len16 = DIV_ROUND_UP(size, 16); wqe->write.plen = cpu_to_be32(plen); return 0; } static int build_rdma_read(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) { if (wr->num_sge > 1) return -EINVAL; - if (wr->num_sge) { - wqe->read.stag_src = cpu_to_be32(wr->wr.rdma.rkey); - wqe->read.to_src_hi = cpu_to_be32((u32)(wr->wr.rdma.remote_addr + if (wr->num_sge && wr->sg_list[0].length) { + wqe->read.stag_src = cpu_to_be32(rdma_wr(wr)->rkey); + wqe->read.to_src_hi = cpu_to_be32((u32)(rdma_wr(wr)->remote_addr >> 32)); - wqe->read.to_src_lo = cpu_to_be32((u32)wr->wr.rdma.remote_addr); + wqe->read.to_src_lo = + cpu_to_be32((u32)rdma_wr(wr)->remote_addr); wqe->read.stag_sink = cpu_to_be32(wr->sg_list[0].lkey); wqe->read.plen = cpu_to_be32(wr->sg_list[0].length); wqe->read.to_sink_hi = cpu_to_be32((u32)(wr->sg_list[0].addr >> 32)); wqe->read.to_sink_lo = cpu_to_be32((u32)(wr->sg_list[0].addr)); } else { wqe->read.stag_src = cpu_to_be32(2); wqe->read.to_src_hi = 0; wqe->read.to_src_lo = 0; wqe->read.stag_sink = cpu_to_be32(2); wqe->read.plen = 0; wqe->read.to_sink_hi = 0; wqe->read.to_sink_lo = 0; } wqe->read.r2 = 0; wqe->read.r5 = 0; *len16 = DIV_ROUND_UP(sizeof wqe->read, 16); return 0; } static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe, struct ib_recv_wr *wr, u8 *len16) { int ret; ret = build_isgl((__be64 *)qhp->wq.rq.queue, (__be64 *)&qhp->wq.rq.queue[qhp->wq.rq.size], &wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL); if (ret) return ret; *len16 = DIV_ROUND_UP(sizeof wqe->recv + wr->num_sge * sizeof(struct fw_ri_sge), 16); return 0; } -static int build_fastreg(struct t4_sq *sq, union t4_wr *wqe, - struct ib_send_wr *wr, u8 *len16) -{ - - struct fw_ri_immd *imdp; - __be64 *p; - int i; - int pbllen = roundup(wr->wr.fast_reg.page_list_len * sizeof(u64), 32); - int rem; - - if (wr->wr.fast_reg.page_list_len > T4_MAX_FR_DEPTH) - return -EINVAL; - - wqe->fr.qpbinde_to_dcacpu = 0; - wqe->fr.pgsz_shift = wr->wr.fast_reg.page_shift - 12; - wqe->fr.addr_type = FW_RI_VA_BASED_TO; - wqe->fr.mem_perms = c4iw_ib_to_tpt_access(wr->wr.fast_reg.access_flags); - wqe->fr.len_hi = 0; - wqe->fr.len_lo = cpu_to_be32(wr->wr.fast_reg.length); - wqe->fr.stag = cpu_to_be32(wr->wr.fast_reg.rkey); - wqe->fr.va_hi = cpu_to_be32(wr->wr.fast_reg.iova_start >> 32); - wqe->fr.va_lo_fbo = cpu_to_be32(wr->wr.fast_reg.iova_start & - 0xffffffff); - WARN_ON(pbllen > T4_MAX_FR_IMMD); - imdp = (struct fw_ri_immd *)(&wqe->fr + 1); - imdp->op = FW_RI_DATA_IMMD; - imdp->r1 = 0; - imdp->r2 = 0; - imdp->immdlen = cpu_to_be32(pbllen); - p = (__be64 *)(imdp + 1); - rem = pbllen; - for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) { - *p = cpu_to_be64((u64)wr->wr.fast_reg.page_list->page_list[i]); - rem -= sizeof *p; - if (++p == (__be64 *)&sq->queue[sq->size]) - p = (__be64 *)sq->queue; - } - BUG_ON(rem < 0); - while (rem) { - *p = 0; - rem -= sizeof *p; - if (++p == (__be64 *)&sq->queue[sq->size]) - p = (__be64 *)sq->queue; - } - *len16 = DIV_ROUND_UP(sizeof wqe->fr + sizeof *imdp + pbllen, 16); - return 0; -} - static int build_inv_stag(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) { wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey); wqe->inv.r2 = 0; *len16 = DIV_ROUND_UP(sizeof wqe->inv, 16); return 0; } +static void free_qp_work(struct work_struct *work) +{ + struct c4iw_ucontext *ucontext; + struct c4iw_qp *qhp; + struct c4iw_dev *rhp; + + qhp = container_of(work, struct c4iw_qp, free_work); + ucontext = qhp->ucontext; + rhp = qhp->rhp; + + CTR3(KTR_IW_CXGBE, "%s qhp %p ucontext %p\n", __func__, + qhp, ucontext); + destroy_qp(&rhp->rdev, &qhp->wq, + ucontext ? &ucontext->uctx : &rhp->rdev.uctx); + + if (ucontext) + c4iw_put_ucontext(ucontext); + kfree(qhp); +} + +static void queue_qp_free(struct kref *kref) +{ + struct c4iw_qp *qhp; + + qhp = container_of(kref, struct c4iw_qp, kref); + CTR2(KTR_IW_CXGBE, "%s qhp %p", __func__, qhp); + queue_work(qhp->rhp->rdev.free_workq, &qhp->free_work); +} + void c4iw_qp_add_ref(struct ib_qp *qp) { CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, qp); - atomic_inc(&(to_c4iw_qp(qp)->refcnt)); + kref_get(&to_c4iw_qp(qp)->kref); } void c4iw_qp_rem_ref(struct ib_qp *qp) { CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, qp); - if (atomic_dec_and_test(&(to_c4iw_qp(qp)->refcnt))) - wake_up(&(to_c4iw_qp(qp)->wait)); + kref_put(&to_c4iw_qp(qp)->kref, queue_qp_free); } static void complete_sq_drain_wr(struct c4iw_qp *qhp, struct ib_send_wr *wr) { struct t4_cqe cqe = {}; struct c4iw_cq *schp; unsigned long flag; struct t4_cq *cq; schp = to_c4iw_cq(qhp->ibqp.send_cq); cq = &schp->cq; PDBG("%s drain sq id %u\n", __func__, qhp->wq.sq.qid); cqe.u.drain_cookie = wr->wr_id; cqe.header = cpu_to_be32(V_CQE_STATUS(T4_ERR_SWFLUSH) | V_CQE_OPCODE(C4IW_DRAIN_OPCODE) | V_CQE_TYPE(1) | V_CQE_SWCQE(1) | V_CQE_QPID(qhp->wq.sq.qid)); spin_lock_irqsave(&schp->lock, flag); cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen)); cq->sw_queue[cq->sw_pidx] = cqe; t4_swcq_produce(cq); spin_unlock_irqrestore(&schp->lock, flag); spin_lock_irqsave(&schp->comp_handler_lock, flag); (*schp->ibcq.comp_handler)(&schp->ibcq, schp->ibcq.cq_context); spin_unlock_irqrestore(&schp->comp_handler_lock, flag); } static void complete_rq_drain_wr(struct c4iw_qp *qhp, struct ib_recv_wr *wr) { struct t4_cqe cqe = {}; struct c4iw_cq *rchp; unsigned long flag; struct t4_cq *cq; rchp = to_c4iw_cq(qhp->ibqp.recv_cq); cq = &rchp->cq; PDBG("%s drain rq id %u\n", __func__, qhp->wq.sq.qid); cqe.u.drain_cookie = wr->wr_id; cqe.header = cpu_to_be32(V_CQE_STATUS(T4_ERR_SWFLUSH) | V_CQE_OPCODE(C4IW_DRAIN_OPCODE) | V_CQE_TYPE(0) | V_CQE_SWCQE(1) | V_CQE_QPID(qhp->wq.sq.qid)); spin_lock_irqsave(&rchp->lock, flag); cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen)); cq->sw_queue[cq->sw_pidx] = cqe; t4_swcq_produce(cq); spin_unlock_irqrestore(&rchp->lock, flag); spin_lock_irqsave(&rchp->comp_handler_lock, flag); (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); spin_unlock_irqrestore(&rchp->comp_handler_lock, flag); } +static void build_tpte_memreg(struct fw_ri_fr_nsmr_tpte_wr *fr, + struct ib_reg_wr *wr, struct c4iw_mr *mhp, u8 *len16) +{ + __be64 *p = (__be64 *)fr->pbl; + + fr->r2 = cpu_to_be32(0); + fr->stag = cpu_to_be32(mhp->ibmr.rkey); + + fr->tpte.valid_to_pdid = cpu_to_be32(F_FW_RI_TPTE_VALID | + V_FW_RI_TPTE_STAGKEY((mhp->ibmr.rkey & M_FW_RI_TPTE_STAGKEY)) | + V_FW_RI_TPTE_STAGSTATE(1) | + V_FW_RI_TPTE_STAGTYPE(FW_RI_STAG_NSMR) | + V_FW_RI_TPTE_PDID(mhp->attr.pdid)); + fr->tpte.locread_to_qpid = cpu_to_be32( + V_FW_RI_TPTE_PERM(c4iw_ib_to_tpt_access(wr->access)) | + V_FW_RI_TPTE_ADDRTYPE(FW_RI_VA_BASED_TO) | + V_FW_RI_TPTE_PS(ilog2(wr->mr->page_size) - 12)); + fr->tpte.nosnoop_pbladdr = cpu_to_be32(V_FW_RI_TPTE_PBLADDR( + PBL_OFF(&mhp->rhp->rdev, mhp->attr.pbl_addr)>>3)); + fr->tpte.dca_mwbcnt_pstag = cpu_to_be32(0); + fr->tpte.len_hi = cpu_to_be32(0); + fr->tpte.len_lo = cpu_to_be32(mhp->ibmr.length); + fr->tpte.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32); + fr->tpte.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova & 0xffffffff); + + p[0] = cpu_to_be64((u64)mhp->mpl[0]); + p[1] = cpu_to_be64((u64)mhp->mpl[1]); + + *len16 = DIV_ROUND_UP(sizeof(*fr), 16); +} + +static int build_memreg(struct t4_sq *sq, union t4_wr *wqe, + struct ib_reg_wr *wr, struct c4iw_mr *mhp, u8 *len16, + bool dsgl_supported) +{ + struct fw_ri_immd *imdp; + __be64 *p; + int i; + int pbllen = roundup(mhp->mpl_len * sizeof(u64), 32); + int rem; + + if (mhp->mpl_len > t4_max_fr_depth(use_dsgl && dsgl_supported)) + return -EINVAL; + + wqe->fr.qpbinde_to_dcacpu = 0; + wqe->fr.pgsz_shift = ilog2(wr->mr->page_size) - 12; + wqe->fr.addr_type = FW_RI_VA_BASED_TO; + wqe->fr.mem_perms = c4iw_ib_to_tpt_access(wr->access); + wqe->fr.len_hi = 0; + wqe->fr.len_lo = cpu_to_be32(mhp->ibmr.length); + wqe->fr.stag = cpu_to_be32(wr->key); + wqe->fr.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32); + wqe->fr.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova & + 0xffffffff); + + if (dsgl_supported && use_dsgl && (pbllen > max_fr_immd)) { + struct fw_ri_dsgl *sglp; + + for (i = 0; i < mhp->mpl_len; i++) + mhp->mpl[i] = + (__force u64)cpu_to_be64((u64)mhp->mpl[i]); + + sglp = (struct fw_ri_dsgl *)(&wqe->fr + 1); + sglp->op = FW_RI_DATA_DSGL; + sglp->r1 = 0; + sglp->nsge = cpu_to_be16(1); + sglp->addr0 = cpu_to_be64(mhp->mpl_addr); + sglp->len0 = cpu_to_be32(pbllen); + + *len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*sglp), 16); + } else { + imdp = (struct fw_ri_immd *)(&wqe->fr + 1); + imdp->op = FW_RI_DATA_IMMD; + imdp->r1 = 0; + imdp->r2 = 0; + imdp->immdlen = cpu_to_be32(pbllen); + p = (__be64 *)(imdp + 1); + rem = pbllen; + for (i = 0; i < mhp->mpl_len; i++) { + *p = cpu_to_be64((u64)mhp->mpl[i]); + rem -= sizeof(*p); + if (++p == (__be64 *)&sq->queue[sq->size]) + p = (__be64 *)sq->queue; + } + BUG_ON(rem < 0); + while (rem) { + *p = 0; + rem -= sizeof(*p); + if (++p == (__be64 *)&sq->queue[sq->size]) + p = (__be64 *)sq->queue; + } + *len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*imdp) + + pbllen, 16); + } + + return 0; +} + int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, struct ib_send_wr **bad_wr) { int err = 0; u8 len16 = 0; enum fw_wr_opcodes fw_opcode = 0; enum fw_ri_wr_flags fw_flags; struct c4iw_qp *qhp; - union t4_wr *wqe; + union t4_wr *wqe = NULL; u32 num_wrs; struct t4_swsqe *swsqe; unsigned long flag; u16 idx = 0; + struct c4iw_rdev *rdev; qhp = to_c4iw_qp(ibqp); + rdev = &qhp->rhp->rdev; spin_lock_irqsave(&qhp->lock, flag); if (t4_wq_in_error(&qhp->wq)) { spin_unlock_irqrestore(&qhp->lock, flag); complete_sq_drain_wr(qhp, wr); return err; } num_wrs = t4_sq_avail(&qhp->wq); if (num_wrs == 0) { spin_unlock_irqrestore(&qhp->lock, flag); + *bad_wr = wr; return -ENOMEM; } while (wr) { if (num_wrs == 0) { err = -ENOMEM; *bad_wr = wr; break; } wqe = (union t4_wr *)((u8 *)qhp->wq.sq.queue + qhp->wq.sq.wq_pidx * T4_EQ_ENTRY_SIZE); fw_flags = 0; if (wr->send_flags & IB_SEND_SOLICITED) fw_flags |= FW_RI_SOLICITED_EVENT_FLAG; if (wr->send_flags & IB_SEND_SIGNALED || qhp->sq_sig_all) fw_flags |= FW_RI_COMPLETION_FLAG; swsqe = &qhp->wq.sq.sw_sq[qhp->wq.sq.pidx]; switch (wr->opcode) { case IB_WR_SEND_WITH_INV: case IB_WR_SEND: if (wr->send_flags & IB_SEND_FENCE) fw_flags |= FW_RI_READ_FENCE_FLAG; fw_opcode = FW_RI_SEND_WR; if (wr->opcode == IB_WR_SEND) swsqe->opcode = FW_RI_SEND; else swsqe->opcode = FW_RI_SEND_WITH_INV; err = build_rdma_send(&qhp->wq.sq, wqe, wr, &len16); break; case IB_WR_RDMA_WRITE: fw_opcode = FW_RI_RDMA_WRITE_WR; swsqe->opcode = FW_RI_RDMA_WRITE; err = build_rdma_write(&qhp->wq.sq, wqe, wr, &len16); break; case IB_WR_RDMA_READ: case IB_WR_RDMA_READ_WITH_INV: fw_opcode = FW_RI_RDMA_READ_WR; swsqe->opcode = FW_RI_READ_REQ; - if (wr->opcode == IB_WR_RDMA_READ_WITH_INV) + if (wr->opcode == IB_WR_RDMA_READ_WITH_INV) { + c4iw_invalidate_mr(qhp->rhp, + wr->sg_list[0].lkey); fw_flags = FW_RI_RDMA_READ_INVALIDATE; - else + } else { fw_flags = 0; + } err = build_rdma_read(wqe, wr, &len16); if (err) break; swsqe->read_len = wr->sg_list[0].length; if (!qhp->wq.sq.oldest_read) qhp->wq.sq.oldest_read = swsqe; break; - case IB_WR_FAST_REG_MR: - fw_opcode = FW_RI_FR_NSMR_WR; + case IB_WR_REG_MR: { + struct c4iw_mr *mhp = to_c4iw_mr(reg_wr(wr)->mr); + swsqe->opcode = FW_RI_FAST_REGISTER; - err = build_fastreg(&qhp->wq.sq, wqe, wr, &len16); + if (rdev->adap->params.fr_nsmr_tpte_wr_support && + !mhp->attr.state && mhp->mpl_len <= 2) { + fw_opcode = FW_RI_FR_NSMR_TPTE_WR; + build_tpte_memreg(&wqe->fr_tpte, reg_wr(wr), + mhp, &len16); + } else { + fw_opcode = FW_RI_FR_NSMR_WR; + err = build_memreg(&qhp->wq.sq, wqe, reg_wr(wr), + mhp, &len16, + rdev->adap->params.ulptx_memwrite_dsgl); + if (err) + break; + } + mhp->attr.state = 1; break; + } case IB_WR_LOCAL_INV: if (wr->send_flags & IB_SEND_FENCE) fw_flags |= FW_RI_LOCAL_FENCE_FLAG; fw_opcode = FW_RI_INV_LSTAG_WR; swsqe->opcode = FW_RI_LOCAL_INV; err = build_inv_stag(wqe, wr, &len16); + c4iw_invalidate_mr(qhp->rhp, wr->ex.invalidate_rkey); break; default: CTR2(KTR_IW_CXGBE, "%s post of type =%d TBD!", __func__, wr->opcode); err = -EINVAL; } if (err) { *bad_wr = wr; break; } swsqe->idx = qhp->wq.sq.pidx; swsqe->complete = 0; swsqe->signaled = (wr->send_flags & IB_SEND_SIGNALED) || qhp->sq_sig_all; + swsqe->flushed = 0; swsqe->wr_id = wr->wr_id; init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16); CTR5(KTR_IW_CXGBE, "%s cookie 0x%llx pidx 0x%x opcode 0x%x read_len %u", __func__, (unsigned long long)wr->wr_id, qhp->wq.sq.pidx, swsqe->opcode, swsqe->read_len); wr = wr->next; num_wrs--; t4_sq_produce(&qhp->wq, len16); idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); } - t4_ring_sq_db(&qhp->wq, idx); + t4_ring_sq_db(&qhp->wq, idx, wqe, rdev->adap->iwt.wc_en); spin_unlock_irqrestore(&qhp->lock, flag); return err; } int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, struct ib_recv_wr **bad_wr) { int err = 0; struct c4iw_qp *qhp; - union t4_recv_wr *wqe; + union t4_recv_wr *wqe = NULL; u32 num_wrs; u8 len16 = 0; unsigned long flag; u16 idx = 0; qhp = to_c4iw_qp(ibqp); spin_lock_irqsave(&qhp->lock, flag); if (t4_wq_in_error(&qhp->wq)) { spin_unlock_irqrestore(&qhp->lock, flag); complete_rq_drain_wr(qhp, wr); return err; } num_wrs = t4_rq_avail(&qhp->wq); if (num_wrs == 0) { spin_unlock_irqrestore(&qhp->lock, flag); + *bad_wr = wr; return -ENOMEM; } while (wr) { if (wr->num_sge > T4_MAX_RECV_SGE) { err = -EINVAL; *bad_wr = wr; break; } wqe = (union t4_recv_wr *)((u8 *)qhp->wq.rq.queue + qhp->wq.rq.wq_pidx * T4_EQ_ENTRY_SIZE); if (num_wrs) err = build_rdma_recv(qhp, wqe, wr, &len16); else err = -ENOMEM; if (err) { *bad_wr = wr; break; } qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].wr_id = wr->wr_id; wqe->recv.opcode = FW_RI_RECV_WR; wqe->recv.r1 = 0; wqe->recv.wrid = qhp->wq.rq.pidx; wqe->recv.r2[0] = 0; wqe->recv.r2[1] = 0; wqe->recv.r2[2] = 0; wqe->recv.len16 = len16; CTR3(KTR_IW_CXGBE, "%s cookie 0x%llx pidx %u", __func__, (unsigned long long) wr->wr_id, qhp->wq.rq.pidx); t4_rq_produce(&qhp->wq, len16); idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); wr = wr->next; num_wrs--; } - t4_ring_rq_db(&qhp->wq, idx); + t4_ring_rq_db(&qhp->wq, idx, wqe, qhp->rhp->rdev.adap->iwt.wc_en); spin_unlock_irqrestore(&qhp->lock, flag); return err; } -int c4iw_bind_mw(struct ib_qp *qp, struct ib_mw *mw, struct ib_mw_bind *mw_bind) -{ - return -ENOSYS; -} - static inline void build_term_codes(struct t4_cqe *err_cqe, u8 *layer_type, u8 *ecode) { int status; int tagged; int opcode; int rqtype; int send_inv; if (!err_cqe) { *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; *ecode = 0; return; } status = CQE_STATUS(err_cqe); opcode = CQE_OPCODE(err_cqe); rqtype = RQ_TYPE(err_cqe); send_inv = (opcode == FW_RI_SEND_WITH_INV) || (opcode == FW_RI_SEND_WITH_SE_INV); tagged = (opcode == FW_RI_RDMA_WRITE) || (rqtype && (opcode == FW_RI_READ_RESP)); switch (status) { case T4_ERR_STAG: if (send_inv) { *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; *ecode = RDMAP_CANT_INV_STAG; } else { *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; *ecode = RDMAP_INV_STAG; } break; case T4_ERR_PDID: *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; if ((opcode == FW_RI_SEND_WITH_INV) || (opcode == FW_RI_SEND_WITH_SE_INV)) *ecode = RDMAP_CANT_INV_STAG; else *ecode = RDMAP_STAG_NOT_ASSOC; break; case T4_ERR_QPID: *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; *ecode = RDMAP_STAG_NOT_ASSOC; break; case T4_ERR_ACCESS: *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; *ecode = RDMAP_ACC_VIOL; break; case T4_ERR_WRAP: *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; *ecode = RDMAP_TO_WRAP; break; case T4_ERR_BOUND: if (tagged) { *layer_type = LAYER_DDP|DDP_TAGGED_ERR; *ecode = DDPT_BASE_BOUNDS; } else { *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; *ecode = RDMAP_BASE_BOUNDS; } break; case T4_ERR_INVALIDATE_SHARED_MR: case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND: *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; *ecode = RDMAP_CANT_INV_STAG; break; case T4_ERR_ECC: case T4_ERR_ECC_PSTAG: case T4_ERR_INTERNAL_ERR: *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA; *ecode = 0; break; case T4_ERR_OUT_OF_RQE: *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; *ecode = DDPU_INV_MSN_NOBUF; break; case T4_ERR_PBL_ADDR_BOUND: *layer_type = LAYER_DDP|DDP_TAGGED_ERR; *ecode = DDPT_BASE_BOUNDS; break; case T4_ERR_CRC: *layer_type = LAYER_MPA|DDP_LLP; *ecode = MPA_CRC_ERR; break; case T4_ERR_MARKER: *layer_type = LAYER_MPA|DDP_LLP; *ecode = MPA_MARKER_ERR; break; case T4_ERR_PDU_LEN_ERR: *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; *ecode = DDPU_MSG_TOOBIG; break; case T4_ERR_DDP_VERSION: if (tagged) { *layer_type = LAYER_DDP|DDP_TAGGED_ERR; *ecode = DDPT_INV_VERS; } else { *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; *ecode = DDPU_INV_VERS; } break; case T4_ERR_RDMA_VERSION: *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; *ecode = RDMAP_INV_VERS; break; case T4_ERR_OPCODE: *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; *ecode = RDMAP_INV_OPCODE; break; case T4_ERR_DDP_QUEUE_NUM: *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; *ecode = DDPU_INV_QN; break; case T4_ERR_MSN: case T4_ERR_MSN_GAP: case T4_ERR_MSN_RANGE: case T4_ERR_IRD_OVERFLOW: *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; *ecode = DDPU_INV_MSN_RANGE; break; case T4_ERR_TBIT: *layer_type = LAYER_DDP|DDP_LOCAL_CATA; *ecode = 0; break; case T4_ERR_MO: *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; *ecode = DDPU_INV_MO; break; default: *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; *ecode = 0; break; } } static void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe, gfp_t gfp) { int ret; struct fw_ri_wr *wqe; struct terminate_message *term; struct wrqe *wr; struct socket *so = qhp->ep->com.so; struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp = intotcpcb(inp); struct toepcb *toep = tp->t_toe; CTR4(KTR_IW_CXGBE, "%s qhp %p qid 0x%x tid %u", __func__, qhp, qhp->wq.sq.qid, qhp->ep->hwtid); wr = alloc_wrqe(sizeof(*wqe), toep->ofld_txq); if (wr == NULL) return; wqe = wrtod(wr); memset(wqe, 0, sizeof *wqe); wqe->op_compl = cpu_to_be32(V_FW_WR_OP(FW_RI_WR)); wqe->flowid_len16 = cpu_to_be32( V_FW_WR_FLOWID(qhp->ep->hwtid) | V_FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16))); wqe->u.terminate.type = FW_RI_TYPE_TERMINATE; wqe->u.terminate.immdlen = cpu_to_be32(sizeof *term); term = (struct terminate_message *)wqe->u.terminate.termmsg; if (qhp->attr.layer_etype == (LAYER_MPA|DDP_LLP)) { term->layer_etype = qhp->attr.layer_etype; term->ecode = qhp->attr.ecode; } else build_term_codes(err_cqe, &term->layer_etype, &term->ecode); ret = creds(toep, inp, sizeof(*wqe)); if (ret) { free_wrqe(wr); return; } t4_wrq_tx(qhp->rhp->rdev.adap, wr); } /* Assumes qhp lock is held. */ static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp, struct c4iw_cq *schp) { int count; - int flushed; + int rq_flushed, sq_flushed; unsigned long flag; CTR4(KTR_IW_CXGBE, "%s qhp %p rchp %p schp %p", __func__, qhp, rchp, schp); /* locking hierarchy: cq lock first, then qp lock. */ spin_lock_irqsave(&rchp->lock, flag); spin_lock(&qhp->lock); - c4iw_flush_hw_cq(&rchp->cq); + + if (qhp->wq.flushed) { + spin_unlock(&qhp->lock); + spin_unlock_irqrestore(&rchp->lock, flag); + return; + } + qhp->wq.flushed = 1; + + c4iw_flush_hw_cq(rchp); c4iw_count_rcqes(&rchp->cq, &qhp->wq, &count); - flushed = c4iw_flush_rq(&qhp->wq, &rchp->cq, count); + rq_flushed = c4iw_flush_rq(&qhp->wq, &rchp->cq, count); spin_unlock(&qhp->lock); spin_unlock_irqrestore(&rchp->lock, flag); - if (flushed && rchp->ibcq.comp_handler) { - spin_lock_irqsave(&rchp->comp_handler_lock, flag); - (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); - spin_unlock_irqrestore(&rchp->comp_handler_lock, flag); - } /* locking hierarchy: cq lock first, then qp lock. */ spin_lock_irqsave(&schp->lock, flag); spin_lock(&qhp->lock); - c4iw_flush_hw_cq(&schp->cq); - c4iw_count_scqes(&schp->cq, &qhp->wq, &count); - flushed = c4iw_flush_sq(&qhp->wq, &schp->cq, count); + if (schp != rchp) + c4iw_flush_hw_cq(schp); + sq_flushed = c4iw_flush_sq(qhp); spin_unlock(&qhp->lock); spin_unlock_irqrestore(&schp->lock, flag); - if (flushed && schp->ibcq.comp_handler) { - spin_lock_irqsave(&schp->comp_handler_lock, flag); - (*schp->ibcq.comp_handler)(&schp->ibcq, schp->ibcq.cq_context); - spin_unlock_irqrestore(&schp->comp_handler_lock, flag); + + if (schp == rchp) { + if (t4_clear_cq_armed(&rchp->cq) && + (rq_flushed || sq_flushed)) { + spin_lock_irqsave(&rchp->comp_handler_lock, flag); + (*rchp->ibcq.comp_handler)(&rchp->ibcq, + rchp->ibcq.cq_context); + spin_unlock_irqrestore(&rchp->comp_handler_lock, flag); + } + } else { + if (t4_clear_cq_armed(&rchp->cq) && rq_flushed) { + spin_lock_irqsave(&rchp->comp_handler_lock, flag); + (*rchp->ibcq.comp_handler)(&rchp->ibcq, + rchp->ibcq.cq_context); + spin_unlock_irqrestore(&rchp->comp_handler_lock, flag); + } + if (t4_clear_cq_armed(&schp->cq) && sq_flushed) { + spin_lock_irqsave(&schp->comp_handler_lock, flag); + (*schp->ibcq.comp_handler)(&schp->ibcq, + schp->ibcq.cq_context); + spin_unlock_irqrestore(&schp->comp_handler_lock, flag); + } } } static void flush_qp(struct c4iw_qp *qhp) { struct c4iw_cq *rchp, *schp; unsigned long flag; - rchp = get_chp(qhp->rhp, qhp->attr.rcq); - schp = get_chp(qhp->rhp, qhp->attr.scq); + rchp = to_c4iw_cq(qhp->ibqp.recv_cq); + schp = to_c4iw_cq(qhp->ibqp.send_cq); + t4_set_wq_in_error(&qhp->wq); if (qhp->ibqp.uobject) { - t4_set_wq_in_error(&qhp->wq); t4_set_cq_in_error(&rchp->cq); spin_lock_irqsave(&rchp->comp_handler_lock, flag); (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); spin_unlock_irqrestore(&rchp->comp_handler_lock, flag); if (schp != rchp) { t4_set_cq_in_error(&schp->cq); spin_lock_irqsave(&schp->comp_handler_lock, flag); (*schp->ibcq.comp_handler)(&schp->ibcq, schp->ibcq.cq_context); spin_unlock_irqrestore(&schp->comp_handler_lock, flag); } return; } __flush_qp(qhp, rchp, schp); } static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp, struct c4iw_ep *ep) { struct c4iw_rdev *rdev = &rhp->rdev; struct adapter *sc = rdev->adap; struct fw_ri_wr *wqe; int ret; struct wrqe *wr; struct socket *so = ep->com.so; struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp = intotcpcb(inp); struct toepcb *toep = tp->t_toe; KASSERT(rhp == qhp->rhp && ep == qhp->ep, ("%s: EDOOFUS", __func__)); - CTR4(KTR_IW_CXGBE, "%s qhp %p qid 0x%x tid %u", __func__, qhp, - qhp->wq.sq.qid, ep->hwtid); + CTR5(KTR_IW_CXGBE, "%s qhp %p qid 0x%x ep %p tid %u", __func__, qhp, + qhp->wq.sq.qid, ep, ep->hwtid); wr = alloc_wrqe(sizeof(*wqe), toep->ofld_txq); if (wr == NULL) return (0); wqe = wrtod(wr); memset(wqe, 0, sizeof *wqe); wqe->op_compl = cpu_to_be32(V_FW_WR_OP(FW_RI_WR) | F_FW_WR_COMPL); wqe->flowid_len16 = cpu_to_be32(V_FW_WR_FLOWID(ep->hwtid) | V_FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16))); wqe->cookie = (unsigned long) &ep->com.wr_wait; wqe->u.fini.type = FW_RI_TYPE_FINI; c4iw_init_wr_wait(&ep->com.wr_wait); ret = creds(toep, inp, sizeof(*wqe)); if (ret) { free_wrqe(wr); return ret; } t4_wrq_tx(sc, wr); ret = c4iw_wait_for_reply(rdev, &ep->com.wr_wait, ep->hwtid, - qhp->wq.sq.qid, __func__); + qhp->wq.sq.qid, ep->com.so, __func__); return ret; } static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init) { CTR2(KTR_IW_CXGBE, "%s p2p_type = %d", __func__, p2p_type); memset(&init->u, 0, sizeof init->u); switch (p2p_type) { case FW_RI_INIT_P2PTYPE_RDMA_WRITE: init->u.write.opcode = FW_RI_RDMA_WRITE_WR; init->u.write.stag_sink = cpu_to_be32(1); init->u.write.to_sink = cpu_to_be64(1); init->u.write.u.immd_src[0].op = FW_RI_DATA_IMMD; init->u.write.len16 = DIV_ROUND_UP(sizeof init->u.write + sizeof(struct fw_ri_immd), 16); break; case FW_RI_INIT_P2PTYPE_READ_REQ: init->u.write.opcode = FW_RI_RDMA_READ_WR; init->u.read.stag_src = cpu_to_be32(1); init->u.read.to_src_lo = cpu_to_be32(1); init->u.read.stag_sink = cpu_to_be32(1); init->u.read.to_sink_lo = cpu_to_be32(1); init->u.read.len16 = DIV_ROUND_UP(sizeof init->u.read, 16); break; } } static int creds(struct toepcb *toep, struct inpcb *inp, size_t wrsize) { struct ofld_tx_sdesc *txsd; CTR3(KTR_IW_CXGBE, "%s:creB %p %u", __func__, toep , wrsize); INP_WLOCK(inp); if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) != 0) { INP_WUNLOCK(inp); return (EINVAL); } txsd = &toep->txsd[toep->txsd_pidx]; txsd->tx_credits = howmany(wrsize, 16); txsd->plen = 0; KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, ("%s: not enough credits (%d)", __func__, toep->tx_credits)); toep->tx_credits -= txsd->tx_credits; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) toep->txsd_pidx = 0; toep->txsd_avail--; INP_WUNLOCK(inp); CTR5(KTR_IW_CXGBE, "%s:creE %p %u %u %u", __func__, toep , txsd->tx_credits, toep->tx_credits, toep->txsd_pidx); return (0); } static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp) { struct fw_ri_wr *wqe; int ret; struct wrqe *wr; struct c4iw_ep *ep = qhp->ep; struct c4iw_rdev *rdev = &qhp->rhp->rdev; struct adapter *sc = rdev->adap; struct socket *so = ep->com.so; struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp = intotcpcb(inp); struct toepcb *toep = tp->t_toe; - CTR4(KTR_IW_CXGBE, "%s qhp %p qid 0x%x tid %u", __func__, qhp, - qhp->wq.sq.qid, ep->hwtid); + CTR5(KTR_IW_CXGBE, "%s qhp %p qid 0x%x ep %p tid %u", __func__, qhp, + qhp->wq.sq.qid, ep, ep->hwtid); wr = alloc_wrqe(sizeof(*wqe), toep->ofld_txq); if (wr == NULL) return (0); wqe = wrtod(wr); + ret = alloc_ird(rhp, qhp->attr.max_ird); + if (ret) { + qhp->attr.max_ird = 0; + free_wrqe(wr); + return ret; + } memset(wqe, 0, sizeof *wqe); wqe->op_compl = cpu_to_be32( V_FW_WR_OP(FW_RI_WR) | F_FW_WR_COMPL); wqe->flowid_len16 = cpu_to_be32(V_FW_WR_FLOWID(ep->hwtid) | V_FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16))); wqe->cookie = (unsigned long) &ep->com.wr_wait; wqe->u.init.type = FW_RI_TYPE_INIT; wqe->u.init.mpareqbit_p2ptype = V_FW_RI_WR_MPAREQBIT(qhp->attr.mpa_attr.initiator) | V_FW_RI_WR_P2PTYPE(qhp->attr.mpa_attr.p2p_type); wqe->u.init.mpa_attrs = FW_RI_MPA_IETF_ENABLE; if (qhp->attr.mpa_attr.recv_marker_enabled) wqe->u.init.mpa_attrs |= FW_RI_MPA_RX_MARKER_ENABLE; if (qhp->attr.mpa_attr.xmit_marker_enabled) wqe->u.init.mpa_attrs |= FW_RI_MPA_TX_MARKER_ENABLE; if (qhp->attr.mpa_attr.crc_enabled) wqe->u.init.mpa_attrs |= FW_RI_MPA_CRC_ENABLE; wqe->u.init.qp_caps = FW_RI_QP_RDMA_READ_ENABLE | FW_RI_QP_RDMA_WRITE_ENABLE | FW_RI_QP_BIND_ENABLE; if (!qhp->ibqp.uobject) wqe->u.init.qp_caps |= FW_RI_QP_FAST_REGISTER_ENABLE | FW_RI_QP_STAG0_ENABLE; wqe->u.init.nrqe = cpu_to_be16(t4_rqes_posted(&qhp->wq)); wqe->u.init.pdid = cpu_to_be32(qhp->attr.pd); wqe->u.init.qpid = cpu_to_be32(qhp->wq.sq.qid); wqe->u.init.sq_eqid = cpu_to_be32(qhp->wq.sq.qid); wqe->u.init.rq_eqid = cpu_to_be32(qhp->wq.rq.qid); wqe->u.init.scqid = cpu_to_be32(qhp->attr.scq); wqe->u.init.rcqid = cpu_to_be32(qhp->attr.rcq); wqe->u.init.ord_max = cpu_to_be32(qhp->attr.max_ord); wqe->u.init.ird_max = cpu_to_be32(qhp->attr.max_ird); wqe->u.init.iss = cpu_to_be32(ep->snd_seq); wqe->u.init.irs = cpu_to_be32(ep->rcv_seq); wqe->u.init.hwrqsize = cpu_to_be32(qhp->wq.rq.rqt_size); wqe->u.init.hwrqaddr = cpu_to_be32(qhp->wq.rq.rqt_hwaddr - sc->vres.rq.start); if (qhp->attr.mpa_attr.initiator) build_rtr_msg(qhp->attr.mpa_attr.p2p_type, &wqe->u.init); c4iw_init_wr_wait(&ep->com.wr_wait); ret = creds(toep, inp, sizeof(*wqe)); if (ret) { free_wrqe(wr); + free_ird(rhp, qhp->attr.max_ird); return ret; } t4_wrq_tx(sc, wr); ret = c4iw_wait_for_reply(rdev, &ep->com.wr_wait, ep->hwtid, - qhp->wq.sq.qid, __func__); + qhp->wq.sq.qid, ep->com.so, __func__); toep->ulp_mode = ULP_MODE_RDMA; + free_ird(rhp, qhp->attr.max_ird); return ret; } int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp, enum c4iw_qp_attr_mask mask, struct c4iw_qp_attributes *attrs, int internal) { int ret = 0; struct c4iw_qp_attributes newattr = qhp->attr; int disconnect = 0; int terminate = 0; int abort = 0; int free = 0; struct c4iw_ep *ep = NULL; CTR5(KTR_IW_CXGBE, "%s qhp %p sqid 0x%x rqid 0x%x ep %p", __func__, qhp, qhp->wq.sq.qid, qhp->wq.rq.qid, qhp->ep); CTR3(KTR_IW_CXGBE, "%s state %d -> %d", __func__, qhp->attr.state, (mask & C4IW_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1); mutex_lock(&qhp->mutex); /* Process attr changes if in IDLE */ if (mask & C4IW_QP_ATTR_VALID_MODIFY) { if (qhp->attr.state != C4IW_QP_STATE_IDLE) { ret = -EIO; goto out; } if (mask & C4IW_QP_ATTR_ENABLE_RDMA_READ) newattr.enable_rdma_read = attrs->enable_rdma_read; if (mask & C4IW_QP_ATTR_ENABLE_RDMA_WRITE) newattr.enable_rdma_write = attrs->enable_rdma_write; if (mask & C4IW_QP_ATTR_ENABLE_RDMA_BIND) newattr.enable_bind = attrs->enable_bind; if (mask & C4IW_QP_ATTR_MAX_ORD) { if (attrs->max_ord > c4iw_max_read_depth) { ret = -EINVAL; goto out; } newattr.max_ord = attrs->max_ord; } if (mask & C4IW_QP_ATTR_MAX_IRD) { - if (attrs->max_ird > c4iw_max_read_depth) { + if (attrs->max_ird > cur_max_read_depth(rhp)) { ret = -EINVAL; goto out; } newattr.max_ird = attrs->max_ird; } qhp->attr = newattr; } if (!(mask & C4IW_QP_ATTR_NEXT_STATE)) goto out; if (qhp->attr.state == attrs->next_state) goto out; switch (qhp->attr.state) { case C4IW_QP_STATE_IDLE: switch (attrs->next_state) { case C4IW_QP_STATE_RTS: if (!(mask & C4IW_QP_ATTR_LLP_STREAM_HANDLE)) { ret = -EINVAL; goto out; } if (!(mask & C4IW_QP_ATTR_MPA_ATTR)) { ret = -EINVAL; goto out; } qhp->attr.mpa_attr = attrs->mpa_attr; qhp->attr.llp_stream_handle = attrs->llp_stream_handle; qhp->ep = qhp->attr.llp_stream_handle; set_state(qhp, C4IW_QP_STATE_RTS); /* * Ref the endpoint here and deref when we * disassociate the endpoint from the QP. This * happens in CLOSING->IDLE transition or *->ERROR * transition. */ c4iw_get_ep(&qhp->ep->com); ret = rdma_init(rhp, qhp); if (ret) goto err; break; case C4IW_QP_STATE_ERROR: set_state(qhp, C4IW_QP_STATE_ERROR); flush_qp(qhp); break; default: ret = -EINVAL; goto out; } break; case C4IW_QP_STATE_RTS: switch (attrs->next_state) { case C4IW_QP_STATE_CLOSING: BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); + t4_set_wq_in_error(&qhp->wq); set_state(qhp, C4IW_QP_STATE_CLOSING); ep = qhp->ep; if (!internal) { abort = 0; disconnect = 1; c4iw_get_ep(&qhp->ep->com); } - if (qhp->ibqp.uobject) - t4_set_wq_in_error(&qhp->wq); ret = rdma_fini(rhp, qhp, ep); if (ret) goto err; break; case C4IW_QP_STATE_TERMINATE: + t4_set_wq_in_error(&qhp->wq); set_state(qhp, C4IW_QP_STATE_TERMINATE); qhp->attr.layer_etype = attrs->layer_etype; qhp->attr.ecode = attrs->ecode; - if (qhp->ibqp.uobject) - t4_set_wq_in_error(&qhp->wq); ep = qhp->ep; - if (!internal) + if (!internal) { + c4iw_get_ep(&qhp->ep->com); terminate = 1; - disconnect = 1; - c4iw_get_ep(&qhp->ep->com); + disconnect = 1; + } else { + terminate = qhp->attr.send_term; + ret = rdma_fini(rhp, qhp, ep); + if (ret) + goto err; + } break; case C4IW_QP_STATE_ERROR: + t4_set_wq_in_error(&qhp->wq); set_state(qhp, C4IW_QP_STATE_ERROR); - if (qhp->ibqp.uobject) - t4_set_wq_in_error(&qhp->wq); if (!internal) { abort = 1; disconnect = 1; ep = qhp->ep; c4iw_get_ep(&qhp->ep->com); } goto err; break; default: ret = -EINVAL; goto out; } break; case C4IW_QP_STATE_CLOSING: /* * Allow kernel users to move to ERROR for qp draining. */ if (!internal && (qhp->ibqp.uobject || attrs->next_state != C4IW_QP_STATE_ERROR)) { ret = -EINVAL; goto out; } switch (attrs->next_state) { case C4IW_QP_STATE_IDLE: flush_qp(qhp); set_state(qhp, C4IW_QP_STATE_IDLE); qhp->attr.llp_stream_handle = NULL; c4iw_put_ep(&qhp->ep->com); qhp->ep = NULL; wake_up(&qhp->wait); break; case C4IW_QP_STATE_ERROR: goto err; default: ret = -EINVAL; goto err; } break; case C4IW_QP_STATE_ERROR: if (attrs->next_state != C4IW_QP_STATE_IDLE) { ret = -EINVAL; goto out; } if (!t4_sq_empty(&qhp->wq) || !t4_rq_empty(&qhp->wq)) { ret = -EINVAL; goto out; } set_state(qhp, C4IW_QP_STATE_IDLE); break; case C4IW_QP_STATE_TERMINATE: if (!internal) { ret = -EINVAL; goto out; } goto err; break; default: printf("%s in a bad state %d\n", __func__, qhp->attr.state); ret = -EINVAL; goto err; break; } goto out; err: CTR3(KTR_IW_CXGBE, "%s disassociating ep %p qpid 0x%x", __func__, qhp->ep, qhp->wq.sq.qid); /* disassociate the LLP connection */ qhp->attr.llp_stream_handle = NULL; if (!ep) ep = qhp->ep; qhp->ep = NULL; set_state(qhp, C4IW_QP_STATE_ERROR); free = 1; abort = 1; BUG_ON(!ep); flush_qp(qhp); wake_up(&qhp->wait); out: mutex_unlock(&qhp->mutex); if (terminate) post_terminate(qhp, NULL, internal ? GFP_ATOMIC : GFP_KERNEL); /* * If disconnect is 1, then we need to initiate a disconnect * on the EP. This can be a normal close (RTS->CLOSING) or * an abnormal close (RTS/CLOSING->ERROR). */ if (disconnect) { - c4iw_ep_disconnect(ep, abort, internal ? GFP_ATOMIC : + __c4iw_ep_disconnect(ep, abort, internal ? GFP_ATOMIC : GFP_KERNEL); c4iw_put_ep(&ep->com); } /* * If free is 1, then we've disassociated the EP from the QP * and we need to dereference the EP. */ if (free) c4iw_put_ep(&ep->com); CTR2(KTR_IW_CXGBE, "%s exit state %d", __func__, qhp->attr.state); return ret; } int c4iw_destroy_qp(struct ib_qp *ib_qp) { struct c4iw_dev *rhp; struct c4iw_qp *qhp; struct c4iw_qp_attributes attrs; - struct c4iw_ucontext *ucontext; CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, ib_qp); qhp = to_c4iw_qp(ib_qp); rhp = qhp->rhp; attrs.next_state = C4IW_QP_STATE_ERROR; if (qhp->attr.state == C4IW_QP_STATE_TERMINATE) c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); else c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); wait_event(qhp->wait, !qhp->ep); - spin_lock_irq(&rhp->lock); - remove_handle_nolock(rhp, &rhp->qpidr, qhp->wq.sq.qid); - spin_unlock_irq(&rhp->lock); - atomic_dec(&qhp->refcnt); - wait_event(qhp->wait, !atomic_read(&qhp->refcnt)); + remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid); - ucontext = ib_qp->uobject ? - to_c4iw_ucontext(ib_qp->uobject->context) : NULL; - destroy_qp(&rhp->rdev, &qhp->wq, - ucontext ? &ucontext->uctx : &rhp->rdev.uctx); + free_ird(rhp, qhp->attr.max_ird); + c4iw_qp_rem_ref(ib_qp); CTR3(KTR_IW_CXGBE, "%s ib_qp %p qpid 0x%0x", __func__, ib_qp, qhp->wq.sq.qid); - kfree(qhp); return 0; } struct ib_qp * c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, struct ib_udata *udata) { struct c4iw_dev *rhp; struct c4iw_qp *qhp; struct c4iw_pd *php; struct c4iw_cq *schp; struct c4iw_cq *rchp; struct c4iw_create_qp_resp uresp; - int sqsize, rqsize; + unsigned int sqsize, rqsize; struct c4iw_ucontext *ucontext; - int ret, spg_ndesc; - struct c4iw_mm_entry *mm1, *mm2, *mm3, *mm4; + int ret; + struct c4iw_mm_entry *sq_key_mm = NULL, *rq_key_mm = NULL; + struct c4iw_mm_entry *sq_db_key_mm = NULL, *rq_db_key_mm = NULL; CTR2(KTR_IW_CXGBE, "%s ib_pd %p", __func__, pd); if (attrs->qp_type != IB_QPT_RC) return ERR_PTR(-EINVAL); php = to_c4iw_pd(pd); rhp = php->rhp; schp = get_chp(rhp, ((struct c4iw_cq *)attrs->send_cq)->cq.cqid); rchp = get_chp(rhp, ((struct c4iw_cq *)attrs->recv_cq)->cq.cqid); if (!schp || !rchp) return ERR_PTR(-EINVAL); if (attrs->cap.max_inline_data > T4_MAX_SEND_INLINE) return ERR_PTR(-EINVAL); - spg_ndesc = rhp->rdev.adap->params.sge.spg_len / EQ_ESIZE; - rqsize = roundup(attrs->cap.max_recv_wr + 1, 16); - if (rqsize > T4_MAX_RQ_SIZE(spg_ndesc)) + if (attrs->cap.max_recv_wr > rhp->rdev.hw_queue.t4_max_rq_size) return ERR_PTR(-E2BIG); + rqsize = attrs->cap.max_recv_wr + 1; + if (rqsize < 8) + rqsize = 8; - sqsize = roundup(attrs->cap.max_send_wr + 1, 16); - if (sqsize > T4_MAX_SQ_SIZE(spg_ndesc)) + if (attrs->cap.max_send_wr > rhp->rdev.hw_queue.t4_max_sq_size) return ERR_PTR(-E2BIG); + sqsize = attrs->cap.max_send_wr + 1; + if (sqsize < 8) + sqsize = 8; ucontext = pd->uobject ? to_c4iw_ucontext(pd->uobject->context) : NULL; - qhp = kzalloc(sizeof(*qhp), GFP_KERNEL); if (!qhp) return ERR_PTR(-ENOMEM); qhp->wq.sq.size = sqsize; - qhp->wq.sq.memsize = (sqsize + spg_ndesc) * sizeof *qhp->wq.sq.queue + - 16 * sizeof(__be64); + qhp->wq.sq.memsize = + (sqsize + rhp->rdev.hw_queue.t4_eq_status_entries) * + sizeof(*qhp->wq.sq.queue) + 16 * sizeof(__be64); + qhp->wq.sq.flush_cidx = -1; qhp->wq.rq.size = rqsize; - qhp->wq.rq.memsize = (rqsize + spg_ndesc) * sizeof *qhp->wq.rq.queue; + qhp->wq.rq.memsize = + (rqsize + rhp->rdev.hw_queue.t4_eq_status_entries) * + sizeof(*qhp->wq.rq.queue); if (ucontext) { qhp->wq.sq.memsize = roundup(qhp->wq.sq.memsize, PAGE_SIZE); qhp->wq.rq.memsize = roundup(qhp->wq.rq.memsize, PAGE_SIZE); } CTR5(KTR_IW_CXGBE, "%s sqsize %u sqmemsize %zu rqsize %u rqmemsize %zu", __func__, sqsize, qhp->wq.sq.memsize, rqsize, qhp->wq.rq.memsize); ret = create_qp(&rhp->rdev, &qhp->wq, &schp->cq, &rchp->cq, ucontext ? &ucontext->uctx : &rhp->rdev.uctx); if (ret) goto err1; attrs->cap.max_recv_wr = rqsize - 1; attrs->cap.max_send_wr = sqsize - 1; attrs->cap.max_inline_data = T4_MAX_SEND_INLINE; qhp->rhp = rhp; qhp->attr.pd = php->pdid; qhp->attr.scq = ((struct c4iw_cq *) attrs->send_cq)->cq.cqid; qhp->attr.rcq = ((struct c4iw_cq *) attrs->recv_cq)->cq.cqid; qhp->attr.sq_num_entries = attrs->cap.max_send_wr; qhp->attr.rq_num_entries = attrs->cap.max_recv_wr; qhp->attr.sq_max_sges = attrs->cap.max_send_sge; qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge; qhp->attr.rq_max_sges = attrs->cap.max_recv_sge; qhp->attr.state = C4IW_QP_STATE_IDLE; qhp->attr.next_state = C4IW_QP_STATE_IDLE; qhp->attr.enable_rdma_read = 1; qhp->attr.enable_rdma_write = 1; qhp->attr.enable_bind = 1; - qhp->attr.max_ord = 1; - qhp->attr.max_ird = 1; + qhp->attr.max_ord = 0; + qhp->attr.max_ird = 0; qhp->sq_sig_all = attrs->sq_sig_type == IB_SIGNAL_ALL_WR; spin_lock_init(&qhp->lock); mutex_init(&qhp->mutex); init_waitqueue_head(&qhp->wait); - atomic_set(&qhp->refcnt, 1); + kref_init(&qhp->kref); + INIT_WORK(&qhp->free_work, free_qp_work); - spin_lock_irq(&rhp->lock); - ret = insert_handle_nolock(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid); - spin_unlock_irq(&rhp->lock); + ret = insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid); if (ret) goto err2; if (udata) { - mm1 = kmalloc(sizeof *mm1, GFP_KERNEL); - if (!mm1) { + sq_key_mm = kmalloc(sizeof(*sq_key_mm), GFP_KERNEL); + if (!sq_key_mm) { ret = -ENOMEM; goto err3; } - mm2 = kmalloc(sizeof *mm2, GFP_KERNEL); - if (!mm2) { + rq_key_mm = kmalloc(sizeof(*rq_key_mm), GFP_KERNEL); + if (!rq_key_mm) { ret = -ENOMEM; goto err4; } - mm3 = kmalloc(sizeof *mm3, GFP_KERNEL); - if (!mm3) { + sq_db_key_mm = kmalloc(sizeof(*sq_db_key_mm), GFP_KERNEL); + if (!sq_db_key_mm) { ret = -ENOMEM; goto err5; } - mm4 = kmalloc(sizeof *mm4, GFP_KERNEL); - if (!mm4) { + rq_db_key_mm = kmalloc(sizeof(*rq_db_key_mm), GFP_KERNEL); + if (!rq_db_key_mm) { ret = -ENOMEM; goto err6; } uresp.flags = 0; uresp.qid_mask = rhp->rdev.qpmask; uresp.sqid = qhp->wq.sq.qid; uresp.sq_size = qhp->wq.sq.size; uresp.sq_memsize = qhp->wq.sq.memsize; uresp.rqid = qhp->wq.rq.qid; uresp.rq_size = qhp->wq.rq.size; uresp.rq_memsize = qhp->wq.rq.memsize; spin_lock(&ucontext->mmap_lock); + uresp.ma_sync_key = 0; uresp.sq_key = ucontext->key; ucontext->key += PAGE_SIZE; uresp.rq_key = ucontext->key; ucontext->key += PAGE_SIZE; uresp.sq_db_gts_key = ucontext->key; ucontext->key += PAGE_SIZE; uresp.rq_db_gts_key = ucontext->key; ucontext->key += PAGE_SIZE; spin_unlock(&ucontext->mmap_lock); ret = ib_copy_to_udata(udata, &uresp, sizeof uresp); if (ret) goto err7; - mm1->key = uresp.sq_key; - mm1->addr = qhp->wq.sq.phys_addr; - mm1->len = PAGE_ALIGN(qhp->wq.sq.memsize); - CTR4(KTR_IW_CXGBE, "%s mm1 %x, %x, %d", __func__, mm1->key, - mm1->addr, mm1->len); - insert_mmap(ucontext, mm1); - mm2->key = uresp.rq_key; - mm2->addr = vtophys(qhp->wq.rq.queue); - mm2->len = PAGE_ALIGN(qhp->wq.rq.memsize); - CTR4(KTR_IW_CXGBE, "%s mm2 %x, %x, %d", __func__, mm2->key, - mm2->addr, mm2->len); - insert_mmap(ucontext, mm2); - mm3->key = uresp.sq_db_gts_key; - mm3->addr = qhp->wq.sq.udb; - mm3->len = PAGE_SIZE; - CTR4(KTR_IW_CXGBE, "%s mm3 %x, %x, %d", __func__, mm3->key, - mm3->addr, mm3->len); - insert_mmap(ucontext, mm3); - mm4->key = uresp.rq_db_gts_key; - mm4->addr = qhp->wq.rq.udb; - mm4->len = PAGE_SIZE; - CTR4(KTR_IW_CXGBE, "%s mm4 %x, %x, %d", __func__, mm4->key, - mm4->addr, mm4->len); - insert_mmap(ucontext, mm4); + sq_key_mm->key = uresp.sq_key; + sq_key_mm->addr = qhp->wq.sq.phys_addr; + sq_key_mm->len = PAGE_ALIGN(qhp->wq.sq.memsize); + CTR4(KTR_IW_CXGBE, "%s sq_key_mm %x, %x, %d", __func__, + sq_key_mm->key, sq_key_mm->addr, + sq_key_mm->len); + insert_mmap(ucontext, sq_key_mm); + rq_key_mm->key = uresp.rq_key; + rq_key_mm->addr = qhp->wq.rq.phys_addr; + rq_key_mm->len = PAGE_ALIGN(qhp->wq.rq.memsize); + CTR4(KTR_IW_CXGBE, "%s rq_key_mm %x, %x, %d", __func__, + rq_key_mm->key, rq_key_mm->addr, + rq_key_mm->len); + insert_mmap(ucontext, rq_key_mm); + sq_db_key_mm->key = uresp.sq_db_gts_key; + sq_db_key_mm->addr = (u64)qhp->wq.sq.bar2_pa; + sq_db_key_mm->len = PAGE_SIZE; + CTR4(KTR_IW_CXGBE, "%s sq_db_key_mm %x, %x, %d", __func__, + sq_db_key_mm->key, sq_db_key_mm->addr, + sq_db_key_mm->len); + insert_mmap(ucontext, sq_db_key_mm); + rq_db_key_mm->key = uresp.rq_db_gts_key; + rq_db_key_mm->addr = (u64)qhp->wq.rq.bar2_pa; + rq_db_key_mm->len = PAGE_SIZE; + CTR4(KTR_IW_CXGBE, "%s rq_db_key_mm %x, %x, %d", __func__, + rq_db_key_mm->key, rq_db_key_mm->addr, + rq_db_key_mm->len); + insert_mmap(ucontext, rq_db_key_mm); + + c4iw_get_ucontext(ucontext); + qhp->ucontext = ucontext; } qhp->ibqp.qp_num = qhp->wq.sq.qid; init_timer(&(qhp->timer)); - CTR5(KTR_IW_CXGBE, - "%s qhp %p sq_num_entries %d, rq_num_entries %d qpid 0x%0x", - __func__, qhp, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries, - qhp->wq.sq.qid); + + CTR5(KTR_IW_CXGBE, "%s sq id %u size %u memsize %zu num_entries %u\n", + __func__, qhp->wq.sq.qid, + qhp->wq.sq.size, qhp->wq.sq.memsize, attrs->cap.max_send_wr); + CTR5(KTR_IW_CXGBE, "%s rq id %u size %u memsize %zu num_entries %u\n", + __func__, qhp->wq.rq.qid, + qhp->wq.rq.size, qhp->wq.rq.memsize, attrs->cap.max_recv_wr); return &qhp->ibqp; err7: - kfree(mm4); + kfree(rq_db_key_mm); err6: - kfree(mm3); + kfree(sq_db_key_mm); err5: - kfree(mm2); + kfree(rq_key_mm); err4: - kfree(mm1); + kfree(sq_key_mm); err3: remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid); err2: destroy_qp(&rhp->rdev, &qhp->wq, ucontext ? &ucontext->uctx : &rhp->rdev.uctx); err1: kfree(qhp); return ERR_PTR(ret); } int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { struct c4iw_dev *rhp; struct c4iw_qp *qhp; enum c4iw_qp_attr_mask mask = 0; struct c4iw_qp_attributes attrs; CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, ibqp); /* iwarp does not support the RTR state */ if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR)) attr_mask &= ~IB_QP_STATE; /* Make sure we still have something left to do */ if (!attr_mask) return 0; memset(&attrs, 0, sizeof attrs); qhp = to_c4iw_qp(ibqp); rhp = qhp->rhp; attrs.next_state = c4iw_convert_state(attr->qp_state); attrs.enable_rdma_read = (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) ? 1 : 0; attrs.enable_rdma_write = (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0; mask |= (attr_mask & IB_QP_STATE) ? C4IW_QP_ATTR_NEXT_STATE : 0; mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ? (C4IW_QP_ATTR_ENABLE_RDMA_READ | C4IW_QP_ATTR_ENABLE_RDMA_WRITE | C4IW_QP_ATTR_ENABLE_RDMA_BIND) : 0; return c4iw_modify_qp(rhp, qhp, mask, &attrs, 0); } struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn) { CTR3(KTR_IW_CXGBE, "%s ib_dev %p qpn 0x%x", __func__, dev, qpn); return (struct ib_qp *)get_qhp(to_c4iw_dev(dev), qpn); } int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_qp_init_attr *init_attr) { struct c4iw_qp *qhp = to_c4iw_qp(ibqp); memset(attr, 0, sizeof *attr); memset(init_attr, 0, sizeof *init_attr); attr->qp_state = to_ib_qp_state(qhp->attr.state); init_attr->cap.max_send_wr = qhp->attr.sq_num_entries; init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries; init_attr->cap.max_send_sge = qhp->attr.sq_max_sges; init_attr->cap.max_recv_sge = qhp->attr.sq_max_sges; init_attr->cap.max_inline_data = T4_MAX_SEND_INLINE; init_attr->sq_sig_type = qhp->sq_sig_all ? IB_SIGNAL_ALL_WR : 0; return 0; } #endif Index: projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/t4.h =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/t4.h (revision 331317) +++ projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/t4.h (revision 331318) @@ -1,579 +1,720 @@ /* * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * * $FreeBSD$ */ #ifndef __T4_H__ #define __T4_H__ +#include "common/t4_regs_values.h" +#include "common/t4_regs.h" /* * Fixme: Adding missing defines */ #define SGE_PF_KDOORBELL 0x0 #define QID_MASK 0xffff8000U #define QID_SHIFT 15 #define QID(x) ((x) << QID_SHIFT) #define DBPRIO 0x00004000U #define PIDX_MASK 0x00003fffU #define PIDX_SHIFT 0 #define PIDX(x) ((x) << PIDX_SHIFT) #define SGE_PF_GTS 0x4 #define INGRESSQID_MASK 0xffff0000U #define INGRESSQID_SHIFT 16 #define INGRESSQID(x) ((x) << INGRESSQID_SHIFT) #define TIMERREG_MASK 0x0000e000U #define TIMERREG_SHIFT 13 #define TIMERREG(x) ((x) << TIMERREG_SHIFT) #define SEINTARM_MASK 0x00001000U #define SEINTARM_SHIFT 12 #define SEINTARM(x) ((x) << SEINTARM_SHIFT) #define CIDXINC_MASK 0x00000fffU #define CIDXINC_SHIFT 0 #define CIDXINC(x) ((x) << CIDXINC_SHIFT) #define T4_MAX_NUM_PD 65536 -#define T4_MAX_EQ_SIZE 65520 -#define T4_MAX_IQ_SIZE 65520 -#define T4_MAX_RQ_SIZE(n) (8192 - (n) - 1) -#define T4_MAX_SQ_SIZE(n) (T4_MAX_EQ_SIZE - (n) - 1) -#define T4_MAX_QP_DEPTH(n) (T4_MAX_RQ_SIZE(n)) -#define T4_MAX_CQ_DEPTH (T4_MAX_IQ_SIZE - 2) -#define T4_MAX_MR_SIZE (~0ULL - 1) -#define T4_PAGESIZE_MASK 0xffffffff000 /* 4KB-8TB */ +#define T4_MAX_MR_SIZE (~0ULL) +#define T4_PAGESIZE_MASK 0xffffffff000 /* 4KB-8TB */ #define T4_STAG_UNSET 0xffffffff #define T4_FW_MAJ 0 #define A_PCIE_MA_SYNC 0x30b4 struct t4_status_page { __be32 rsvd1; /* flit 0 - hw owns */ __be16 rsvd2; __be16 qid; __be16 cidx; __be16 pidx; u8 qp_err; /* flit 1 - sw owns */ u8 db_off; u8 pad; u16 host_wq_pidx; u16 host_cidx; u16 host_pidx; }; #define T4_EQ_ENTRY_SIZE 64 #define T4_SQ_NUM_SLOTS 5 #define T4_SQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_SQ_NUM_SLOTS) #define T4_MAX_SEND_SGE ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_send_wr) - \ sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge)) #define T4_MAX_SEND_INLINE ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_send_wr) - \ sizeof(struct fw_ri_immd))) #define T4_MAX_WRITE_INLINE ((T4_SQ_NUM_BYTES - \ sizeof(struct fw_ri_rdma_write_wr) - \ sizeof(struct fw_ri_immd))) #define T4_MAX_WRITE_SGE ((T4_SQ_NUM_BYTES - \ sizeof(struct fw_ri_rdma_write_wr) - \ sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge)) #define T4_MAX_FR_IMMD ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_fr_nsmr_wr) - \ sizeof(struct fw_ri_immd)) & ~31UL) -#define T4_MAX_FR_DEPTH (T4_MAX_FR_IMMD / sizeof(u64)) +#define T4_MAX_FR_IMMD_DEPTH (T4_MAX_FR_IMMD / sizeof(u64)) +#define T4_MAX_FR_DSGL 1024 +#define T4_MAX_FR_DSGL_DEPTH (T4_MAX_FR_DSGL / sizeof(u64)) +static inline int t4_max_fr_depth(int use_dsgl) +{ + return use_dsgl ? T4_MAX_FR_DSGL_DEPTH : T4_MAX_FR_IMMD_DEPTH; +} + #define T4_RQ_NUM_SLOTS 2 #define T4_RQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_RQ_NUM_SLOTS) #define T4_MAX_RECV_SGE 4 union t4_wr { struct fw_ri_res_wr res; struct fw_ri_wr ri; struct fw_ri_rdma_write_wr write; struct fw_ri_send_wr send; struct fw_ri_rdma_read_wr read; struct fw_ri_bind_mw_wr bind; struct fw_ri_fr_nsmr_wr fr; + struct fw_ri_fr_nsmr_tpte_wr fr_tpte; struct fw_ri_inv_lstag_wr inv; struct t4_status_page status; __be64 flits[T4_EQ_ENTRY_SIZE / sizeof(__be64) * T4_SQ_NUM_SLOTS]; }; union t4_recv_wr { struct fw_ri_recv_wr recv; struct t4_status_page status; __be64 flits[T4_EQ_ENTRY_SIZE / sizeof(__be64) * T4_RQ_NUM_SLOTS]; }; static inline void init_wr_hdr(union t4_wr *wqe, u16 wrid, enum fw_wr_opcodes opcode, u8 flags, u8 len16) { wqe->send.opcode = (u8)opcode; wqe->send.flags = flags; wqe->send.wrid = wrid; wqe->send.r1[0] = 0; wqe->send.r1[1] = 0; wqe->send.r1[2] = 0; wqe->send.len16 = len16; } /* CQE/AE status codes */ #define T4_ERR_SUCCESS 0x0 #define T4_ERR_STAG 0x1 /* STAG invalid: either the */ /* STAG is offlimt, being 0, */ /* or STAG_key mismatch */ #define T4_ERR_PDID 0x2 /* PDID mismatch */ #define T4_ERR_QPID 0x3 /* QPID mismatch */ #define T4_ERR_ACCESS 0x4 /* Invalid access right */ #define T4_ERR_WRAP 0x5 /* Wrap error */ #define T4_ERR_BOUND 0x6 /* base and bounds voilation */ #define T4_ERR_INVALIDATE_SHARED_MR 0x7 /* attempt to invalidate a */ /* shared memory region */ #define T4_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8 /* attempt to invalidate a */ /* shared memory region */ #define T4_ERR_ECC 0x9 /* ECC error detected */ #define T4_ERR_ECC_PSTAG 0xA /* ECC error detected when */ /* reading PSTAG for a MW */ /* Invalidate */ #define T4_ERR_PBL_ADDR_BOUND 0xB /* pbl addr out of bounds: */ /* software error */ #define T4_ERR_SWFLUSH 0xC /* SW FLUSHED */ #define T4_ERR_CRC 0x10 /* CRC error */ #define T4_ERR_MARKER 0x11 /* Marker error */ #define T4_ERR_PDU_LEN_ERR 0x12 /* invalid PDU length */ #define T4_ERR_OUT_OF_RQE 0x13 /* out of RQE */ #define T4_ERR_DDP_VERSION 0x14 /* wrong DDP version */ #define T4_ERR_RDMA_VERSION 0x15 /* wrong RDMA version */ #define T4_ERR_OPCODE 0x16 /* invalid rdma opcode */ #define T4_ERR_DDP_QUEUE_NUM 0x17 /* invalid ddp queue number */ #define T4_ERR_MSN 0x18 /* MSN error */ #define T4_ERR_TBIT 0x19 /* tag bit not set correctly */ #define T4_ERR_MO 0x1A /* MO not 0 for TERMINATE */ /* or READ_REQ */ #define T4_ERR_MSN_GAP 0x1B #define T4_ERR_MSN_RANGE 0x1C #define T4_ERR_IRD_OVERFLOW 0x1D #define T4_ERR_RQE_ADDR_BOUND 0x1E /* RQE addr out of bounds: */ /* software error */ #define T4_ERR_INTERNAL_ERR 0x1F /* internal error (opcode */ /* mismatch) */ /* * CQE defs */ struct t4_cqe { __be32 header; __be32 len; union { struct { __be32 stag; __be32 msn; } rcqe; struct { - u32 nada1; + u32 stag; u16 nada2; u16 cidx; } scqe; struct { __be32 wrid_hi; __be32 wrid_low; } gen; u64 drain_cookie; } u; __be64 reserved; __be64 bits_type_ts; }; /* macros for flit 0 of the cqe */ #define S_CQE_QPID 12 #define M_CQE_QPID 0xFFFFF #define G_CQE_QPID(x) ((((x) >> S_CQE_QPID)) & M_CQE_QPID) #define V_CQE_QPID(x) ((x)<> S_CQE_SWCQE)) & M_CQE_SWCQE) #define V_CQE_SWCQE(x) ((x)<> S_CQE_STATUS)) & M_CQE_STATUS) #define V_CQE_STATUS(x) ((x)<> S_CQE_TYPE)) & M_CQE_TYPE) #define V_CQE_TYPE(x) ((x)<> S_CQE_OPCODE)) & M_CQE_OPCODE) #define V_CQE_OPCODE(x) ((x)<header))) #define CQE_QPID(x) (G_CQE_QPID(be32_to_cpu((x)->header))) #define CQE_TYPE(x) (G_CQE_TYPE(be32_to_cpu((x)->header))) #define SQ_TYPE(x) (CQE_TYPE((x))) #define RQ_TYPE(x) (!CQE_TYPE((x))) #define CQE_STATUS(x) (G_CQE_STATUS(be32_to_cpu((x)->header))) #define CQE_OPCODE(x) (G_CQE_OPCODE(be32_to_cpu((x)->header))) #define CQE_SEND_OPCODE(x)(\ (G_CQE_OPCODE(be32_to_cpu((x)->header)) == FW_RI_SEND) || \ (G_CQE_OPCODE(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_SE) || \ (G_CQE_OPCODE(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_INV) || \ (G_CQE_OPCODE(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_SE_INV)) #define CQE_LEN(x) (be32_to_cpu((x)->len)) /* used for RQ completion processing */ #define CQE_WRID_STAG(x) (be32_to_cpu((x)->u.rcqe.stag)) #define CQE_WRID_MSN(x) (be32_to_cpu((x)->u.rcqe.msn)) /* used for SQ completion processing */ #define CQE_WRID_SQ_IDX(x) ((x)->u.scqe.cidx) +#define CQE_WRID_FR_STAG(x) (be32_to_cpu((x)->u.scqe.stag)) /* generic accessor macros */ #define CQE_WRID_HI(x) ((x)->u.gen.wrid_hi) #define CQE_WRID_LOW(x) ((x)->u.gen.wrid_low) #define CQE_DRAIN_COOKIE(x) (x)->u.drain_cookie; /* macros for flit 3 of the cqe */ #define S_CQE_GENBIT 63 #define M_CQE_GENBIT 0x1 #define G_CQE_GENBIT(x) (((x) >> S_CQE_GENBIT) & M_CQE_GENBIT) #define V_CQE_GENBIT(x) ((x)<> S_CQE_OVFBIT)) & M_CQE_OVFBIT) #define S_CQE_IQTYPE 60 #define M_CQE_IQTYPE 0x3 #define G_CQE_IQTYPE(x) ((((x) >> S_CQE_IQTYPE)) & M_CQE_IQTYPE) #define M_CQE_TS 0x0fffffffffffffffULL #define G_CQE_TS(x) ((x) & M_CQE_TS) #define CQE_OVFBIT(x) ((unsigned)G_CQE_OVFBIT(be64_to_cpu((x)->bits_type_ts))) #define CQE_GENBIT(x) ((unsigned)G_CQE_GENBIT(be64_to_cpu((x)->bits_type_ts))) #define CQE_TS(x) (G_CQE_TS(be64_to_cpu((x)->bits_type_ts))) struct t4_swsqe { u64 wr_id; struct t4_cqe cqe; int read_len; int opcode; int complete; int signaled; u16 idx; + int flushed; + struct timespec host_ts; + u64 sge_ts; }; +static inline pgprot_t t4_pgprot_wc(pgprot_t prot) +{ +#if defined(__i386__) || defined(__x86_64__) || defined(CONFIG_PPC64) + return pgprot_writecombine(prot); +#else + return pgprot_noncached(prot); +#endif +} + +enum { + T4_SQ_ONCHIP = (1<<0), +}; + struct t4_sq { union t4_wr *queue; bus_addr_t dma_addr; - DECLARE_PCI_UNMAP_ADDR(mapping); + DEFINE_DMA_UNMAP_ADDR(mapping); unsigned long phys_addr; struct t4_swsqe *sw_sq; struct t4_swsqe *oldest_read; - u64 udb; + void __iomem *bar2_va; + u64 bar2_pa; size_t memsize; + u32 bar2_qid; u32 qid; u16 in_use; u16 size; u16 cidx; u16 pidx; u16 wq_pidx; + u16 wq_pidx_inc; u16 flags; + short flush_cidx; }; struct t4_swrqe { u64 wr_id; }; struct t4_rq { union t4_recv_wr *queue; bus_addr_t dma_addr; - DECLARE_PCI_UNMAP_ADDR(mapping); + DEFINE_DMA_UNMAP_ADDR(mapping); + unsigned long phys_addr; struct t4_swrqe *sw_rq; - u64 udb; + void __iomem *bar2_va; + u64 bar2_pa; size_t memsize; + u32 bar2_qid; u32 qid; u32 msn; u32 rqt_hwaddr; u16 rqt_size; u16 in_use; u16 size; u16 cidx; u16 pidx; u16 wq_pidx; + u16 wq_pidx_inc; }; struct t4_wq { struct t4_sq sq; struct t4_rq rq; - void __iomem *db; - void __iomem *gts; struct c4iw_rdev *rdev; + int flushed; }; static inline int t4_rqes_posted(struct t4_wq *wq) { return wq->rq.in_use; } static inline int t4_rq_empty(struct t4_wq *wq) { return wq->rq.in_use == 0; } static inline int t4_rq_full(struct t4_wq *wq) { return wq->rq.in_use == (wq->rq.size - 1); } static inline u32 t4_rq_avail(struct t4_wq *wq) { return wq->rq.size - 1 - wq->rq.in_use; } static inline void t4_rq_produce(struct t4_wq *wq, u8 len16) { wq->rq.in_use++; if (++wq->rq.pidx == wq->rq.size) wq->rq.pidx = 0; wq->rq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); if (wq->rq.wq_pidx >= wq->rq.size * T4_RQ_NUM_SLOTS) wq->rq.wq_pidx %= wq->rq.size * T4_RQ_NUM_SLOTS; } static inline void t4_rq_consume(struct t4_wq *wq) { wq->rq.in_use--; wq->rq.msn++; if (++wq->rq.cidx == wq->rq.size) wq->rq.cidx = 0; } static inline u16 t4_rq_host_wq_pidx(struct t4_wq *wq) { return wq->rq.queue[wq->rq.size].status.host_wq_pidx; } static inline u16 t4_rq_wq_size(struct t4_wq *wq) { - return wq->rq.size * T4_RQ_NUM_SLOTS; + return wq->rq.size * T4_RQ_NUM_SLOTS; } +static inline int t4_sq_onchip(struct t4_sq *sq) +{ + return sq->flags & T4_SQ_ONCHIP; +} + static inline int t4_sq_empty(struct t4_wq *wq) { return wq->sq.in_use == 0; } static inline int t4_sq_full(struct t4_wq *wq) { return wq->sq.in_use == (wq->sq.size - 1); } static inline u32 t4_sq_avail(struct t4_wq *wq) { return wq->sq.size - 1 - wq->sq.in_use; } static inline void t4_sq_produce(struct t4_wq *wq, u8 len16) { wq->sq.in_use++; if (++wq->sq.pidx == wq->sq.size) wq->sq.pidx = 0; wq->sq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); if (wq->sq.wq_pidx >= wq->sq.size * T4_SQ_NUM_SLOTS) wq->sq.wq_pidx %= wq->sq.size * T4_SQ_NUM_SLOTS; } static inline void t4_sq_consume(struct t4_wq *wq) { + BUG_ON(wq->sq.in_use < 1); + if (wq->sq.cidx == wq->sq.flush_cidx) + wq->sq.flush_cidx = -1; wq->sq.in_use--; if (++wq->sq.cidx == wq->sq.size) wq->sq.cidx = 0; } static inline u16 t4_sq_host_wq_pidx(struct t4_wq *wq) { return wq->sq.queue[wq->sq.size].status.host_wq_pidx; } static inline u16 t4_sq_wq_size(struct t4_wq *wq) { return wq->sq.size * T4_SQ_NUM_SLOTS; } -static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc) +/* This function copies 64 byte coalesced work request to memory + * mapped BAR2 space. For coalesced WRs, the SGE fetches data + * from the FIFO instead of from Host. + */ +static inline void pio_copy(u64 __iomem *dst, u64 *src) { + int count = 8; + + while (count) { + writeq(*src, dst); + src++; + dst++; + count--; + } +} + +static inline void +t4_ring_sq_db(struct t4_wq *wq, u16 inc, union t4_wr *wqe, u8 wc) +{ + + /* Flush host queue memory writes. */ wmb(); - writel(QID(wq->sq.qid) | PIDX(inc), wq->db); + if (wc && inc == 1 && wq->sq.bar2_qid == 0 && wqe) { + CTR2(KTR_IW_CXGBE, "%s: WC wq->sq.pidx = %d\n", + __func__, wq->sq.pidx); + pio_copy((u64 __iomem *) + ((u64)wq->sq.bar2_va + SGE_UDB_WCDOORBELL), + (u64 *)wqe); + } else { + CTR2(KTR_IW_CXGBE, "%s: DB wq->sq.pidx = %d\n", + __func__, wq->sq.pidx); + writel(V_PIDX_T5(inc) | V_QID(wq->sq.bar2_qid), + (void __iomem *)((u64)wq->sq.bar2_va + + SGE_UDB_KDOORBELL)); + } + + /* Flush user doorbell area writes. */ + wmb(); + return; } -static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc) +static inline void +t4_ring_rq_db(struct t4_wq *wq, u16 inc, union t4_recv_wr *wqe, u8 wc) { + + /* Flush host queue memory writes. */ wmb(); - writel(QID(wq->rq.qid) | PIDX(inc), wq->db); + if (wc && inc == 1 && wq->rq.bar2_qid == 0 && wqe) { + CTR2(KTR_IW_CXGBE, "%s: WC wq->rq.pidx = %d\n", + __func__, wq->rq.pidx); + pio_copy((u64 __iomem *)((u64)wq->rq.bar2_va + + SGE_UDB_WCDOORBELL), (u64 *)wqe); + } else { + CTR2(KTR_IW_CXGBE, "%s: DB wq->rq.pidx = %d\n", + __func__, wq->rq.pidx); + writel(V_PIDX_T5(inc) | V_QID(wq->rq.bar2_qid), + (void __iomem *)((u64)wq->rq.bar2_va + + SGE_UDB_KDOORBELL)); + } + + /* Flush user doorbell area writes. */ + wmb(); + return; } static inline int t4_wq_in_error(struct t4_wq *wq) { return wq->rq.queue[wq->rq.size].status.qp_err; } static inline void t4_set_wq_in_error(struct t4_wq *wq) { wq->rq.queue[wq->rq.size].status.qp_err = 1; } +enum t4_cq_flags { + CQ_ARMED = 1, +}; + struct t4_cq { struct t4_cqe *queue; bus_addr_t dma_addr; - DECLARE_PCI_UNMAP_ADDR(mapping); + DEFINE_DMA_UNMAP_ADDR(mapping); struct t4_cqe *sw_queue; - void __iomem *gts; + void __iomem *bar2_va; + u64 bar2_pa; + u32 bar2_qid; struct c4iw_rdev *rdev; - u64 ugts; size_t memsize; __be64 bits_type_ts; u32 cqid; + u32 qid_mask; + int vector; u16 size; /* including status page */ u16 cidx; u16 sw_pidx; u16 sw_cidx; u16 sw_in_use; u16 cidx_inc; u8 gen; u8 error; + unsigned long flags; }; +static inline void write_gts(struct t4_cq *cq, u32 val) +{ + writel(val | V_INGRESSQID(cq->bar2_qid), + (void __iomem *)((u64)cq->bar2_va + SGE_UDB_GTS)); +} + +static inline int t4_clear_cq_armed(struct t4_cq *cq) +{ + return test_and_clear_bit(CQ_ARMED, &cq->flags); +} + static inline int t4_arm_cq(struct t4_cq *cq, int se) { u32 val; + set_bit(CQ_ARMED, &cq->flags); while (cq->cidx_inc > CIDXINC_MASK) { - val = SEINTARM(0) | CIDXINC(CIDXINC_MASK) | TIMERREG(7) | - INGRESSQID(cq->cqid); - writel(val, cq->gts); + val = SEINTARM(0) | CIDXINC(CIDXINC_MASK) | TIMERREG(7); + writel(val | V_INGRESSQID(cq->bar2_qid), + (void __iomem *)((u64)cq->bar2_va + SGE_UDB_GTS)); cq->cidx_inc -= CIDXINC_MASK; } - val = SEINTARM(se) | CIDXINC(cq->cidx_inc) | TIMERREG(6) | - INGRESSQID(cq->cqid); - writel(val, cq->gts); + val = SEINTARM(se) | CIDXINC(cq->cidx_inc) | TIMERREG(6); + writel(val | V_INGRESSQID(cq->bar2_qid), + (void __iomem *)((u64)cq->bar2_va + SGE_UDB_GTS)); cq->cidx_inc = 0; return 0; } static inline void t4_swcq_produce(struct t4_cq *cq) { cq->sw_in_use++; + if (cq->sw_in_use == cq->size) { + CTR2(KTR_IW_CXGBE, "%s cxgb4 sw cq overflow cqid %u\n", + __func__, cq->cqid); + cq->error = 1; + BUG_ON(1); + } if (++cq->sw_pidx == cq->size) cq->sw_pidx = 0; } static inline void t4_swcq_consume(struct t4_cq *cq) { + BUG_ON(cq->sw_in_use < 1); cq->sw_in_use--; if (++cq->sw_cidx == cq->size) cq->sw_cidx = 0; } static inline void t4_hwcq_consume(struct t4_cq *cq) { cq->bits_type_ts = cq->queue[cq->cidx].bits_type_ts; if (++cq->cidx_inc == (cq->size >> 4) || cq->cidx_inc == M_CIDXINC) { u32 val; - val = SEINTARM(0) | CIDXINC(cq->cidx_inc) | TIMERREG(7) | - INGRESSQID(cq->cqid); - writel(val, cq->gts); + val = SEINTARM(0) | CIDXINC(cq->cidx_inc) | TIMERREG(7); + write_gts(cq, val); cq->cidx_inc = 0; } if (++cq->cidx == cq->size) { cq->cidx = 0; cq->gen ^= 1; } } static inline int t4_valid_cqe(struct t4_cq *cq, struct t4_cqe *cqe) { return (CQE_GENBIT(cqe) == cq->gen); } +static inline int t4_cq_notempty(struct t4_cq *cq) +{ + return cq->sw_in_use || t4_valid_cqe(cq, &cq->queue[cq->cidx]); +} + static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe) { int ret; u16 prev_cidx; if (cq->cidx == 0) prev_cidx = cq->size - 1; else prev_cidx = cq->cidx - 1; if (cq->queue[prev_cidx].bits_type_ts != cq->bits_type_ts) { ret = -EOVERFLOW; cq->error = 1; printk(KERN_ERR MOD "cq overflow cqid %u\n", cq->cqid); + BUG_ON(1); } else if (t4_valid_cqe(cq, &cq->queue[cq->cidx])) { + + /* Ensure CQE is flushed to memory */ + rmb(); *cqe = &cq->queue[cq->cidx]; ret = 0; } else ret = -ENODATA; return ret; } static inline struct t4_cqe *t4_next_sw_cqe(struct t4_cq *cq) { + if (cq->sw_in_use == cq->size) { + CTR2(KTR_IW_CXGBE, "%s cxgb4 sw cq overflow cqid %u\n", + __func__, cq->cqid); + cq->error = 1; + BUG_ON(1); + return NULL; + } if (cq->sw_in_use) return &cq->sw_queue[cq->sw_cidx]; return NULL; } static inline int t4_next_cqe(struct t4_cq *cq, struct t4_cqe **cqe) { int ret = 0; if (cq->error) ret = -ENODATA; else if (cq->sw_in_use) *cqe = &cq->sw_queue[cq->sw_cidx]; else ret = t4_next_hw_cqe(cq, cqe); return ret; } static inline int t4_cq_in_error(struct t4_cq *cq) { return ((struct t4_status_page *)&cq->queue[cq->size])->qp_err; } static inline void t4_set_cq_in_error(struct t4_cq *cq) { ((struct t4_status_page *)&cq->queue[cq->size])->qp_err = 1; } +struct t4_dev_status_page { + u8 db_off; + u8 wc_supported; + u16 pad2; + u32 pad3; + u64 qp_start; + u64 qp_size; + u64 cq_start; + u64 cq_size; +}; #endif Index: projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/user.h =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/user.h (revision 331317) +++ projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe/iw_cxgbe/user.h (revision 331318) @@ -1,71 +1,77 @@ /* * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * * $FreeBSD$ */ #ifndef __C4IW_USER_H__ #define __C4IW_USER_H__ -#define C4IW_UVERBS_ABI_VERSION 2 +#define C4IW_UVERBS_ABI_VERSION 3 /* * Make sure that all structs defined in this file remain laid out so * that they pack the same way on 32-bit and 64-bit architectures (to * avoid incompatibility between 32-bit userspace and 64-bit kernels). * In particular do not use pointer types -- pass pointers in __u64 * instead. */ struct c4iw_create_cq_resp { __u64 key; __u64 gts_key; __u64 memsize; __u32 cqid; __u32 size; __u32 qid_mask; __u32 reserved; /* explicit padding (optional for i386) */ }; struct c4iw_create_qp_resp { __u64 ma_sync_key; __u64 sq_key; __u64 rq_key; __u64 sq_db_gts_key; __u64 rq_db_gts_key; __u64 sq_memsize; __u64 rq_memsize; __u32 sqid; __u32 rqid; __u32 sq_size; __u32 rq_size; __u32 qid_mask; __u32 flags; +}; + +struct c4iw_alloc_ucontext_resp { + __u64 status_page_key; + __u32 status_page_size; + __u32 reserved; /* explicit padding (optional for i386) */ }; #endif Index: projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe (revision 331317) +++ projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe (revision 331318) Property changes on: projects/bsd_rdma_4_9_stable_11/sys/dev/cxgbe ___________________________________________________________________ Added: svn:mergeinfo ## -0,0 +0,11 ## Merged /projects/clang390-import/sys/dev/cxgbe:r303250-309123 Merged /projects/clang391-import/sys/dev/cxgbe:r309166-310192 Merged /head/sys/dev/cxgbe:r302438-302440,302448-302450,302456-302457,302459-302460,302465,302470-302476,302480-302484,302486-302497,302499-302500,302502-302508,302510-302511,302514-302518,302520-302524,302528-302532,302534,302540-302554,302556-302561,302565,302567,302571-302573,302576-302577,302580-302581,302584-302587,302591-302592,302605-302612,302614,302617-302624,302626,302629-302638,302664,302666-302668,302673-302674,302692-302704,302706-302710,302713,302723,302726,302731,302733,302737,302770-302772,302778,302783-302785,302787-302790,302793-302795,302797-302798,302801-302806,302808-302819,302823-302824,302826-302827,302831,302834-302843,302845,302847-302853,302855-302861,302864,302867-302876,302878-302894,302896-302897,302899-302900,302902,302904,302906-302908,302911,302916-302917,302919-302921,302928-302930,302932-302933,302935-302937,302940-302947,302949-302952,302957,302960-302961,302965-302966,302972-302973,302980-302981,302986,302988,302999-303004,303009-303014,303016-303026,303028,303030-303034,303039-303040,303042-303046,303052,303059,303061,303066-303074,303076,303084,303086-303088,303090-303091,303094,303098,303101,303103-303104,303106,303109,303111,303113,303115,303123,303125-303129,303131-303132,303134,303142,303144-303145,303147,303151,303153-303157,303160,303164,303166,303171,303178,303180,303182,303186,303188,303190,303195,303199,303202,303204-303205,303208,303210-303213,303217,303219,303223,303225-303227,303230,303243-303244,303253,303261,303263-303266,303271-303274,303277,303279,303282-303285,303287-303289,303299,303307-303309,303315,303322,303326-303329,303335,303338,303343,303345-303354,303356,303361-303362,303366-303370,303373-303374,303379,303382-303388,303393,303396,303399-303400,303404-303406,303410,303413-303426,303429,303437-303438,303442,303444,303446,303448,303450,303453-303454,303456-303457,303461,303464-303468,303470-303478,303486,303488,303490-303492,303494,303497,303501,303503,303505-303506,303510,303512-303516,303518-303522,303530,303532,303536,303541-303543,303545-303548,303551,303553-303557,303559,303561-303569,303573,303576,303581,303583-303585,303587,303592-303594,303603-303605,303610-303613,303615,303626-303627,303630-303631,303637-303638,303643,303645-303647,303650,303652,303655,303657,303661,303663,303670,303674,303676-303677,303685,303687-303688,303693,303702-303707,303710,303712-303713,303715-303716,303719-303723,303725-303726,303728-303729,303737-303739,303744,303747,303750,303753-303755,303760,303763-303766,303771,303773,303782-303786,303788,303791-303796,303798-303800,303802,303804,303806,303813,303816,303818-303819,303822-303824,303830-303834,303836-303837,303841-303842,303845,303847,303855,303859-303860,303863,303867,303869-303870,303874,303877,303880-303881,303886-303887,303889,303891-303894,303897,303900-303901,303903-303904,303913-303914,303916,303919,303921-303924,303927-303937,303941-303949,303955,303958,303961-303962,303964,303971,303974,303978,303982,303989-303992,303998,304000-304002,304004-304009,304011-304012,304016,304018,304021,304023,304026,304033-304034,304040-304041,304043,304045-304050,304052-304055,304058-304059,304061,304066,304068,304071,304075-304077,304085,304087,304089,304102,304106,304108-304109,304111,304140,304142,304146,304149,304151-304152,304154,304160,304162,304168-304170,304174,304176,304178,304180,304182,304184-304185,304191-304192,304194,304202,304204-304206,304209,304217,304221-304222,304225-304232,304238-304239,304246-304249,304251-304256,304270-304271,304273,304279,304285-304292,304295,304297,304313,304315-304322,304327,304329-304331,304342,304374,304378,304430-304431,304438,304440-304441,304443-304444,304446-304447,304459-304461,304464,304476,304478-304479,304481-304485,304487,304492,304509,304520-304521,304530,304532,304535,304543,304545,304554-304559,304561,304563-304566,304570-304576,304579,304584,304591,304593-304601,304604-304605,304607-304608,304615,304620,304624-304626,304629,304635-304638,304640-304641,304643,304647,304649,304652,304654,304674,304676-304677,304680,304685,304687-304689,304691,304693-304694,304696-304698,304703,304710,304713,304721-304724,304728,304730,304736,304740,304742-304751,304753-304755,304779-304786,304788,304790-304791,304793,304795-304797,304799,304801-304802,304806-304812,304819,304821,304825-304826,304832-304834,304837-304838,304854-304855,304858,304873-304875,304877,304886,304892,304911,304916,304918,304920,304927-304928,304931,304953,304957-304959,304962,304964,304966,304969-304970,304972-304973,304975-304976,304979,304989,304991,305009,305012-305013,305015-305016,305018-305020,305022-305024,305026,305032-305034,305036,305039-305041,305044-305050,305052-305056,305058,305061,305065-305066,305071,305077-305078,305085-305086,305090,305093,305095,305104,305107-305116,305119,305122-305123,305125,305128-305130,305132-305133,305140-305141,305144-305145,305147-305149,305154-305155,305160,305166-305167,305169-305171,305173-305179,305182,305191,305193-305195,305197-305203,305205-305207,305209-305213,305215-305216,305218-305219,305221-305222,305224,305241,305244,305248,305253-305258,305264-305265,305268-305270,305276-305281,305283-305285,305303-305304,305312,305319-305320,305323-305328,305331-305334,305338-305340,305342-305345,305349,305353,305355-305358,305360,305363,305365,305367,305378-305386,305388-305389,305393,305396,305398,305405-305414,305421-305422,305425,305430,305432-305433,305449,305451,305453-305456,305467-305471,305482-305486,305492,305497,305502,305507-305509,305521,305524-305527,305535-305536,305539,305545-305546,305548-305549,305561,305563,305575,305578-305581,305585-305588,305590-305610,305612-305613,305620,305624,305626,305628-305629,305633,305652,305659,305661,305663,305665,305667,305671,305676-305677,305679,305684-305685,305689,305692,305695-305696,305699,305701-305704,305706-305708,305713,305715,305719,305722,305724-305725,305727-305730,305737,305739-305740,305744,305749,305751-305752,305756,305760-305761,305763,305767,305769-305772,305778,305788-305796,305801,305804,305807,305810-305814,305819,305822,305827,305834,305836,305840-305841,305851-305852,305854-305856,305862-305865,305867-305877,305887,305894,305896-305898,305902-305903,305906,305908,305916,305918-305928,305937-305943,305947,305951,305954,305956,305960-305970,305977-305978,305985,305987,305994-305995,306008-306009,306011-306015,306020,306026,306029-306031,306033,306036,306038,306041,306046,306048,306050,306061-306063,306071-306076,306081-306082,306087-306092,306094-306097,306102,306104,306109,306126,306129,306131-306132,306134,306137-306138,306159-306162,306174,306184-306185,306188,306193,306205-306206,306209,306212,306216,306218,306220,306224,306226,306228,306232,306257,306260-306262,306264,306266-306267,306273-306277,306279,306281,306285,306289-306292,306295,306297,306299-306301,306304,306306,306310-306313,306318,306320,306325,306329-306330,306332-306334,306342,306346,306349-306351,306355-306357,306359-306360,306366,306375-306377,306380,306387-306393,306395-306396,306409-306410,306417,306424-306426,306430-306433,306441-306445,306447,306451,306453-306454,306456-306460,306464-306465,306474,306477-306486,306488,306494-306496,306498,306504-306508,306510-306511,306513,306522,306526-306532,306534,306538,306541,306544,306550-306552,306554-306555,306557,306559-306571,306577-306579,306581-306585,306588-306589,306597-306608,306610-306622,306631,306634,306638,306640-306641,306647,306650,306652,306654,306656,306658,306665-306668,306670,306674,306677,306680,306686,306691,306695-306696,306699,306702-306704,306706,306709-306710,306712,306714-306715,306717-306725,306727-306728,306731-306736,306740,306742-306744,306746-306747,306751,306754-306756,306759,306762,306766,306771,306773,306775,306782-306783,306787,306790,306792-306795,306801,306803,306806-306808,306812,306821,306823,306825-306826,306828-306830,306840,306852-306855,306857,306860,306877,306897-306899,306901-306902,306936-306939,306944,306962,306966,306999-307003,307008,307012-307013,307036-307038,307041-307042,307044,307059-307060,307064,307066-307068,307070-307072,307074,307078,307087-307089,307091-307093,307095,307098,307115,307130-307132,307141,307145,307150-307152,307154,307159,307182,307189-307192,307195-307196,307204-307205,307215-307218,307220,307224,307227,307233,307236,307238-307241,307261-307263,307316,307318,307322-307324,307326-307327,307332-307334,307338-307339,307343,307348,307350,307364,307366,307368-307369,307372,307374,307379-307381,307384,307390-307392,307397-307398,307400-307402,307469,307499,307501,307507,307509,307515,307517-307518,307520-307523,307531,307538,307541,307544-307545,307550-307551,307556-307558,307561,307564,307566-307567,307570,307578,307584,307598,307624,307626,307628,307637-307638,307649-307651,307653,307657,307684-307688,307691-307694,307698,307700-307702,307710-307714,307726-307727,307731,307741,307745-307747,307752,307755-307756,307759,307768-307769,307774,307779,307783,307785-307787,307800-307802,307804-307805,307808,307821,307838-307840,307842-307845,307857,307861,307863,307865-307866,307869,307873,307876,307879-307880,307890-307891,307893,307902-307904,307908,307911,307917,307936-307937,307942,307948,307950-307955,307966,307969,307975-307978,307983,307985-307991,307994,308004-308006,308010-308031,308038-308040,308049-308051,308055-308056,308064,308066,308088-308089,308094-308099,308101,308104,308106,308108-308109,308112-308114,308116-308120,308124-308126,308129,308133-308134,308139,308142,308144,308148,308150,308156-308158,308160-308169,308171-308176,308186-308189,308194-308195,308197,308201,308206,308209-308212,308216-308221,308225,308228,308231-308233,308235,308237,308240,308242,308247,308250,308261,308270,308278,308286-308288,308295,308297-308298,308300-308301,308309,308312,308314,308339-308340,308343,308347-308348,308358,308375,308386,308406-308407,308409,308411-308414,308416,308420,308423-308425,308428,308430-308431,308434,308437,308440-308443,308456,308458-308459,308461-308462,308464,308471,308474,308476-308480,308487,308489,308493,308527-308530,308532,308534-308535,308538,308559,308564-308565,308567,308569-308570,308579-308581,308583,308597-308603,308608,308611-308612,308614-308619,308637,308640,308642-308643,308659,308664-308665,308667-308669,308671,308673,308687-308692,308694,308696,308700,308704-308706,308708,308723-308725,308729-308730,308733,308737,308742-308743,308745,308772-308776,308779-308782,308786-308787,308789,308793-308795,308797-308799,308806,308808-308809,308813,308820-308821,308824,308826-308827,308832,308843,308848-308856,308869,308871,308887,308895,308898,308904-308909,308925-308926,308940-308945,308948,308953,308957-308958,308969,308973,308980,308985,308995-308996,309006,309017,309026-309030,309039,309051,309060,309062-309063,309066-309069,309073,309078,309080-309083,309085,309090,309092-309093,309096-309099,309111-309113,309119-309121,309124,309126-309129,309131-309136,309138-309140,309142-309144,309146-309149,309151,309167,309169,309171,309174,309179,309184,309187,309189,309191-309192,309194-309195,309197-309201,309203,309209,309212,309216,309224-309245,309250-309252,309257,309260,309262,309264,309266-309270,309274,309281-309282,309284-309295,309297-309298,309300,309302,309307-309311,309316,309318-309321,309332,309345-309348,309353,309356-309357,309360-309361,309363-309365,309367,309369,309373-309374,309392-309394,309397,309400,309404-309413,309416,309425,309460-309461,309464,309472,309474,309477-309478,309491,309504,309513,309521,309523,309527-309528,309532,309534,309537-309539,309544,309547,309550-309552,309563,309566,309581-309582,309588-309591,309602,309607,309613,309624-309627,309630,309638-309640,309649-309650,309656-309657,309659-309660,309666,309669-309670,309676,309679,309682-309685,309688,309698-309700,309703-309705,309708-309712,309714,309716-309717,309719-309720,309722-309723,309725-309726,309728,309731-309734,309736-309738,309743-309744,309774-309780,309782-309784,309787,309795-309803,309805,309822-309823,309833,309835-309837,309839-309840,309851,309854,309856,309858,309869,309873-309875,309883,309886-309888,309893,309898,309901-309902,309904-309920,309922-309924,309926,309928-309930,309932,309934-309935,309937-309942,309944-309952,309957-310000,310012-310014,310021,310023,310025,310027-310028,310031,310033,310036,310038,310048-310050,310058,310083-310084,310086,310088-310090,310092,310095-310101,310104,310112,310114,310118,310124,310128,310132,310136-310137,310146,310150-310152,310155,310159-310160,310170-310171,310175,310177,310179-310182,310185-310187,310189,310193-310194,310196,310201-310203,310205,310225,310230,310232,310234,310236,310239,310242,310257-310259,310265-310268,310271-310275,310279,310284-310286,310298,310302,310304,310309,310312-310318,310324,310327,310329,310332,310336,310338-310341,310343-310345,310347-310348,310356,310360-310361,310365-310367,310373,310375,310387-310390,310417-310418,310420-310426,310430,310454-310455,310457-310459,310462,310465-310467,310478,310481,310488-310489,310491-310492,310496-310501,310503,310524,310534,310539,310547,310552-310555,310557,310559-310560,310574-310575,310583,310586-310593,310599,310608-310609,310611-310616,310621,310623,310627,310630,310633-310636,310638,310640,310642-310649,310651-310652,310654-310658,310660-310669,310674-310675,310677-310696,310698-310699,310702-310709,310711-310721,310724,310726-310729,310734,310741-310742,310744-310750,310752-310756,310758,310760,310762-310767,310770,310774,310778-310779,310782-310783,310785-310786,310791,310803,310805-310807,310810-310821,310823,310834,310840,310845-310846,310849-310850,310853-310854,310865-310866,310868,310870,310872-310874,310886-310889,310892,310894,310907,310925,310931,310942,310950,310952,310954,310957-310958,310960-310961,310963,310972,310976-310977,310982-310989,310996,311000-311005,311012,311014,311055,311101-311103,311105,311108,311111-311115,311122,311131,311133,311135-311136,311140,311151,311157,311160,311168-311169,311171-311175,311194,311204,311210,311221-311222,311224-311229,311233,311235-311236,311239-311240,311242-311243,311245-311250,311263,311265,311268-311275,311282-311285,311287-311288,311290-311294,311298,311305,311311,311340,311346,311349,311352,311377-311378,311381-311382,311384,311390,311392-311394,311426,311445-311447,311452-311453,311458-311459,311461-311462,311469-311475,311504-311505,311511,311517,311520,311522-311527,311529-311531,311548,311556,311565,311568-311570,311572,311584-311585,311601,311623,311636,311638-311640,311646-311647,311649-311651,311655,311657-311660,311664,311679-311680,311688,311693,311695,311699-311702,311706,311709-311715,311727,311733-311736,311739-311744,311748,311750,311754,311757-311762,311769,311780-311781,311787,311793-311794,311797,311804,311806-311807,311811,311815,311817,311831-311832,311846,311848,311850,311870-311871,311873,311877,311879-311881,311886,311888,311890-311898,311901-311903,311907,311911-311912,311914,311917,311919,311923-311926,311928-311929,311932-311933,311935,311941-311942,311946-311947,311949-311951,311953-311954,311958,311961-311963,311968-311969,311971-311972,311977,311979,311981,311983-311985,311989,311993,311996,312001,312003,312008-312009,312012,312026,312063,312077-312081,312083,312086-312087,312102,312105,312108,312110-312114,312118-312122,312124,312153,312162,312164,312191,312199,312208-312209,312213-312214,312216,312226,312228,312230-312232,312237,312250-312251,312288,312291-312293,312305,312307,312314,312328-312332,312335,312338,312341,312343,312346-312348,312353-312355,312368-312369,312374,312387,312389-312390,312392,312395-312396,312404,312407-312412,312414,312417-312419,312422-312430,312432,312434-312435,312437-312438,312443,312446,312450-312514,312519-312521,312523,312526-312528,312531-312539,312544,312547,312551,312553,312555,312558-312560,312563,312593,312598-312606,312608,312610,312614,312617,312621-312622,312636-312637,312641,312644-312650,312659,312663-312664,312667,312669,312678-312679,312684-312686,312688-312690,312692,312694,312698-312699,312703,312721-312724,312728,312750,312759-312760,312764-312765,312767,312777,312779-312780,312782,312787,312791-312792,312814-312815,312825,312831-312832,312855,312857,312859,312865,312872,312875-312882,312886,312888-312890,312893,312897,312899,312901-312902,312904,312906,312913-312914,312919,312925,312932-312935,312937,312939,312942-312943,312947,312949,312952,312954,312973-312975,312977,312979-312983,312986-312988,312991-312995,313005-313009,313015-313018,313020-313021,313025,313027,313030-313032,313040-313042,313044-313045,313047-313048,313052,313069,313074,313080-313081,313083,313097,313104-313107,313109,313111,313113,313133,313141,313154,313160,313162-313163,313166,313169,313174,313176-313177,313180,313182,313184-313186,313188,313190-313191,313194,313249-313252,313254,313257-313261,313268-313275,313277-313285,313287,313318,313327-313328,313330-313333,313335,313337,313341,313345-313346,313348-313352,313356,313358-313360,313374,313376-313379,313386,313389-313392,313394-313398,313401,313404,313407,313409,313411,313436-313439,313442,313448-313449,313451,313453-313455,313457-313458,313467,313472,313477,313483,313493-313497,313547,313549,313554-313556,313559,313561-313564,313568,313572-313573,313575,313645,313650,313652-313656,313659-313674,313679,313683-313684,313686-313688,313690,313692-313697,313707,313709-313710,313712-313713,313715,313727,313730-313731,313733-313740,313744-313746,313748-313749,313751-313752,313756-313757,313760-313761,313769-313772,313774-313775,313777-313779,313782,313784,313797-313800,313804-313810,313813,313817,313819-313821,313823,313841,313850-313855,313859,313863-313867,313869-313875,313877-313879,313893,313895,313897-313898,313901-313910,313912-313915,313917-313919,313922,313924-313925,313928,313933-313934,313936-313937,313939-313941,313944-313949,313953,313955,313958-313963,313965-313967,313972-313974,313982,313992-313996,313999,314001,314027,314035,314037-314040,314043-314046,314048,314050,314054-314056,314058-314062,314064,314068,314071,314075,314086-314088,314094,314097,314100-314101,314104-314106,314109,314112-314114,314118,314131-314132,314136,314139,314145,314148,314150,314152-314155,314158,314167,314176,314179,314183,314185,314187,314189-314191,314193-314196,314200,314203-314208,314211,314213-314215,314217-314219,314226,314228-314233,314239-314242,314244-314247,314252-314257,314267-314268,314271-314275,314280,314282,314287,314290-314295,314299-314300,314302,314306-314314,314316,314319-314321,314326,314328,314336-314339,314341-314345,314357-314358,314365,314370,314372,314374-314375,314382,314387,314396-314400,314402-314404,314419-314420,314424,314429-314430,314435,314442,314450-314464,314466-314468,314470,314472,314474-314479,314483-314487,314489-314490,314496-314497,314502,314504-314505,314507,314509,314520,314527-314528,314533,314535,314541-314543,314545,314547-314549,314553-314558,314561-314562,314564,314571-314572,314576,314578-314579,314587,314592,314596,314598,314601,314604,314610-314611,314614-314621,314623,314625-314627,314636-314638,314643-314645,314647-314651,314653-314659,314666,314671,314675-314676,314686,314691,314693-314694,314700-314701,314708-314709,314714-314718,314720,314723,314768-314774,314782-314783,314785-314786,314790,314793-314805,314807-314808,314810-314812,314814,314826,314828,314830-314831,314833,314837,314840-314843,314849-314853,314859-314861,314863-314872,314877-314878,314884-314896,314902-314908,314912-314914,314916,314918-314920,314924-314928,314933-314935,314942-314948,314950-314953,314956,314960,314962-314968,314970-314971,314973-314974,314987-314989,314993-314996,314998,315001-315002,315004-315006,315008-315010,315016,315018-315019,315022,315025,315030-315036,315038-315041,315046-315050,315053-315054,315056,315059,315067,315074-315078,315082-315089,315091,315095-315098,315102,315106,315108,315111-315114,315132-315133,315143,315146,315155-315163,315165,315167,315170,315176-315178,315186-315187,315192,315196,315199-315204,315206,315208,315211-315215,315225,315228-315230,315233-315238,315242,315244,315272-315273,315277-315281,315283,315286-315287,315289,315292,315294,315298,315303,315305,315307,315310,315318,315320,315323,315325-315327,315331,315334-315337,315360-315365,315368,315370,315402,315404-315405,315408,315410,315412,315419-315420,315422-315423,315426-315427,315429-315431,315435,315442-315443,315449,315452-315453,315455,315457-315461,315466,315478,315482,315484-315485,315488-315490,315496-315510,315516,315518,315523-315526,315529-315530,315533-315534,315536,315545,315552,315560-315561,315564,315571-315580,315586-315589,315591,315595,315597-315598,315601,315603,315615,315619,315621,315636,315639-315642,315645-315648,315652-315654,315656,315670,315672-315674,315677-315678,315681-315683,315685-315692,315694,315697-315701,315708-315709,315711,315713-315716,315718-315720,315732,315734,315736,315738,315745,315749,315753-315754,315759-315762,315764-315766,315768,315770-315772,315774-315776,315788-315790,315793,315795-315799,315802-315804,315845,315850-315853,315856-315861,315863-315864,315869-315871,315876,315878,315892,315896,315900,315908,315910,315912-315913,315920-315922,315932-315935,315947,315956-315957,315960-315961,315964,315968,315973-315974,315983,316005,316008-316009,316011-316013,316017,316020,316022-316023,316029-316038,316049-316051,316053,316055-316057,316060-316062,316064-316066,316076-316081,316095,316099-316100,316102-316113,316115,316118-316119,316125,316131-316133,316171-316172,316174-316175,316179-316181,316183-316184,316211-316214,316258,316260,316275,316279-316280,316283-316289,316304,316309-316311,316313-316314,316328-316329,316341-316343,316355,316357-316358,316368,316374,316377,316393-316395,316397,316412-316413,316424,316426-316428,316430,316433-316437,316460-316461,316466-316468,316473,316477,316484-316485,316489-316493,316495,316497,316500-316501,316503-316508,316510-316512,316515,316519-316526,316528-316537,316541-316542,316544,316548-316554,316557-316558,316561-316566,316568,316571,316573,316577-316578,316580,316585-316587,316589-316590,316592-316593,316599-316603,316606-316612,316617,316625,316627-316635,316637-316638,316643-316644,316647-316649,316652-316659,316661,316664-316667,316669-316670,316676-316677,316679,316681-316682,316684-316687,316689-316692,316694-316695,316698-316699,316702,316704,316706,316715-316720,316732,316734,316738-316740,316742-316743,316745-316747,316749-316750,316754-316756,316758-316761,316766,316768,316770-316771,316774,316776-316777,316780,316782,316792-316794,316802,316809-316818,316822-316824,316829,316851-316854,316856,316859,316866-316867,316869,316874,316936-316938,316940-316941,316945-316946,316949,316953,316956,316971-316972,316989,316991,316993-316997,317033-317034,317036,317041,317049-317052,317058,317060,317064,317075-317076,317079-317080,317092,317097,317099-317100,317107,317135,317137-317139,317141,317148,317154-317156,317159-317164,317168-317171,317173,317179-317180,317185-317186,317192-317193,317196-317197,317200-317201,317208,317212,317214-317216,317235-317239,317244,317246-317247,317253-317254,317265,317267,317269-317270,317272-317276,317279-317280,317283-317284,317288-317297,317299-317300,317304-317313,317315-317316,317323-317324,317340,317343-317346,317348-317350,317353,317356,317361,317369-317372,317376,317378,317382,317385,317388-317390,317409-317415,317428,317431,317435-317437,317444-317445,317457-317460,317463-317465,317483,317504-317505,317507,317511-317512,317517-317518,317522-317523,317527,317533,317541-317545,317547,317558,317560-317561,317566,317568,317576,317578,317583-317585,317591-317592,317594,317596-317597,317601,317606,317608,317610-317611,317632,317645,317648,317651-317652,317657-317660,317663,317665-317667,317678-317682,317694,317696,317700-317710,317712-317713,317715,317723,317729-317732,317736-317737,317739-317743,317745,317747-317748,317750-317752,317754-317755,317758-317759,317774-317775,317777,317779,317782-317786,317789-317790,317799,317801-317803,317806,317809-317821,317824,317828,317830,317835-317837,317839-317840,317842-317848,317854,317862-317863,317886-317888,317901,317904-317906,317908-317909,317912,317915,317923,317928,317931,317933-317935,317938,317981-317982,317996,317998,318004,318006-318011,318014-318017,318021,318024-318026,318090-318092,318094,318098-318100,318107,318109,318116,318124-318126,318130,318135-318138,318141-318144,318147,318150,318160-318161,318163,318167,318175-318182,318185,318188-318191,318193-318194,318210-318211,318242-318243,318246,318251-318252,318254-318257,318259,318263,318276,318278,318280-318283,318287,318294,318298-318299,318302-318303,318305,318307,318312-318318,318320,318325-318326,318352-318354,318356,318360,318364,318380,318382-318383,318386,318398-318401,318427,318431-318432,318434,318436-318437,318439,318444-318446,318448,318450,318476,318478-318479,318481,318511-318512,318514-318517,318520,318523,318527,318530-318531,318539,318545-318546,318565,318571,318574,318578,318580-318582,318584-318586,318588,318590-318593,318599,318601,318603,318606,318631-318633,318643,318646-318647,318649,318651,318655,318676-318679,318688-318689,318693-318695,318701-318712,318715,318717-318723,318734,318738-318739,318745,318748-318749,318754-318755,318762,318765,318774,318780-318781,318788,318790,318794-318795,318812,318814,318816,318818-318824,318827-318834,318860-318861,318863,318867,318872,318879,318884,318894-318896,318899-318900,318906-318908,318914,318916-318917,318920-318926,318928,318930,318932,318935,318943,318945,318952-318955,318960,318962,318966,318968,318970,318973-318975,318982,318986-318996,318998-318999,319001-319002,319008,319010,319015,319019-319020,319022,319026-319031,319033-319049,319051-319052,319054,319056-319063,319071-319078,319083-319086,319096,319118,319125,319132,319137-319138,319156-319159,319162,319167,319197-319201,319212,319219,319229,319246,319293-319295,319312,319316-319321,319336-319341,319350,319355,319365,319369,319409-319410,319412-319414,319432,319444,319454-319456,319473,319475,319481,319487-319488,319490-319491,319493,319499-319501,319509,319513,319516-319520,319539-319540,319542-319546,319548,319551,319556-319558,319561-319562,319571,319595,319603-319605,319608-319610,319612,319620-319621,319626-319627,319634,319637,319642,319655-319656,319659-319660,319662,319670-319672,319675-319682,319688-319690,319694,319699,319702,319709,319714,319717-319718,319731,319746-319751,319755-319758,319761-319769,319774-319775,319793,319796,319798,319800,319803,319806,319811,319813-319815,319817-319818,319824-319825,319827,319829-319830,319834,319836,319841-319842,319844-319846,319850,319852,319854-319859,319861-319864,319866-319867,319869,319871,319873-319876,319882,319885,319888,319890,319895,319899-319900,319905,319916,319918,319928,319932-319934,319942,319947,319949,319953-319957,319962,319964,319968-319970,319972,319975,319987,319995-319996,320007-320008,320010-320012,320028,320030,320034-320035,320038,320048-320049,320051-320052,320056,320061-320063,320065,320069-320070,320072,320076-320079,320093,320108,320118,320121,320123-320126,320135,320145-320146,320151-320153,320156,320163,320165-320167,320170,320172-320176,320181,320183-320186,320189,320192-320196,320201-320202,320204,320206-320208,320216,320234-320235,320237-320240,320242,320252,320255-320256,320259,320262-320264,320266-320267,320269-320271,320273,320277,320288,320292,320296,320300-320301,320304,320307-320308,320313-320314,320316-320319,320322-320323,320332-320339,320343-320345,320352-320353,320359-320360,320363-320364,320372,320387,320390,320392,320403,320408-320409,320412,320417-320418,320420-320425,320430,320433,320441-320446,320451-320452,320456,320458,320460-320463,320467-320468,320472,320478-320479,320481-320483,320488-320495,320497-320498,320501,320508-320509,320517,320521-320522,320527,320546-320547,320553,320555,320560-320561,320570,320574-320578,320580,320595,320599,320604,320619-320620,320624-320636,320644,320655-320656,320658-320659,320663,320665,320670-320672,320675,320678,320683,320686,320688,320690,320694,320696,320701-320703,320705,320714,320723-320724,320729-320730,320733-320734,320736-320737,320740-320743,320748,320750,320754-320755,320761-320762,320773-320775,320785,320796,320801,320803-320804,320806-320807,320810-320820,320836-320837,320842-320843,320851-320854,320865,320868,320876,320883,320892-320893,320895-320896,320900-320902,320906,320914,320916,320918,320927-320932,320936-320938,320941,320956,320969,320974-320975,320977,320980,320982,320984,320986,320988-320991,320996-320997,320999,321001-321003,321011,321015,321023,321035-321036,321045-321046,321048,321075,321080-321082,321102,321104,321106,321109,321111,321138,321173,321196,321203,321206-321207,321214,321217-321218,321226,321228,321230,321233,321235,321240,321247-321248,321256,321261,321284,321286,321293-321294,321298,321302,321305-321306,321314,321327-321330,321332,321342,321347-321349,321356,321366,321368-321371,321376-321378,321385-321387,321393,321397-321401,321406-321410,321420,321422-321423,321433,321436-321437,321439,321444,321446,321450,321455-321457,321460-321461,321471,321483-321484,321486,321489-321490,321502,321512,321514,321580-321581,321583-321584,321586,321588-321589,321601,321603,321605-321608,321620-321623,321625,321627-321628,321639-321640,321652,321664,321668-321670,321674-321675,321684-321686,321688-321689,321702-321706,321708-321714,321719-321723,321726-321727,321730,321733-321734,321737,321743-321746,321751,321762,321772-321773,321779-321780,321782,321791-321792,321794-321795,321798-321799,321803,321805-321806,321810,321812,321816,321821,321823,321826,321828,321836-321847,321849,321851-321852,321856,321858-321859,321864,321876,321884,321896,321899,321915,321919,321921-321922,321924,321926,321933-321934,321938,321947-321952,321954,321959,321963,321965,321969,321983-321986,321991,322015-322016,322019,322023,322025-322026,322028,322036-322041,322050,322052,322056,322059,322073-322074,322076-322077,322097,322101,322112-322113,322124,322139,322169-322171,322175,322177-322179,322200-322201,322203,322209,322212-322214,322218,322222,322226,322228,322230,322233-322234,322237,322239,322241,322245,322248,322250-322251,322255-322258,322270-322272,322280,322282,322288,322291,322293,322296-322297,322299,322302,322304-322306,322308-322310,322316,322320-322323,322326,322328-322329,322331,322336,322351,322354-322357,322360,322364,322368,322371,322373-322374,322380,322383,322386,322391-322392,322397,322401,322404-322406,322408,322410-322411,322418,322426-322427,322431,322434-322437,322441,322443,322445,322447-322451,322456,322459,322463-322465,322473-322479,322483,322485-322488,322493-322496,322527,322530,322542,322544,322546-322548,322550,322567,322571,322580-322581,322590,322592-322593,322596,322618,322627-322628,322633,322636,322648,322667,322669,322672,322677-322678,322680,322682-322684,322694-322695,322706,322710,322713-322714,322718-322723,322726,322740,322746,322749-322752,322756-322757,322762-322763,322770-322771,322773-322775,322795-322800,322802-322803,322810,322812,322816,322821,322825-322826,322830,322832-322833,322852,322854-322855,322868,322872,322874-322875,322883,322885,322888,322894,322896-322897,322899,322901-322903,322913,322915,322921,322923,322925-322941,322947-322948,322951,322959,322965,322970-322971,322980-322982,322984,322986-322987,322992,322994-322995,322997-322999,323001-323004,323010-323011,323014-323018,323020-323022,323024-323025,323032,323039-323040,323045-323047,323051,323053-323054,323058-323059,323062-323065,323068,323074,323076,323082,323084,323086,323100,323102,323112,323114,323126-323128,323131-323133,323151,323155,323160,323166,323170,323174-323178,323183,323192-323194,323197,323217,323220,323224-323226,323228-323230,323234-323236,323245,323252-323253,323258,323261,323263,323272,323275,323278,323280,323286,323290,323305,323307-323308,323310,323317,323324-323327,323341,323347,323349-323351,323355,323364,323367-323368,323379,323382,323385-323392,323394,323405,323407,323428,323433,323435-323436,323438-323439,323448,323450,323465,323474,323478-323483,323491,323494,323496-323499,323509,323513,323522,323524-323525,323528,323539-323541,323544,323547,323552-323554,323557-323559,323561-323562,323564,323572-323575,323578-323585,323587-323589,323597-323598,323611-323613,323616,323625,323630-323631,323642,323654-323656,323671,323675-323676,323683,323689,323691-323692,323703-323705,323707,323709-323710,323714-323715,323722,323727-323729,323768-323769,323772,323781-323782,323785-323786,323791,323793,323796-323797,323812-323813,323822,323824,323834,323836,323839-323840,323851-323854,323856,323858-323860,323864,323867-323868,323873,323880,323882,323885-323886,323895-323897,323905-323910,323915-323916,323918,323924,323933-323934,323945,323952,323961-323962,323973,323978,323981,323985,323991,323993-323995,323997-323998,324007,324011-324012,324014,324016,324026,324039,324041,324044-324045,324048-324050,324065-324066,324072-324075,324077-324081,324086-324088,324098-324100,324107-324108,324112-324114,324123,324125,324127,324143,324146,324148,324155-324156,324163-324164,324168-324170,324172-324174,324178-324179,324185,324193-324197,324199-324202,324206,324212,324220-324224,324226,324228,324241,324243,324248-324249,324257-324258,324260-324261,324276-324278,324281,324285-324286,324292,324301-324316,324320-324321,324326-324328,324330,324334-324335,324343-324350,324354-324355,324359-324360,324362,324364,324366-324367,324369,324373,324378,324388,324413,324415,324425,324427,324429-324433,324437-324439,324445,324449-324454,324456-324457,324460,324471,324478,324487-324492,324497,324501-324502,324506,324509,324516-324517,324528,324535-324536,324538,324547,324552-324559,324566,324568,324590,324592-324595,324597,324600,324606,324609-324610,324613,324621-324623,324628-324637,324639,324644,324646-324654,324659-324661,324665,324668-324672,324681,324683,324689,324694,324696,324700-324704,324709,324716-324717,324719,324721,324727,324738,324752,324757,324770,324772,324774,324778,324780,324787,324789,324792-324795,324803-324805,324807,324809,324814,324817,324821,324823-324824,324826,324836,324841-324845,324850,324853,324857,324862-324865,324867-324869,324876-324883,324920,324923,324926,324928-324929,324940-324941,324947,324956-324957,324972,324977-324982,324991-324996,325007-325011,325014,325016-325018,325028,325030,325035,325039,325042,325044-325045,325050-325051,325054-325056,325059-325061,325063,325065-325067,325093-325094,325101,325103,325108,325114-325116,325156-325158,325170-325176,325227-325228,325232,325248,325266,325268-325279,325285-325286,325294-325299,325310,325321,325328,325332,325338-325339,325355,325359-325364,325371,325373,325376-325377,325379-325380,325382-325383,325386-325389,325397,325420,325433,325436,325438,325441,325444,325447,325451,325456,325458,325469,325478-325485,325515,325517,325528,325530,325532-325533,325552-325556,325558-325559,325561,325566-325567,325571,325588,325606,325608,325610,325614-325616,325635,325637-325638,325641,325648-325658,325662,325670-325671,325681,325683,325685-325694,325706-325708,325716-325718,325723-325725,325737,325743-325744,325748,325755,325758-325759,325764,325766-325767,325769,325775,325779-325780,325795,325805-325807,325811-325813,325817,325827,325834-325835,325841,325850-325851,325857,325861,325863,325865,325887-325888,325897,325917-325922,325924-325925,325946-325947,325950,325953,325959-325960,325962-325963,325976,325997,326019-326021,326028,326030,326032,326034,326036,326038-326041,326046,326052,326055,326058,326060-326061,326063,326065,326067-326070,326072,326074,326082,326084,326086,326089-326090,326093-326094,326096,326098,326100-326101,326106-326107,326110-326112,326115-326118,326122,326132,326134-326137,326140,326143-326144,326150,326161,326169,326173,326175-326178,326182,326184-326187,326194-326200,326228-326229,326234-326235,326237,326244,326273,326281,326284,326286,326288-326290,326307,326309,326311,326315,326330-326331,326343,326361-326362,326371,326378,326383-326384,326391-326392,326401,326407,326409-326410,326412,326420-326422,326424,326429,326438,326440-326448,326455,326480,326483-326496,326502,326504,326507,326509-326510,326518,326522,326526-326527,326544,326547,326558,326560-326563,326565-326567,326584-326594,326597,326600,326609-326610,326613,326615-326616,326624,326628-326629,326633,326640,326646,326654-326655,326657,326662,326666,326668-326671,326698,326707-326712,326714,326716,326720,326731-326732,326738,326748,326750,326752-326754,326764-326765,326768-326769,326772,326774,326776,326784,326791-326792,326794,326796-326799,326811-326813,326831,326834-326835,326847,326851,326853-326858,326861-326863,326866,326872-326888,326897-326898,326909,326912,326914,326919,326924-326927,326935,326937,326940,326953-326954,326957,326960-326963,326971,326973,326977,326982-326983,326986,326988-326990,326993-326995,327004,327029,327032,327034,327047-327050,327053,327074,327088,327094,327097,327099,327118,327140,327164,327167-327168,327183,327211,327213,327220-327222,327226,327235-327236,327264,327283-327286,327289,327295,327298-327299,327316,327319,327329,327336,327346-327347,327350-327354,327356,327359,327367,327375-327377,327385-327386,327390,327393-327399,327401-327402,327416,327420,327435,327437-327439,327444,327448-327451,327453-327454,327468-327469,327472,327489,327496-327498,327503,327517,327523-327525,327540,327555,327560-327561,327567,327578,327593,327597,327606,327612,327625,327662,327666,327674-327676,327682-327683,327685,327697-327700,327703-327705,327718,327721-327726,327730,327737,327751-327753,327755,327760,327768,327770,327775,327779-327780,327788,327792,327796,327816-327822,327826,327834,327845,327862,327874-327875,327878-327881,327888,327905,327911-327914,327917,327925,327930,327946-327947,327951,327963-327965,327972-327973,327991,327996,328007-328009,328013,328027,328029-328032,328035,328052,328055,328060-328061,328082-328083,328096,328106-328107,328116,328119-328120,328126,328128,328135,328153,328156-328157,328160-328161,328163,328166,328169,328171,328173,328177,328188-328192,328199-328200,328202,328205-328206,328208-328209,328212,328221-328222,328224,328226,328228,328230,328232,328234,328237,328243,328246,328248,328250,328252,328254,328256,328264,328273,328283-328284,328287-328293,328297,328304-328305,328318,328326,328329,328332,328344,328350,328352,328382,328398-328399,328411-328412,328419,328436-328439,328441,328444,328446-328450,328468,328470,328491,328497,328504-328505,328536,328552,328567,328584,328591,328603,328611-328615,328622-328625,328627-328628,328641-328643,328647,328659,328694-328695,328720,328731,328769-328770,328773-328774,328779-328784,328795,328799,328806,328808,328826,328830,328835,328853,328864-328865,328876,328880,328895,328913,328917,328937,328971,328986-328987,328990,328995,328999-329000,329009,329019,329022,329024,329050,329054,329060,329092,329101,329214,329222,329264,329297,329365 Merged /projects/contrib-netbsd-update-12/sys/dev/cxgbe:r303899-303984 Merged /stable/11/sys/dev/cxgbe:r304545 Merged /projects/bsd_rdma_4_9/sys/dev/cxgbe:r319973-326168 Merged /projects/clang400-import/sys/dev/cxgbe:r311132-314524 Merged /projects/clang500-import/sys/dev/cxgbe:r316992-321352 Merged /projects/netbsd-tests-update-12/sys/dev/cxgbe:r303985-305029,305031-305318 Merged /projects/netbsd-tests-upstream-01-2017/sys/dev/cxgbe:r312125-313435 Merged /releng/11.1/sys/dev/cxgbe:r320566