diff --git a/sys/dev/irdma/fbsd_kcompat.c b/sys/dev/irdma/fbsd_kcompat.c index 2f6b350ac5b0..cc6dd5c947ae 100644 --- a/sys/dev/irdma/fbsd_kcompat.c +++ b/sys/dev/irdma/fbsd_kcompat.c @@ -1,736 +1,738 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * * Copyright (c) 2021 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenFabrics.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /*$FreeBSD$*/ #include "osdep.h" #include "ice_rdma.h" #include "irdma_di_if.h" #include "irdma_main.h" #include #include #include #include /* additional QP debuging option. Keep false unless needed */ bool irdma_upload_context = false; inline u32 irdma_rd32(struct irdma_dev_ctx *dev_ctx, u32 reg){ KASSERT(reg < dev_ctx->mem_bus_space_size, ("irdma: register offset %#jx too large (max is %#jx)", (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size)); return (bus_space_read_4(dev_ctx->mem_bus_space_tag, dev_ctx->mem_bus_space_handle, reg)); } inline void irdma_wr32(struct irdma_dev_ctx *dev_ctx, u32 reg, u32 value) { KASSERT(reg < dev_ctx->mem_bus_space_size, ("irdma: register offset %#jx too large (max is %#jx)", (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size)); bus_space_write_4(dev_ctx->mem_bus_space_tag, dev_ctx->mem_bus_space_handle, reg, value); } inline u64 irdma_rd64(struct irdma_dev_ctx *dev_ctx, u32 reg){ KASSERT(reg < dev_ctx->mem_bus_space_size, ("irdma: register offset %#jx too large (max is %#jx)", (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size)); return (bus_space_read_8(dev_ctx->mem_bus_space_tag, dev_ctx->mem_bus_space_handle, reg)); } inline void irdma_wr64(struct irdma_dev_ctx *dev_ctx, u32 reg, u64 value) { KASSERT(reg < dev_ctx->mem_bus_space_size, ("irdma: register offset %#jx too large (max is %#jx)", (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size)); bus_space_write_8(dev_ctx->mem_bus_space_tag, dev_ctx->mem_bus_space_handle, reg, value); } int irdma_register_qset(struct irdma_sc_vsi *vsi, struct irdma_ws_node *tc_node) { struct irdma_device *iwdev = vsi->back_vsi; struct ice_rdma_peer *peer = iwdev->rf->peer_info; struct ice_rdma_request req = {0}; struct ice_rdma_qset_update *res = &req.res; req.type = ICE_RDMA_EVENT_QSET_REGISTER; res->cnt_req = 1; res->res_type = ICE_RDMA_QSET_ALLOC; res->qsets.qs_handle = tc_node->qs_handle; res->qsets.tc = tc_node->traffic_class; res->qsets.vsi_id = vsi->vsi_idx; IRDMA_DI_REQ_HANDLER(peer, &req); tc_node->l2_sched_node_id = res->qsets.teid; vsi->qos[tc_node->user_pri].l2_sched_node_id = res->qsets.teid; return 0; } void irdma_unregister_qset(struct irdma_sc_vsi *vsi, struct irdma_ws_node *tc_node) { struct irdma_device *iwdev = vsi->back_vsi; struct ice_rdma_peer *peer = iwdev->rf->peer_info; struct ice_rdma_request req = {0}; struct ice_rdma_qset_update *res = &req.res; req.type = ICE_RDMA_EVENT_QSET_REGISTER; res->res_allocated = 1; res->res_type = ICE_RDMA_QSET_FREE; res->qsets.vsi_id = vsi->vsi_idx; res->qsets.teid = tc_node->l2_sched_node_id; res->qsets.qs_handle = tc_node->qs_handle; IRDMA_DI_REQ_HANDLER(peer, &req); } void * hw_to_dev(struct irdma_hw *hw) { struct irdma_pci_f *rf; rf = container_of(hw, struct irdma_pci_f, hw); return rf->pcidev; } void irdma_free_hash_desc(void *desc) { return; } int irdma_init_hash_desc(void **desc) { return 0; } int irdma_ieq_check_mpacrc(void *desc, void *addr, u32 len, u32 val) { u32 crc = calculate_crc32c(0xffffffff, addr, len) ^ 0xffffffff; int ret_code = 0; if (crc != val) { irdma_pr_err("mpa crc check fail %x %x\n", crc, val); ret_code = -EINVAL; } printf("%s: result crc=%x value=%x\n", __func__, crc, val); return ret_code; } /** * irdma_add_ipv6_addr - add ipv6 address to the hw arp table * @iwdev: irdma device * @ifp: interface network device pointer */ static void irdma_add_ipv6_addr(struct irdma_device *iwdev, struct ifnet *ifp) { struct ifaddr *ifa, *tmp; struct sockaddr_in6 *sin6; u32 local_ipaddr6[4]; u8 *mac_addr; char ip6buf[INET6_ADDRSTRLEN]; if_addr_rlock(ifp); IRDMA_TAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, tmp) { sin6 = (struct sockaddr_in6 *)ifa->ifa_addr; if (sin6->sin6_family != AF_INET6) continue; irdma_copy_ip_ntohl(local_ipaddr6, (u32 *)&sin6->sin6_addr); mac_addr = IF_LLADDR(ifp); printf("%s:%d IP=%s, MAC=%02x:%02x:%02x:%02x:%02x:%02x\n", __func__, __LINE__, ip6_sprintf(ip6buf, &sin6->sin6_addr), mac_addr[0], mac_addr[1], mac_addr[2], mac_addr[3], mac_addr[4], mac_addr[5]); irdma_manage_arp_cache(iwdev->rf, mac_addr, local_ipaddr6, IRDMA_ARP_ADD); } if_addr_runlock(ifp); } /** * irdma_add_ipv4_addr - add ipv4 address to the hw arp table * @iwdev: irdma device * @ifp: interface network device pointer */ static void irdma_add_ipv4_addr(struct irdma_device *iwdev, struct ifnet *ifp) { struct ifaddr *ifa; struct sockaddr_in *sin; u32 ip_addr[4] = {}; u8 *mac_addr; if_addr_rlock(ifp); IRDMA_TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { sin = (struct sockaddr_in *)ifa->ifa_addr; if (sin->sin_family != AF_INET) continue; ip_addr[0] = ntohl(sin->sin_addr.s_addr); mac_addr = IF_LLADDR(ifp); printf("%s:%d IP=%d.%d.%d.%d, MAC=%02x:%02x:%02x:%02x:%02x:%02x\n", __func__, __LINE__, ip_addr[0] >> 24, (ip_addr[0] >> 16) & 0xFF, (ip_addr[0] >> 8) & 0xFF, ip_addr[0] & 0xFF, mac_addr[0], mac_addr[1], mac_addr[2], mac_addr[3], mac_addr[4], mac_addr[5]); irdma_manage_arp_cache(iwdev->rf, mac_addr, ip_addr, IRDMA_ARP_ADD); } if_addr_runlock(ifp); } /** * irdma_add_ip - add ip addresses * @iwdev: irdma device * * Add ipv4/ipv6 addresses to the arp cache */ void irdma_add_ip(struct irdma_device *iwdev) { struct ifnet *ifp = iwdev->netdev; struct ifnet *ifv; int i; irdma_add_ipv4_addr(iwdev, ifp); irdma_add_ipv6_addr(iwdev, ifp); for (i = 0; ifp->if_vlantrunk != NULL && i < VLAN_N_VID; ++i) { ifv = VLAN_DEVAT(ifp, i); if (!ifv) continue; irdma_add_ipv4_addr(iwdev, ifv); irdma_add_ipv6_addr(iwdev, ifv); } } static void irdma_ifaddrevent_handler(void *arg, struct ifnet *ifp, struct ifaddr *ifa, int event) { struct irdma_pci_f *rf = arg; struct ifnet *ifv = NULL; struct sockaddr_in *sin; struct epoch_tracker et; int arp_index = 0, i = 0; u32 ip[4] = {}; if (!ifa || !ifa->ifa_addr || !ifp) return; if (rf->iwdev->netdev != ifp) { for (i = 0; rf->iwdev->netdev->if_vlantrunk != NULL && i < VLAN_N_VID; ++i) { NET_EPOCH_ENTER(et); ifv = VLAN_DEVAT(rf->iwdev->netdev, i); NET_EPOCH_EXIT(et); if (ifv == ifp) break; } if (ifv != ifp) return; } sin = (struct sockaddr_in *)ifa->ifa_addr; switch (event) { case IFADDR_EVENT_ADD: if (sin->sin_family == AF_INET) irdma_add_ipv4_addr(rf->iwdev, ifp); else if (sin->sin_family == AF_INET6) irdma_add_ipv6_addr(rf->iwdev, ifp); break; case IFADDR_EVENT_DEL: if (sin->sin_family == AF_INET) { ip[0] = ntohl(sin->sin_addr.s_addr); } else if (sin->sin_family == AF_INET6) { irdma_copy_ip_ntohl(ip, (u32 *)&((struct sockaddr_in6 *)sin)->sin6_addr); } else { break; } for_each_set_bit(arp_index, rf->allocated_arps, rf->arp_table_size) { if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip))) { irdma_manage_arp_cache(rf, rf->arp_table[arp_index].mac_addr, rf->arp_table[arp_index].ip_addr, IRDMA_ARP_DELETE); } } break; default: break; } } void irdma_reg_ipaddr_event_cb(struct irdma_pci_f *rf) { rf->irdma_ifaddr_event = EVENTHANDLER_REGISTER(ifaddr_event_ext, irdma_ifaddrevent_handler, rf, EVENTHANDLER_PRI_ANY); } void irdma_dereg_ipaddr_event_cb(struct irdma_pci_f *rf) { EVENTHANDLER_DEREGISTER(ifaddr_event_ext, rf->irdma_ifaddr_event); } static int irdma_get_route_ifp(struct sockaddr *dst_sin, struct ifnet *netdev, struct ifnet **ifp, struct sockaddr **nexthop, bool *gateway) { struct nhop_object *nh; if (dst_sin->sa_family == AF_INET6) nh = fib6_lookup(RT_DEFAULT_FIB, &((struct sockaddr_in6 *)dst_sin)->sin6_addr, 0, NHR_NONE, 0); else nh = fib4_lookup(RT_DEFAULT_FIB, ((struct sockaddr_in *)dst_sin)->sin_addr, 0, NHR_NONE, 0); if (!nh || (nh->nh_ifp != netdev && rdma_vlan_dev_real_dev(nh->nh_ifp) != netdev)) goto rt_not_found; *gateway = (nh->nh_flags & NHF_GATEWAY) ? true : false; *nexthop = (*gateway) ? &nh->gw_sa : dst_sin; *ifp = nh->nh_ifp; return 0; rt_not_found: pr_err("irdma: route not found\n"); return -ENETUNREACH; } /** * irdma_get_dst_mac - get destination mac address * @cm_node: connection's node * @dst_sin: destination address information * @dst_mac: mac address array to return */ int irdma_get_dst_mac(struct irdma_cm_node *cm_node, struct sockaddr *dst_sin, u8 *dst_mac) { struct ifnet *netdev = cm_node->iwdev->netdev; #ifdef VIMAGE struct rdma_cm_id *rdma_id = (struct rdma_cm_id *)cm_node->cm_id->context; struct vnet *vnet = rdma_id->route.addr.dev_addr.net; #endif struct ifnet *ifp; struct llentry *lle; struct sockaddr *nexthop; struct epoch_tracker et; int err; bool gateway; NET_EPOCH_ENTER(et); CURVNET_SET_QUIET(vnet); err = irdma_get_route_ifp(dst_sin, netdev, &ifp, &nexthop, &gateway); if (err) goto get_route_fail; if (dst_sin->sa_family == AF_INET) { err = arpresolve(ifp, gateway, NULL, nexthop, dst_mac, NULL, &lle); } else if (dst_sin->sa_family == AF_INET6) { err = nd6_resolve(ifp, gateway, NULL, nexthop, dst_mac, NULL, &lle); } else { err = -EPROTONOSUPPORT; } get_route_fail: CURVNET_RESTORE(); NET_EPOCH_EXIT(et); if (err) { pr_err("failed to resolve neighbor address (err=%d)\n", err); return -ENETUNREACH; } return 0; } /** * irdma_addr_resolve_neigh - resolve neighbor address * @cm_node: connection's node * @dst_ip: remote ip address * @arpindex: if there is an arp entry */ int irdma_addr_resolve_neigh(struct irdma_cm_node *cm_node, u32 dst_ip, int arpindex) { struct irdma_device *iwdev = cm_node->iwdev; struct sockaddr_in dst_sin = {}; int err; u32 ip[4] = {}; u8 dst_mac[MAX_ADDR_LEN]; dst_sin.sin_len = sizeof(dst_sin); dst_sin.sin_family = AF_INET; dst_sin.sin_port = 0; dst_sin.sin_addr.s_addr = htonl(dst_ip); err = irdma_get_dst_mac(cm_node, (struct sockaddr *)&dst_sin, dst_mac); if (err) return arpindex; ip[0] = dst_ip; return irdma_add_arp(iwdev->rf, ip, dst_mac); } /** * irdma_addr_resolve_neigh_ipv6 - resolve neighbor ipv6 address * @cm_node: connection's node * @dest: remote ip address * @arpindex: if there is an arp entry */ int irdma_addr_resolve_neigh_ipv6(struct irdma_cm_node *cm_node, u32 *dest, int arpindex) { struct irdma_device *iwdev = cm_node->iwdev; struct sockaddr_in6 dst_addr = {}; int err; u8 dst_mac[MAX_ADDR_LEN]; dst_addr.sin6_family = AF_INET6; dst_addr.sin6_len = sizeof(dst_addr); dst_addr.sin6_scope_id = iwdev->netdev->if_index; irdma_copy_ip_htonl(dst_addr.sin6_addr.__u6_addr.__u6_addr32, dest); err = irdma_get_dst_mac(cm_node, (struct sockaddr *)&dst_addr, dst_mac); if (err) return arpindex; return irdma_add_arp(iwdev->rf, dest, dst_mac); } int irdma_resolve_neigh_lpb_chk(struct irdma_device *iwdev, struct irdma_cm_node *cm_node, struct irdma_cm_info *cm_info) { + struct rdma_cm_id *rdma_id = (struct rdma_cm_id *)cm_node->cm_id->context; + struct vnet *vnet = rdma_id->route.addr.dev_addr.net; int arpindex; int oldarpindex; if ((cm_node->ipv4 && - irdma_ipv4_is_lpb(cm_node->loc_addr[0], cm_node->rem_addr[0])) || + irdma_ipv4_is_lpb(vnet, cm_node->loc_addr[0], cm_node->rem_addr[0])) || (!cm_node->ipv4 && irdma_ipv6_is_lpb(cm_node->loc_addr, cm_node->rem_addr))) { cm_node->do_lpb = true; arpindex = irdma_arp_table(iwdev->rf, cm_node->rem_addr, NULL, IRDMA_ARP_RESOLVE); } else { oldarpindex = irdma_arp_table(iwdev->rf, cm_node->rem_addr, NULL, IRDMA_ARP_RESOLVE); if (cm_node->ipv4) arpindex = irdma_addr_resolve_neigh(cm_node, cm_info->rem_addr[0], oldarpindex); else arpindex = irdma_addr_resolve_neigh_ipv6(cm_node, cm_info->rem_addr, oldarpindex); } return arpindex; } /** * irdma_add_handler - add a handler to the list * @hdl: handler to be added to the handler list */ void irdma_add_handler(struct irdma_handler *hdl) { unsigned long flags; spin_lock_irqsave(&irdma_handler_lock, flags); list_add(&hdl->list, &irdma_handlers); spin_unlock_irqrestore(&irdma_handler_lock, flags); } /** * irdma_del_handler - delete a handler from the list * @hdl: handler to be deleted from the handler list */ void irdma_del_handler(struct irdma_handler *hdl) { unsigned long flags; spin_lock_irqsave(&irdma_handler_lock, flags); list_del(&hdl->list); spin_unlock_irqrestore(&irdma_handler_lock, flags); } /** * irdma_set_rf_user_cfg_params - apply user configurable settings * @rf: RDMA PCI function */ void irdma_set_rf_user_cfg_params(struct irdma_pci_f *rf) { int en_rem_endpoint_trk = 0; int limits_sel = 4; rf->en_rem_endpoint_trk = en_rem_endpoint_trk; rf->limits_sel = limits_sel; rf->rst_to = IRDMA_RST_TIMEOUT_HZ; /* Enable DCQCN algorithm by default */ rf->dcqcn_ena = true; } /** * irdma_sysctl_dcqcn_update - handle dcqcn_ena sysctl update * @arg1: pointer to rf * @arg2: unused * @oidp: sysctl oid structure * @req: sysctl request pointer */ static int irdma_sysctl_dcqcn_update(SYSCTL_HANDLER_ARGS) { struct irdma_pci_f *rf = (struct irdma_pci_f *)arg1; int ret; u8 dcqcn_ena = rf->dcqcn_ena; ret = sysctl_handle_8(oidp, &dcqcn_ena, 0, req); if ((ret) || (req->newptr == NULL)) return ret; if (dcqcn_ena == 0) rf->dcqcn_ena = false; else rf->dcqcn_ena = true; return 0; } /** * irdma_dcqcn_tunables_init - create tunables for dcqcn settings * @rf: RDMA PCI function * * Create DCQCN related sysctls for the driver. * dcqcn_ena is writeable settings and applicable to next QP creation or * context setting. * all other settings are of RDTUN type (read on driver load) and are * applicable only to CQP creation. */ void irdma_dcqcn_tunables_init(struct irdma_pci_f *rf) { struct sysctl_oid_list *irdma_sysctl_oid_list; irdma_sysctl_oid_list = SYSCTL_CHILDREN(rf->tun_info.irdma_sysctl_tree); SYSCTL_ADD_PROC(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_enable", CTLFLAG_RW | CTLTYPE_U8, rf, 0, irdma_sysctl_dcqcn_update, "A", "enables DCQCN algorithm for RoCEv2 on all ports, default=true"); SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_cc_cfg_valid", CTLFLAG_RDTUN, &rf->dcqcn_params.cc_cfg_valid, 0, "set DCQCN parameters to be valid, default=false"); rf->dcqcn_params.min_dec_factor = 1; SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_min_dec_factor", CTLFLAG_RDTUN, &rf->dcqcn_params.min_dec_factor, 0, "set minimum percentage factor by which tx rate can be changed for CNP, Range: 1-100, default=1"); SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_min_rate_MBps", CTLFLAG_RDTUN, &rf->dcqcn_params.min_rate, 0, "set minimum rate limit value, in MBits per second, default=0"); SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_F", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_f, 0, "set number of times to stay in each stage of bandwidth recovery, default=0"); SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_T", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_t, 0, "set number of usecs that should elapse before increasing the CWND in DCQCN mode, default=0"); SYSCTL_ADD_U32(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_B", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_b, 0, "set number of MSS to add to the congestion window in additive increase mode, default=0"); SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_rai_factor", CTLFLAG_RDTUN, &rf->dcqcn_params.rai_factor, 0, "set number of MSS to add to the congestion window in additive increase mode, default=0"); SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_hai_factor", CTLFLAG_RDTUN, &rf->dcqcn_params.hai_factor, 0, "set number of MSS to add to the congestion window in hyperactive increase mode, default=0"); SYSCTL_ADD_U32(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_rreduce_mperiod", CTLFLAG_RDTUN, &rf->dcqcn_params.rreduce_mperiod, 0, "set minimum time between 2 consecutive rate reductions for a single flow, default=0"); } /** * irdma_dmamap_cb - callback for bus_dmamap_load */ static void irdma_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error) { if (error) return; *(bus_addr_t *) arg = segs->ds_addr; return; } /** * irdma_allocate_dma_mem - allocate dma memory * @hw: pointer to hw structure * @mem: structure holding memory information * @size: requested size * @alignment: requested alignment */ void * irdma_allocate_dma_mem(struct irdma_hw *hw, struct irdma_dma_mem *mem, u64 size, u32 alignment) { struct irdma_dev_ctx *dev_ctx = (struct irdma_dev_ctx *)hw->dev_context; device_t dev = dev_ctx->dev; void *va; int ret; ret = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ alignment, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ size, /* maxsize */ 1, /* nsegments */ size, /* maxsegsize */ BUS_DMA_ALLOCNOW, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &mem->tag); if (ret != 0) { device_printf(dev, "%s: bus_dma_tag_create failed, error %u\n", __func__, ret); goto fail_0; } ret = bus_dmamem_alloc(mem->tag, (void **)&va, BUS_DMA_NOWAIT | BUS_DMA_ZERO, &mem->map); if (ret != 0) { device_printf(dev, "%s: bus_dmamem_alloc failed, error %u\n", __func__, ret); goto fail_1; } ret = bus_dmamap_load(mem->tag, mem->map, va, size, irdma_dmamap_cb, &mem->pa, BUS_DMA_NOWAIT); if (ret != 0) { device_printf(dev, "%s: bus_dmamap_load failed, error %u\n", __func__, ret); goto fail_2; } mem->nseg = 1; mem->size = size; bus_dmamap_sync(mem->tag, mem->map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); return va; fail_2: bus_dmamem_free(mem->tag, va, mem->map); fail_1: bus_dma_tag_destroy(mem->tag); fail_0: mem->map = NULL; mem->tag = NULL; return NULL; } /** * irdma_free_dma_mem - Memory free helper fn * @hw: pointer to hw structure * @mem: ptr to mem struct to free */ int irdma_free_dma_mem(struct irdma_hw *hw, struct irdma_dma_mem *mem) { if (!mem) return -EINVAL; bus_dmamap_sync(mem->tag, mem->map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(mem->tag, mem->map); if (!mem->va) return -ENOMEM; bus_dmamem_free(mem->tag, mem->va, mem->map); bus_dma_tag_destroy(mem->tag); mem->va = NULL; return 0; } inline void irdma_prm_rem_bitmapmem(struct irdma_hw *hw, struct irdma_chunk *chunk) { kfree(chunk->bitmapmem.va); } diff --git a/sys/dev/irdma/irdma_cm.c b/sys/dev/irdma/irdma_cm.c index e107903efe4d..410774226e78 100644 --- a/sys/dev/irdma/irdma_cm.c +++ b/sys/dev/irdma/irdma_cm.c @@ -1,4253 +1,4262 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * * Copyright (c) 2015 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenFabrics.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /*$FreeBSD$*/ #include "irdma_main.h" static void irdma_cm_post_event(struct irdma_cm_event *event); static void irdma_disconnect_worker(struct work_struct *work); /** * irdma_free_sqbuf - put back puda buffer if refcount is 0 * @vsi: The VSI structure of the device * @bufp: puda buffer to free */ void irdma_free_sqbuf(struct irdma_sc_vsi *vsi, void *bufp) { struct irdma_puda_buf *buf = bufp; struct irdma_puda_rsrc *ilq = vsi->ilq; if (atomic_dec_and_test(&buf->refcount)) irdma_puda_ret_bufpool(ilq, buf); } /** * irdma_record_ird_ord - Record IRD/ORD passed in * @cm_node: connection's node * @conn_ird: connection IRD * @conn_ord: connection ORD */ static void irdma_record_ird_ord(struct irdma_cm_node *cm_node, u32 conn_ird, u32 conn_ord) { if (conn_ird > cm_node->dev->hw_attrs.max_hw_ird) conn_ird = cm_node->dev->hw_attrs.max_hw_ird; if (conn_ord > cm_node->dev->hw_attrs.max_hw_ord) conn_ord = cm_node->dev->hw_attrs.max_hw_ord; else if (!conn_ord && cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO) conn_ord = 1; cm_node->ird_size = conn_ird; cm_node->ord_size = conn_ord; } /** * irdma_copy_ip_ntohl - copy IP address from network to host * @dst: IP address in host order * @src: IP address in network order (big endian) */ void irdma_copy_ip_ntohl(u32 *dst, __be32 *src) { *dst++ = ntohl(*src++); *dst++ = ntohl(*src++); *dst++ = ntohl(*src++); *dst = ntohl(*src); } /** * irdma_copy_ip_htonl - copy IP address from host to network order * @dst: IP address in network order (big endian) * @src: IP address in host order */ void irdma_copy_ip_htonl(__be32 *dst, u32 *src) { *dst++ = htonl(*src++); *dst++ = htonl(*src++); *dst++ = htonl(*src++); *dst = htonl(*src); } /** * irdma_get_addr_info * @cm_node: contains ip/tcp info * @cm_info: to get a copy of the cm_node ip/tcp info */ static void irdma_get_addr_info(struct irdma_cm_node *cm_node, struct irdma_cm_info *cm_info) { memset(cm_info, 0, sizeof(*cm_info)); cm_info->ipv4 = cm_node->ipv4; cm_info->vlan_id = cm_node->vlan_id; memcpy(cm_info->loc_addr, cm_node->loc_addr, sizeof(cm_info->loc_addr)); memcpy(cm_info->rem_addr, cm_node->rem_addr, sizeof(cm_info->rem_addr)); cm_info->loc_port = cm_node->loc_port; cm_info->rem_port = cm_node->rem_port; } /** * irdma_fill_sockaddr4 - fill in addr info for IPv4 connection * @cm_node: connection's node * @event: upper layer's cm event */ static inline void irdma_fill_sockaddr4(struct irdma_cm_node *cm_node, struct iw_cm_event *event) { struct sockaddr_in *laddr = (struct sockaddr_in *)&event->local_addr; struct sockaddr_in *raddr = (struct sockaddr_in *)&event->remote_addr; laddr->sin_family = AF_INET; raddr->sin_family = AF_INET; laddr->sin_port = htons(cm_node->loc_port); raddr->sin_port = htons(cm_node->rem_port); laddr->sin_addr.s_addr = htonl(cm_node->loc_addr[0]); raddr->sin_addr.s_addr = htonl(cm_node->rem_addr[0]); } /** * irdma_fill_sockaddr6 - fill in addr info for IPv6 connection * @cm_node: connection's node * @event: upper layer's cm event */ static inline void irdma_fill_sockaddr6(struct irdma_cm_node *cm_node, struct iw_cm_event *event) { struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&event->local_addr; struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)&event->remote_addr; laddr6->sin6_family = AF_INET6; raddr6->sin6_family = AF_INET6; laddr6->sin6_port = htons(cm_node->loc_port); raddr6->sin6_port = htons(cm_node->rem_port); irdma_copy_ip_htonl(laddr6->sin6_addr.__u6_addr.__u6_addr32, cm_node->loc_addr); irdma_copy_ip_htonl(raddr6->sin6_addr.__u6_addr.__u6_addr32, cm_node->rem_addr); } /** * irdma_get_cmevent_info - for cm event upcall * @cm_node: connection's node * @cm_id: upper layers cm struct for the event * @event: upper layer's cm event */ static inline void irdma_get_cmevent_info(struct irdma_cm_node *cm_node, struct iw_cm_id *cm_id, struct iw_cm_event *event) { memcpy(&event->local_addr, &cm_id->m_local_addr, sizeof(event->local_addr)); memcpy(&event->remote_addr, &cm_id->m_remote_addr, sizeof(event->remote_addr)); if (cm_node) { event->private_data = cm_node->pdata_buf; event->private_data_len = (u8)cm_node->pdata.size; event->ird = cm_node->ird_size; event->ord = cm_node->ord_size; } } /** * irdma_send_cm_event - upcall cm's event handler * @cm_node: connection's node * @cm_id: upper layer's cm info struct * @type: Event type to indicate * @status: status for the event type */ static int irdma_send_cm_event(struct irdma_cm_node *cm_node, struct iw_cm_id *cm_id, enum iw_cm_event_type type, int status) { struct iw_cm_event event = {0}; event.event = type; event.status = status; irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "cm_node %p cm_id=%p state=%d accel=%d event_type=%d status=%d\n", cm_node, cm_id, cm_node->accelerated, cm_node->state, type, status); switch (type) { case IW_CM_EVENT_CONNECT_REQUEST: if (cm_node->ipv4) irdma_fill_sockaddr4(cm_node, &event); else irdma_fill_sockaddr6(cm_node, &event); event.provider_data = cm_node; event.private_data = cm_node->pdata_buf; event.private_data_len = (u8)cm_node->pdata.size; event.ird = cm_node->ird_size; break; case IW_CM_EVENT_CONNECT_REPLY: irdma_get_cmevent_info(cm_node, cm_id, &event); break; case IW_CM_EVENT_ESTABLISHED: event.ird = cm_node->ird_size; event.ord = cm_node->ord_size; break; case IW_CM_EVENT_DISCONNECT: case IW_CM_EVENT_CLOSE: /* Wait if we are in RTS but havent issued the iwcm event upcall */ if (!cm_node->accelerated) wait_for_completion(&cm_node->establish_comp); break; default: return -EINVAL; } return cm_id->event_handler(cm_id, &event); } /** * irdma_timer_list_prep - add connection nodes to a list to perform timer tasks * @cm_core: cm's core * @timer_list: a timer list to which cm_node will be selected */ static void irdma_timer_list_prep(struct irdma_cm_core *cm_core, struct list_head *timer_list) { struct irdma_cm_node *cm_node; int bkt; HASH_FOR_EACH_RCU(cm_core->cm_hash_tbl, bkt, cm_node, list) { if ((cm_node->close_entry || cm_node->send_entry) && atomic_inc_not_zero(&cm_node->refcnt)) list_add(&cm_node->timer_entry, timer_list); } } /** * irdma_create_event - create cm event * @cm_node: connection's node * @type: Event type to generate */ static struct irdma_cm_event * irdma_create_event(struct irdma_cm_node *cm_node, enum irdma_cm_event_type type) { struct irdma_cm_event *event; if (!cm_node->cm_id) return NULL; event = kzalloc(sizeof(*event), GFP_ATOMIC); if (!event) return NULL; event->type = type; event->cm_node = cm_node; memcpy(event->cm_info.rem_addr, cm_node->rem_addr, sizeof(event->cm_info.rem_addr)); memcpy(event->cm_info.loc_addr, cm_node->loc_addr, sizeof(event->cm_info.loc_addr)); event->cm_info.rem_port = cm_node->rem_port; event->cm_info.loc_port = cm_node->loc_port; event->cm_info.cm_id = cm_node->cm_id; irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "node=%p event=%p type=%u dst=%pI4 src=%pI4\n", cm_node, event, type, event->cm_info.loc_addr, event->cm_info.rem_addr); irdma_cm_post_event(event); return event; } /** * irdma_free_retrans_entry - free send entry * @cm_node: connection's node */ static void irdma_free_retrans_entry(struct irdma_cm_node *cm_node) { struct irdma_device *iwdev = cm_node->iwdev; struct irdma_timer_entry *send_entry; send_entry = cm_node->send_entry; if (!send_entry) return; cm_node->send_entry = NULL; irdma_free_sqbuf(&iwdev->vsi, send_entry->sqbuf); kfree(send_entry); atomic_dec(&cm_node->refcnt); } /** * irdma_cleanup_retrans_entry - free send entry with lock * @cm_node: connection's node */ static void irdma_cleanup_retrans_entry(struct irdma_cm_node *cm_node) { unsigned long flags; spin_lock_irqsave(&cm_node->retrans_list_lock, flags); irdma_free_retrans_entry(cm_node); spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); } /** * irdma_form_ah_cm_frame - get a free packet and build frame with address handle * @cm_node: connection's node ionfo to use in frame * @options: pointer to options info * @hdr: pointer mpa header * @pdata: pointer to private data * @flags: indicates FIN or ACK */ static struct irdma_puda_buf * irdma_form_ah_cm_frame(struct irdma_cm_node *cm_node, struct irdma_kmem_info *options, struct irdma_kmem_info *hdr, struct irdma_mpa_priv_info *pdata, u8 flags) { struct irdma_puda_buf *sqbuf; struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi; u8 *buf; struct tcphdr *tcph; u16 pktsize; u32 opts_len = 0; u32 pd_len = 0; u32 hdr_len = 0; if (!cm_node->ah || !cm_node->ah->ah_info.ah_valid) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "AH invalid\n"); return NULL; } sqbuf = irdma_puda_get_bufpool(vsi->ilq); if (!sqbuf) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "SQ buf NULL\n"); return NULL; } sqbuf->ah_id = cm_node->ah->ah_info.ah_idx; buf = sqbuf->mem.va; if (options) opts_len = (u32)options->size; if (hdr) hdr_len = hdr->size; if (pdata) pd_len = pdata->size; pktsize = sizeof(*tcph) + opts_len + hdr_len + pd_len; memset(buf, 0, pktsize); sqbuf->totallen = pktsize; sqbuf->tcphlen = sizeof(*tcph) + opts_len; sqbuf->scratch = cm_node; tcph = (struct tcphdr *)buf; buf += sizeof(*tcph); tcph->th_sport = htons(cm_node->loc_port); tcph->th_dport = htons(cm_node->rem_port); tcph->th_seq = htonl(cm_node->tcp_cntxt.loc_seq_num); if (flags & SET_ACK) { cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt; tcph->th_ack = htonl(cm_node->tcp_cntxt.loc_ack_num); tcph->th_flags |= TH_ACK; } else { tcph->th_ack = 0; } if (flags & SET_SYN) { cm_node->tcp_cntxt.loc_seq_num++; tcph->th_flags |= TH_SYN; } else { cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len; } if (flags & SET_FIN) { cm_node->tcp_cntxt.loc_seq_num++; tcph->th_flags |= TH_FIN; } if (flags & SET_RST) tcph->th_flags |= TH_RST; tcph->th_off = (u16)((sizeof(*tcph) + opts_len + 3) >> 2); sqbuf->tcphlen = tcph->th_off << 2; tcph->th_win = htons(cm_node->tcp_cntxt.rcv_wnd); tcph->th_urp = 0; if (opts_len) { memcpy(buf, options->addr, opts_len); buf += opts_len; } if (hdr_len) { memcpy(buf, hdr->addr, hdr_len); buf += hdr_len; } if (pdata && pdata->addr) memcpy(buf, pdata->addr, pdata->size); atomic_set(&sqbuf->refcount, 1); irdma_debug_buf(vsi->dev, IRDMA_DEBUG_ILQ, "TRANSMIT ILQ BUFFER", sqbuf->mem.va, sqbuf->totallen); return sqbuf; } /** * irdma_form_uda_cm_frame - get a free packet and build frame full tcpip packet * @cm_node: connection's node ionfo to use in frame * @options: pointer to options info * @hdr: pointer mpa header * @pdata: pointer to private data * @flags: indicates FIN or ACK */ static struct irdma_puda_buf * irdma_form_uda_cm_frame(struct irdma_cm_node *cm_node, struct irdma_kmem_info *options, struct irdma_kmem_info *hdr, struct irdma_mpa_priv_info *pdata, u8 flags) { struct irdma_puda_buf *sqbuf; struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi; u8 *buf; struct tcphdr *tcph; struct ip *iph; struct ip6_hdr *ip6h; struct ether_header *ethh; u16 pktsize; u16 eth_hlen = ETH_HLEN; u32 opts_len = 0; u32 pd_len = 0; u32 hdr_len = 0; u16 vtag; sqbuf = irdma_puda_get_bufpool(vsi->ilq); if (!sqbuf) return NULL; buf = sqbuf->mem.va; if (options) opts_len = (u32)options->size; if (hdr) hdr_len = hdr->size; if (pdata) pd_len = pdata->size; if (cm_node->vlan_id < VLAN_N_VID) eth_hlen += 4; if (cm_node->ipv4) pktsize = sizeof(*iph) + sizeof(*tcph); else pktsize = sizeof(*ip6h) + sizeof(*tcph); pktsize += opts_len + hdr_len + pd_len; memset(buf, 0, eth_hlen + pktsize); sqbuf->totallen = pktsize + eth_hlen; sqbuf->maclen = eth_hlen; sqbuf->tcphlen = sizeof(*tcph) + opts_len; sqbuf->scratch = cm_node; ethh = (struct ether_header *)buf; buf += eth_hlen; if (cm_node->do_lpb) sqbuf->do_lpb = true; if (cm_node->ipv4) { sqbuf->ipv4 = true; iph = (struct ip *)buf; buf += sizeof(*iph); tcph = (struct tcphdr *)buf; buf += sizeof(*tcph); ether_addr_copy(ethh->ether_dhost, cm_node->rem_mac); ether_addr_copy(ethh->ether_shost, cm_node->loc_mac); if (cm_node->vlan_id < VLAN_N_VID) { ((struct ether_vlan_header *)ethh)->evl_proto = htons(ETH_P_8021Q); vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) | cm_node->vlan_id; ((struct ether_vlan_header *)ethh)->evl_tag = htons(vtag); ((struct ether_vlan_header *)ethh)->evl_encap_proto = htons(ETH_P_IP); } else { ethh->ether_type = htons(ETH_P_IP); } iph->ip_v = IPVERSION; iph->ip_hl = 5; /* 5 * 4Byte words, IP headr len */ iph->ip_tos = cm_node->tos; iph->ip_len = htons(pktsize); iph->ip_id = htons(++cm_node->tcp_cntxt.loc_id); iph->ip_off = htons(0x4000); iph->ip_ttl = 0x40; iph->ip_p = IPPROTO_TCP; iph->ip_src.s_addr = htonl(cm_node->loc_addr[0]); iph->ip_dst.s_addr = htonl(cm_node->rem_addr[0]); } else { sqbuf->ipv4 = false; ip6h = (struct ip6_hdr *)buf; buf += sizeof(*ip6h); tcph = (struct tcphdr *)buf; buf += sizeof(*tcph); ether_addr_copy(ethh->ether_dhost, cm_node->rem_mac); ether_addr_copy(ethh->ether_shost, cm_node->loc_mac); if (cm_node->vlan_id < VLAN_N_VID) { ((struct ether_vlan_header *)ethh)->evl_proto = htons(ETH_P_8021Q); vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) | cm_node->vlan_id; ((struct ether_vlan_header *)ethh)->evl_tag = htons(vtag); ((struct ether_vlan_header *)ethh)->evl_encap_proto = htons(ETH_P_IPV6); } else { ethh->ether_type = htons(ETH_P_IPV6); } ip6h->ip6_vfc = 6 << 4; ip6h->ip6_vfc |= cm_node->tos >> 4; ip6h->ip6_flow = cm_node->tos << 20; ip6h->ip6_plen = htons(pktsize - sizeof(*ip6h)); ip6h->ip6_nxt = 6; ip6h->ip6_hops = 128; irdma_copy_ip_htonl(ip6h->ip6_src.__u6_addr.__u6_addr32, cm_node->loc_addr); irdma_copy_ip_htonl(ip6h->ip6_dst.__u6_addr.__u6_addr32, cm_node->rem_addr); } tcph->th_sport = htons(cm_node->loc_port); tcph->th_dport = htons(cm_node->rem_port); tcph->th_seq = htonl(cm_node->tcp_cntxt.loc_seq_num); if (flags & SET_ACK) { cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt; tcph->th_ack = htonl(cm_node->tcp_cntxt.loc_ack_num); tcph->th_flags |= TH_ACK; } else { tcph->th_ack = 0; } if (flags & SET_SYN) { cm_node->tcp_cntxt.loc_seq_num++; tcph->th_flags |= TH_SYN; } else { cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len; } if (flags & SET_FIN) { cm_node->tcp_cntxt.loc_seq_num++; tcph->th_flags |= TH_FIN; } if (flags & SET_RST) tcph->th_flags |= TH_RST; tcph->th_off = (u16)((sizeof(*tcph) + opts_len + 3) >> 2); sqbuf->tcphlen = tcph->th_off << 2; tcph->th_win = htons(cm_node->tcp_cntxt.rcv_wnd); tcph->th_urp = 0; if (opts_len) { memcpy(buf, options->addr, opts_len); buf += opts_len; } if (hdr_len) { memcpy(buf, hdr->addr, hdr_len); buf += hdr_len; } if (pdata && pdata->addr) memcpy(buf, pdata->addr, pdata->size); atomic_set(&sqbuf->refcount, 1); irdma_debug_buf(vsi->dev, IRDMA_DEBUG_ILQ, "TRANSMIT ILQ BUFFER", sqbuf->mem.va, sqbuf->totallen); return sqbuf; } /** * irdma_send_reset - Send RST packet * @cm_node: connection's node */ int irdma_send_reset(struct irdma_cm_node *cm_node) { struct irdma_puda_buf *sqbuf; int flags = SET_RST | SET_ACK; sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL, flags); if (!sqbuf) return -ENOMEM; irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "caller: %pS cm_node %p cm_id=%p accel=%d state=%d rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n", __builtin_return_address(0), cm_node, cm_node->cm_id, cm_node->accelerated, cm_node->state, cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr); return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 0, 1); } /** * irdma_active_open_err - send event for active side cm error * @cm_node: connection's node * @reset: Flag to send reset or not */ static void irdma_active_open_err(struct irdma_cm_node *cm_node, bool reset) { irdma_cleanup_retrans_entry(cm_node); cm_node->cm_core->stats_connect_errs++; if (reset) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "cm_node=%p state=%d\n", cm_node, cm_node->state); atomic_inc(&cm_node->refcnt); irdma_send_reset(cm_node); } cm_node->state = IRDMA_CM_STATE_CLOSED; irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED); } /** * irdma_passive_open_err - handle passive side cm error * @cm_node: connection's node * @reset: send reset or just free cm_node */ static void irdma_passive_open_err(struct irdma_cm_node *cm_node, bool reset) { irdma_cleanup_retrans_entry(cm_node); cm_node->cm_core->stats_passive_errs++; cm_node->state = IRDMA_CM_STATE_CLOSED; irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "cm_node=%p state =%d\n", cm_node, cm_node->state); if (reset) irdma_send_reset(cm_node); else irdma_rem_ref_cm_node(cm_node); } /** * irdma_event_connect_error - to create connect error event * @event: cm information for connect event */ static void irdma_event_connect_error(struct irdma_cm_event *event) { struct irdma_qp *iwqp; struct iw_cm_id *cm_id; cm_id = event->cm_node->cm_id; if (!cm_id) return; iwqp = cm_id->provider_data; if (!iwqp || !iwqp->iwdev) return; iwqp->cm_id = NULL; cm_id->provider_data = NULL; irdma_send_cm_event(event->cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, -ECONNRESET); irdma_rem_ref_cm_node(event->cm_node); } /** * irdma_process_options - process options from TCP header * @cm_node: connection's node * @optionsloc: point to start of options * @optionsize: size of all options * @syn_pkt: flag if syn packet */ static int irdma_process_options(struct irdma_cm_node *cm_node, u8 *optionsloc, u32 optionsize, u32 syn_pkt) { u32 tmp; u32 offset = 0; union all_known_options *all_options; char got_mss_option = 0; while (offset < optionsize) { all_options = (union all_known_options *)(optionsloc + offset); switch (all_options->base.optionnum) { case OPTION_NUM_EOL: offset = optionsize; break; case OPTION_NUM_NONE: offset += 1; continue; case OPTION_NUM_MSS: irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "MSS Length: %d Offset: %d Size: %d\n", all_options->mss.len, offset, optionsize); got_mss_option = 1; if (all_options->mss.len != 4) return -EINVAL; tmp = ntohs(all_options->mss.mss); if ((cm_node->ipv4 && (tmp + IRDMA_MTU_TO_MSS_IPV4) < IRDMA_MIN_MTU_IPV4) || (!cm_node->ipv4 && (tmp + IRDMA_MTU_TO_MSS_IPV6) < IRDMA_MIN_MTU_IPV6)) return -EINVAL; if (tmp < cm_node->tcp_cntxt.mss) cm_node->tcp_cntxt.mss = tmp; break; case OPTION_NUM_WINDOW_SCALE: cm_node->tcp_cntxt.snd_wscale = all_options->windowscale.shiftcount; break; default: irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "Unsupported TCP Option: %x\n", all_options->base.optionnum); break; } offset += all_options->base.len; } if (!got_mss_option && syn_pkt) cm_node->tcp_cntxt.mss = IRDMA_CM_DEFAULT_MSS; return 0; } /** * irdma_handle_tcp_options - setup TCP context info after parsing TCP options * @cm_node: connection's node * @tcph: pointer tcp header * @optionsize: size of options rcvd * @passive: active or passive flag */ static int irdma_handle_tcp_options(struct irdma_cm_node *cm_node, struct tcphdr *tcph, int optionsize, int passive) { u8 *optionsloc = (u8 *)&tcph[1]; int ret; if (optionsize) { ret = irdma_process_options(cm_node, optionsloc, optionsize, (u32)tcph->th_flags & TH_SYN); if (ret) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "Node %p, Sending Reset\n", cm_node); if (passive) irdma_passive_open_err(cm_node, true); else irdma_active_open_err(cm_node, true); return ret; } } cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->th_win) << cm_node->tcp_cntxt.snd_wscale; if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd) cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd; return 0; } /** * irdma_build_mpa_v1 - build a MPA V1 frame * @cm_node: connection's node * @start_addr: address where to build frame * @mpa_key: to do read0 or write0 */ static void irdma_build_mpa_v1(struct irdma_cm_node *cm_node, void *start_addr, u8 mpa_key) { struct ietf_mpa_v1 *mpa_frame = start_addr; switch (mpa_key) { case MPA_KEY_REQUEST: memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE); break; case MPA_KEY_REPLY: memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE); break; default: break; } mpa_frame->flags = IETF_MPA_FLAGS_CRC; mpa_frame->rev = cm_node->mpa_frame_rev; mpa_frame->priv_data_len = htons(cm_node->pdata.size); } /** * irdma_build_mpa_v2 - build a MPA V2 frame * @cm_node: connection's node * @start_addr: buffer start address * @mpa_key: to do read0 or write0 */ static void irdma_build_mpa_v2(struct irdma_cm_node *cm_node, void *start_addr, u8 mpa_key) { struct ietf_mpa_v2 *mpa_frame = start_addr; struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg; u16 ctrl_ird, ctrl_ord; /* initialize the upper 5 bytes of the frame */ irdma_build_mpa_v1(cm_node, start_addr, mpa_key); mpa_frame->flags |= IETF_MPA_V2_FLAG; if (cm_node->iwdev->iw_ooo) { mpa_frame->flags |= IETF_MPA_FLAGS_MARKERS; cm_node->rcv_mark_en = true; } mpa_frame->priv_data_len = cpu_to_be16(be16_to_cpu(mpa_frame->priv_data_len) + IETF_RTR_MSG_SIZE); /* initialize RTR msg */ if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) { ctrl_ird = IETF_NO_IRD_ORD; ctrl_ord = IETF_NO_IRD_ORD; } else { ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ? IETF_NO_IRD_ORD : cm_node->ird_size; ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ? IETF_NO_IRD_ORD : cm_node->ord_size; } ctrl_ird |= IETF_PEER_TO_PEER; switch (mpa_key) { case MPA_KEY_REQUEST: ctrl_ord |= IETF_RDMA0_WRITE; ctrl_ord |= IETF_RDMA0_READ; break; case MPA_KEY_REPLY: switch (cm_node->send_rdma0_op) { case SEND_RDMA_WRITE_ZERO: ctrl_ord |= IETF_RDMA0_WRITE; break; case SEND_RDMA_READ_ZERO: ctrl_ord |= IETF_RDMA0_READ; break; } break; default: break; } rtr_msg->ctrl_ird = htons(ctrl_ird); rtr_msg->ctrl_ord = htons(ctrl_ord); } /** * irdma_cm_build_mpa_frame - build mpa frame for mpa version 1 or version 2 * @cm_node: connection's node * @mpa: mpa: data buffer * @mpa_key: to do read0 or write0 */ static int irdma_cm_build_mpa_frame(struct irdma_cm_node *cm_node, struct irdma_kmem_info *mpa, u8 mpa_key) { int hdr_len = 0; switch (cm_node->mpa_frame_rev) { case IETF_MPA_V1: hdr_len = sizeof(struct ietf_mpa_v1); irdma_build_mpa_v1(cm_node, mpa->addr, mpa_key); break; case IETF_MPA_V2: hdr_len = sizeof(struct ietf_mpa_v2); irdma_build_mpa_v2(cm_node, mpa->addr, mpa_key); break; default: break; } return hdr_len; } /** * irdma_send_mpa_request - active node send mpa request to passive node * @cm_node: connection's node */ static int irdma_send_mpa_request(struct irdma_cm_node *cm_node) { struct irdma_puda_buf *sqbuf; cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame; cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node, &cm_node->mpa_hdr, MPA_KEY_REQUEST); if (!cm_node->mpa_hdr.size) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "mpa size = %d\n", cm_node->mpa_hdr.size); return -EINVAL; } sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, &cm_node->mpa_hdr, &cm_node->pdata, SET_ACK); if (!sqbuf) return -ENOMEM; return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1, 0); } /** * irdma_send_mpa_reject - * @cm_node: connection's node * @pdata: reject data for connection * @plen: length of reject data */ static int irdma_send_mpa_reject(struct irdma_cm_node *cm_node, const void *pdata, u8 plen) { struct irdma_puda_buf *sqbuf; struct irdma_mpa_priv_info priv_info; cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame; cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node, &cm_node->mpa_hdr, MPA_KEY_REPLY); cm_node->mpa_v2_frame.flags |= IETF_MPA_FLAGS_REJECT; priv_info.addr = pdata; priv_info.size = plen; sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, &cm_node->mpa_hdr, &priv_info, SET_ACK | SET_FIN); if (!sqbuf) return -ENOMEM; cm_node->state = IRDMA_CM_STATE_FIN_WAIT1; return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1, 0); } /** * irdma_negotiate_mpa_v2_ird_ord - negotiate MPAv2 IRD/ORD * @cm_node: connection's node * @buf: Data pointer */ static int irdma_negotiate_mpa_v2_ird_ord(struct irdma_cm_node *cm_node, u8 *buf) { struct ietf_mpa_v2 *mpa_v2_frame; struct ietf_rtr_msg *rtr_msg; u16 ird_size; u16 ord_size; u16 ctrl_ord; u16 ctrl_ird; mpa_v2_frame = (struct ietf_mpa_v2 *)buf; rtr_msg = &mpa_v2_frame->rtr_msg; /* parse rtr message */ ctrl_ord = ntohs(rtr_msg->ctrl_ord); ctrl_ird = ntohs(rtr_msg->ctrl_ird); ird_size = ctrl_ird & IETF_NO_IRD_ORD; ord_size = ctrl_ord & IETF_NO_IRD_ORD; if (!(ctrl_ird & IETF_PEER_TO_PEER)) return -EOPNOTSUPP; if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) { cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD; goto negotiate_done; } if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) { /* responder */ if (!ord_size && (ctrl_ord & IETF_RDMA0_READ)) cm_node->ird_size = 1; if (cm_node->ord_size > ird_size) cm_node->ord_size = ird_size; } else { /* initiator */ if (!ird_size && (ctrl_ord & IETF_RDMA0_READ)) /* Remote peer doesn't support RDMA0_READ */ return -EOPNOTSUPP; if (cm_node->ord_size > ird_size) cm_node->ord_size = ird_size; if (cm_node->ird_size < ord_size) /* no resources available */ return -EINVAL; } negotiate_done: if (ctrl_ord & IETF_RDMA0_READ) cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; else if (ctrl_ord & IETF_RDMA0_WRITE) cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO; else /* Not supported RDMA0 operation */ return -EOPNOTSUPP; irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "MPAV2 Negotiated ORD: %d, IRD: %d\n", cm_node->ord_size, cm_node->ird_size); return 0; } /** * irdma_parse_mpa - process an IETF MPA frame * @cm_node: connection's node * @buf: Data pointer * @type: to return accept or reject * @len: Len of mpa buffer */ static int irdma_parse_mpa(struct irdma_cm_node *cm_node, u8 *buf, u32 *type, u32 len) { struct ietf_mpa_v1 *mpa_frame; int mpa_hdr_len, priv_data_len, ret; *type = IRDMA_MPA_REQUEST_ACCEPT; if (len < sizeof(struct ietf_mpa_v1)) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "ietf buffer small (%x)\n", len); return -EINVAL; } mpa_frame = (struct ietf_mpa_v1 *)buf; mpa_hdr_len = sizeof(struct ietf_mpa_v1); priv_data_len = ntohs(mpa_frame->priv_data_len); if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "private_data too big %d\n", priv_data_len); return -EOVERFLOW; } if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "unsupported mpa rev = %d\n", mpa_frame->rev); return -EINVAL; } if (mpa_frame->rev > cm_node->mpa_frame_rev) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "rev %d\n", mpa_frame->rev); return -EINVAL; } cm_node->mpa_frame_rev = mpa_frame->rev; if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) { if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE)) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "Unexpected MPA Key received\n"); return -EINVAL; } } else { if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE)) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "Unexpected MPA Key received\n"); return -EINVAL; } } if (priv_data_len + mpa_hdr_len > len) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "ietf buffer len(%x + %x != %x)\n", priv_data_len, mpa_hdr_len, len); return -EOVERFLOW; } if (len > IRDMA_MAX_CM_BUF) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "ietf buffer large len = %d\n", len); return -EOVERFLOW; } switch (mpa_frame->rev) { case IETF_MPA_V2: mpa_hdr_len += IETF_RTR_MSG_SIZE; ret = irdma_negotiate_mpa_v2_ird_ord(cm_node, buf); if (ret) return ret; break; case IETF_MPA_V1: default: break; } memcpy(cm_node->pdata_buf, buf + mpa_hdr_len, priv_data_len); cm_node->pdata.size = priv_data_len; if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT) *type = IRDMA_MPA_REQUEST_REJECT; if (mpa_frame->flags & IETF_MPA_FLAGS_MARKERS) cm_node->snd_mark_en = true; return 0; } /** * irdma_schedule_cm_timer * @cm_node: connection's node * @sqbuf: buffer to send * @type: if it is send or close * @send_retrans: if rexmits to be done * @close_when_complete: is cm_node to be removed * * note - cm_node needs to be protected before calling this. Encase in: * irdma_rem_ref_cm_node(cm_core, cm_node); * irdma_schedule_cm_timer(...) * atomic_inc(&cm_node->refcnt); */ int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node, struct irdma_puda_buf *sqbuf, enum irdma_timer_type type, int send_retrans, int close_when_complete) { struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi; struct irdma_cm_core *cm_core = cm_node->cm_core; struct irdma_timer_entry *new_send; u32 was_timer_set; unsigned long flags; new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC); if (!new_send) { if (type != IRDMA_TIMER_TYPE_CLOSE) irdma_free_sqbuf(vsi, sqbuf); return -ENOMEM; } new_send->retrycount = IRDMA_DEFAULT_RETRYS; new_send->retranscount = IRDMA_DEFAULT_RETRANS; new_send->sqbuf = sqbuf; new_send->timetosend = jiffies; new_send->type = type; new_send->send_retrans = send_retrans; new_send->close_when_complete = close_when_complete; if (type == IRDMA_TIMER_TYPE_CLOSE) { new_send->timetosend += (HZ / 10); if (cm_node->close_entry) { kfree(new_send); irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "already close entry\n"); return -EINVAL; } cm_node->close_entry = new_send; } else { /* type == IRDMA_TIMER_TYPE_SEND */ spin_lock_irqsave(&cm_node->retrans_list_lock, flags); cm_node->send_entry = new_send; atomic_inc(&cm_node->refcnt); spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); new_send->timetosend = jiffies + IRDMA_RETRY_TIMEOUT; atomic_inc(&sqbuf->refcount); irdma_puda_send_buf(vsi->ilq, sqbuf); if (!send_retrans) { irdma_cleanup_retrans_entry(cm_node); if (close_when_complete) irdma_rem_ref_cm_node(cm_node); return 0; } } spin_lock_irqsave(&cm_core->ht_lock, flags); was_timer_set = timer_pending(&cm_core->tcp_timer); if (!was_timer_set) { cm_core->tcp_timer.expires = new_send->timetosend; add_timer(&cm_core->tcp_timer); } spin_unlock_irqrestore(&cm_core->ht_lock, flags); return 0; } /** * irdma_retrans_expired - Could not rexmit the packet * @cm_node: connection's node */ static void irdma_retrans_expired(struct irdma_cm_node *cm_node) { enum irdma_cm_node_state state = cm_node->state; cm_node->state = IRDMA_CM_STATE_CLOSED; switch (state) { case IRDMA_CM_STATE_SYN_RCVD: case IRDMA_CM_STATE_CLOSING: irdma_rem_ref_cm_node(cm_node); break; case IRDMA_CM_STATE_FIN_WAIT1: case IRDMA_CM_STATE_LAST_ACK: irdma_send_reset(cm_node); break; default: atomic_inc(&cm_node->refcnt); irdma_send_reset(cm_node); irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED); break; } } /** * irdma_handle_close_entry - for handling retry/timeouts * @cm_node: connection's node * @rem_node: flag for remove cm_node */ static void irdma_handle_close_entry(struct irdma_cm_node *cm_node, u32 rem_node) { struct irdma_timer_entry *close_entry = cm_node->close_entry; struct irdma_qp *iwqp; unsigned long flags; if (!close_entry) return; iwqp = (struct irdma_qp *)close_entry->sqbuf; if (iwqp) { spin_lock_irqsave(&iwqp->lock, flags); if (iwqp->cm_id) { iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED; iwqp->hw_iwarp_state = IRDMA_QP_STATE_ERROR; iwqp->last_aeq = IRDMA_AE_RESET_SENT; iwqp->ibqp_state = IB_QPS_ERR; spin_unlock_irqrestore(&iwqp->lock, flags); irdma_cm_disconn(iwqp); } else { spin_unlock_irqrestore(&iwqp->lock, flags); } } else if (rem_node) { /* TIME_WAIT state */ irdma_rem_ref_cm_node(cm_node); } kfree(close_entry); cm_node->close_entry = NULL; } /** * irdma_cm_timer_tick - system's timer expired callback * @t: Pointer to timer_list */ static void irdma_cm_timer_tick(struct timer_list *t) { unsigned long nexttimeout = jiffies + IRDMA_LONG_TIME; struct irdma_cm_node *cm_node; struct irdma_timer_entry *send_entry, *close_entry; struct list_head *list_core_temp; struct list_head *list_node; struct irdma_cm_core *cm_core = from_timer(cm_core, t, tcp_timer); struct irdma_sc_vsi *vsi; u32 settimer = 0; unsigned long timetosend; unsigned long flags; struct list_head timer_list; INIT_LIST_HEAD(&timer_list); rcu_read_lock(); irdma_timer_list_prep(cm_core, &timer_list); rcu_read_unlock(); list_for_each_safe(list_node, list_core_temp, &timer_list) { cm_node = container_of(list_node, struct irdma_cm_node, timer_entry); close_entry = cm_node->close_entry; if (close_entry) { if (time_after(close_entry->timetosend, jiffies)) { if (nexttimeout > close_entry->timetosend || !settimer) { nexttimeout = close_entry->timetosend; settimer = 1; } } else { irdma_handle_close_entry(cm_node, 1); } } spin_lock_irqsave(&cm_node->retrans_list_lock, flags); send_entry = cm_node->send_entry; if (!send_entry) goto done; if (time_after(send_entry->timetosend, jiffies)) { if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) { if (nexttimeout > send_entry->timetosend || !settimer) { nexttimeout = send_entry->timetosend; settimer = 1; } } else { irdma_free_retrans_entry(cm_node); } goto done; } if (cm_node->state == IRDMA_CM_STATE_OFFLOADED || cm_node->state == IRDMA_CM_STATE_CLOSED) { irdma_free_retrans_entry(cm_node); goto done; } if (!send_entry->retranscount || !send_entry->retrycount) { irdma_free_retrans_entry(cm_node); spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); irdma_retrans_expired(cm_node); cm_node->state = IRDMA_CM_STATE_CLOSED; spin_lock_irqsave(&cm_node->retrans_list_lock, flags); goto done; } spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); vsi = &cm_node->iwdev->vsi; if (!cm_node->ack_rcvd) { atomic_inc(&send_entry->sqbuf->refcount); irdma_puda_send_buf(vsi->ilq, send_entry->sqbuf); cm_node->cm_core->stats_pkt_retrans++; } spin_lock_irqsave(&cm_node->retrans_list_lock, flags); if (send_entry->send_retrans) { send_entry->retranscount--; timetosend = (IRDMA_RETRY_TIMEOUT << (IRDMA_DEFAULT_RETRANS - send_entry->retranscount)); send_entry->timetosend = jiffies + min(timetosend, IRDMA_MAX_TIMEOUT); if (nexttimeout > send_entry->timetosend || !settimer) { nexttimeout = send_entry->timetosend; settimer = 1; } } else { int close_when_complete; close_when_complete = send_entry->close_when_complete; irdma_free_retrans_entry(cm_node); if (close_when_complete) irdma_rem_ref_cm_node(cm_node); } done: spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); irdma_rem_ref_cm_node(cm_node); } if (settimer) { spin_lock_irqsave(&cm_core->ht_lock, flags); if (!timer_pending(&cm_core->tcp_timer)) { cm_core->tcp_timer.expires = nexttimeout; add_timer(&cm_core->tcp_timer); } spin_unlock_irqrestore(&cm_core->ht_lock, flags); } } /** * irdma_send_syn - send SYN packet * @cm_node: connection's node * @sendack: flag to set ACK bit or not */ int irdma_send_syn(struct irdma_cm_node *cm_node, u32 sendack) { struct irdma_puda_buf *sqbuf; int flags = SET_SYN; char optionsbuf[sizeof(struct option_mss) + sizeof(struct option_windowscale) + sizeof(struct option_base) + TCP_OPTIONS_PADDING]; struct irdma_kmem_info opts; int optionssize = 0; /* Sending MSS option */ union all_known_options *options; opts.addr = optionsbuf; if (!cm_node) return -EINVAL; options = (union all_known_options *)&optionsbuf[optionssize]; options->mss.optionnum = OPTION_NUM_MSS; options->mss.len = sizeof(struct option_mss); options->mss.mss = htons(cm_node->tcp_cntxt.mss); optionssize += sizeof(struct option_mss); options = (union all_known_options *)&optionsbuf[optionssize]; options->windowscale.optionnum = OPTION_NUM_WINDOW_SCALE; options->windowscale.len = sizeof(struct option_windowscale); options->windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale; optionssize += sizeof(struct option_windowscale); options = (union all_known_options *)&optionsbuf[optionssize]; options->eol = OPTION_NUM_EOL; optionssize += 1; if (sendack) flags |= SET_ACK; opts.size = optionssize; sqbuf = cm_node->cm_core->form_cm_frame(cm_node, &opts, NULL, NULL, flags); if (!sqbuf) return -ENOMEM; return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1, 0); } /** * irdma_send_ack - Send ACK packet * @cm_node: connection's node */ void irdma_send_ack(struct irdma_cm_node *cm_node) { struct irdma_puda_buf *sqbuf; struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi; sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL, SET_ACK); if (sqbuf) irdma_puda_send_buf(vsi->ilq, sqbuf); } /** * irdma_send_fin - Send FIN pkt * @cm_node: connection's node */ static int irdma_send_fin(struct irdma_cm_node *cm_node) { struct irdma_puda_buf *sqbuf; sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL, SET_ACK | SET_FIN); if (!sqbuf) return -ENOMEM; return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1, 0); } /** * irdma_find_listener - find a cm node listening on this addr-port pair * @cm_core: cm's core * @dst_addr: listener ip addr * @dst_port: listener tcp port num * @vlan_id: virtual LAN ID * @listener_state: state to match with listen node's */ static struct irdma_cm_listener * irdma_find_listener(struct irdma_cm_core *cm_core, u32 *dst_addr, u16 dst_port, u16 vlan_id, enum irdma_cm_listener_state listener_state) { struct irdma_cm_listener *listen_node; static const u32 ip_zero[4] = {0, 0, 0, 0}; u32 listen_addr[4]; u16 listen_port; unsigned long flags; /* walk list and find cm_node associated with this session ID */ spin_lock_irqsave(&cm_core->listen_list_lock, flags); list_for_each_entry(listen_node, &cm_core->listen_list, list) { memcpy(listen_addr, listen_node->loc_addr, sizeof(listen_addr)); listen_port = listen_node->loc_port; if (listen_port != dst_port || !(listener_state & listen_node->listener_state)) continue; /* compare node pair, return node handle if a match */ if (!memcmp(listen_addr, ip_zero, sizeof(listen_addr)) || (!memcmp(listen_addr, dst_addr, sizeof(listen_addr)) && vlan_id == listen_node->vlan_id)) { atomic_inc(&listen_node->refcnt); spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); return listen_node; } } spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); return NULL; } /** * irdma_del_multiple_qhash - Remove qhash and child listens * @iwdev: iWarp device * @cm_info: CM info for parent listen node * @cm_parent_listen_node: The parent listen node */ static int irdma_del_multiple_qhash(struct irdma_device *iwdev, struct irdma_cm_info *cm_info, struct irdma_cm_listener *cm_parent_listen_node) { struct irdma_cm_listener *child_listen_node; struct list_head *pos, *tpos; unsigned long flags; int ret = -EINVAL; spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags); list_for_each_safe(pos, tpos, &cm_parent_listen_node->child_listen_list) { child_listen_node = list_entry(pos, struct irdma_cm_listener, child_listen_list); if (child_listen_node->ipv4) irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "removing child listen for IP=%pI4, port=%d, vlan=%d\n", child_listen_node->loc_addr, child_listen_node->loc_port, child_listen_node->vlan_id); else irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "removing child listen for IP=%pI6, port=%d, vlan=%d\n", child_listen_node->loc_addr, child_listen_node->loc_port, child_listen_node->vlan_id); list_del(pos); memcpy(cm_info->loc_addr, child_listen_node->loc_addr, sizeof(cm_info->loc_addr)); cm_info->vlan_id = child_listen_node->vlan_id; if (child_listen_node->qhash_set) { ret = irdma_manage_qhash(iwdev, cm_info, IRDMA_QHASH_TYPE_TCP_SYN, IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL, false); child_listen_node->qhash_set = false; } else { ret = 0; } irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "Child listen node freed = %p\n", child_listen_node); kfree(child_listen_node); cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++; } spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags); return ret; } /** * irdma_netdev_vlan_ipv6 - Gets the netdev and mac * @addr: local IPv6 address * @vlan_id: vlan id for the given IPv6 address * @mac: mac address for the given IPv6 address * * Returns the net_device of the IPv6 address and also sets the * vlan id and mac for that address. */ struct ifnet * irdma_netdev_vlan_ipv6(u32 *addr, u16 *vlan_id, u8 *mac) { struct ifnet *ip_dev = NULL; struct in6_addr laddr6; irdma_copy_ip_htonl(laddr6.__u6_addr.__u6_addr32, addr); if (vlan_id) *vlan_id = 0xFFFF; /* Match rdma_vlan_dev_vlan_id() */ if (mac) eth_zero_addr(mac); ip_dev = ip6_ifp_find(&init_net, laddr6, 0); if (ip_dev) { if (vlan_id) *vlan_id = rdma_vlan_dev_vlan_id(ip_dev); if (ip_dev->if_addr && ip_dev->if_addr->ifa_addr && mac) ether_addr_copy(mac, IF_LLADDR(ip_dev)); } return ip_dev; } /** * irdma_get_vlan_ipv4 - Returns the vlan_id for IPv4 address * @addr: local IPv4 address */ u16 irdma_get_vlan_ipv4(u32 *addr) { struct ifnet *netdev; u16 vlan_id = 0xFFFF; netdev = ip_ifp_find(&init_net, htonl(addr[0])); if (netdev) { vlan_id = rdma_vlan_dev_vlan_id(netdev); dev_put(netdev); } return vlan_id; } /** * irdma_add_mqh_6 - Adds multiple qhashes for IPv6 * @iwdev: iWarp device * @cm_info: CM info for parent listen node * @cm_parent_listen_node: The parent listen node * * Adds a qhash and a child listen node for every IPv6 address * on the adapter and adds the associated qhash filter */ static int irdma_add_mqh_6(struct irdma_device *iwdev, struct irdma_cm_info *cm_info, struct irdma_cm_listener *cm_parent_listen_node) { struct ifnet *ip_dev; struct ifaddr *ifp; struct irdma_cm_listener *child_listen_node; unsigned long flags; int ret = 0; IFNET_RLOCK(); IRDMA_TAILQ_FOREACH((ip_dev), &V_ifnet, if_link) { if (!(ip_dev->if_flags & IFF_UP)) continue; if (((rdma_vlan_dev_vlan_id(ip_dev) >= VLAN_N_VID) || (rdma_vlan_dev_real_dev(ip_dev) != iwdev->netdev)) && ip_dev != iwdev->netdev) continue; if_addr_rlock(ip_dev); IRDMA_TAILQ_FOREACH(ifp, &ip_dev->if_addrhead, ifa_link) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "IP=%pI6, vlan_id=%d, MAC=%pM\n", &((struct sockaddr_in6 *)ifp->ifa_addr)->sin6_addr, rdma_vlan_dev_vlan_id(ip_dev), IF_LLADDR(ip_dev)); if (((struct sockaddr_in6 *)ifp->ifa_addr)->sin6_family != AF_INET6) continue; child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL); irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "Allocating child listener %p\n", child_listen_node); if (!child_listen_node) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "listener memory allocation\n"); ret = -ENOMEM; if_addr_runlock(ip_dev); goto exit; } memcpy(child_listen_node, cm_parent_listen_node, sizeof(*child_listen_node)); cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev); child_listen_node->vlan_id = cm_info->vlan_id; irdma_copy_ip_ntohl(child_listen_node->loc_addr, ((struct sockaddr_in6 *)ifp->ifa_addr)->sin6_addr.__u6_addr.__u6_addr32); memcpy(cm_info->loc_addr, child_listen_node->loc_addr, sizeof(cm_info->loc_addr)); ret = irdma_manage_qhash(iwdev, cm_info, IRDMA_QHASH_TYPE_TCP_SYN, IRDMA_QHASH_MANAGE_TYPE_ADD, NULL, true); if (ret) { kfree(child_listen_node); continue; } child_listen_node->qhash_set = true; spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags); list_add(&child_listen_node->child_listen_list, &cm_parent_listen_node->child_listen_list); spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags); cm_parent_listen_node->cm_core->stats_listen_nodes_created++; } if_addr_runlock(ip_dev); } exit: IFNET_RUNLOCK(); return ret; } /** * irdma_add_mqh_4 - Adds multiple qhashes for IPv4 * @iwdev: iWarp device * @cm_info: CM info for parent listen node * @cm_parent_listen_node: The parent listen node * * Adds a qhash and a child listen node for every IPv4 address * on the adapter and adds the associated qhash filter */ static int irdma_add_mqh_4(struct irdma_device *iwdev, struct irdma_cm_info *cm_info, struct irdma_cm_listener *cm_parent_listen_node) { struct ifnet *ip_dev; struct irdma_cm_listener *child_listen_node; unsigned long flags; struct ifaddr *ifa; int ret = 0; IFNET_RLOCK(); IRDMA_TAILQ_FOREACH((ip_dev), &V_ifnet, if_link) { if (!(ip_dev->if_flags & IFF_UP)) continue; if (((rdma_vlan_dev_vlan_id(ip_dev) >= VLAN_N_VID) || (rdma_vlan_dev_real_dev(ip_dev) != iwdev->netdev)) && ip_dev != iwdev->netdev) continue; if_addr_rlock(ip_dev); IRDMA_TAILQ_FOREACH(ifa, &ip_dev->if_addrhead, ifa_link) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n", &ifa->ifa_addr, rdma_vlan_dev_vlan_id(ip_dev), IF_LLADDR(ip_dev)); if (((struct sockaddr_in *)ifa->ifa_addr)->sin_family != AF_INET) continue; child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL); cm_parent_listen_node->cm_core->stats_listen_nodes_created++; irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "Allocating child listener %p\n", child_listen_node); if (!child_listen_node) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "listener memory allocation\n"); if_addr_runlock(ip_dev); ret = -ENOMEM; goto exit; } memcpy(child_listen_node, cm_parent_listen_node, sizeof(*child_listen_node)); child_listen_node->vlan_id = rdma_vlan_dev_vlan_id(ip_dev); cm_info->vlan_id = child_listen_node->vlan_id; child_listen_node->loc_addr[0] = ntohl(((struct sockaddr_in *)ifa->ifa_addr)->sin_addr.s_addr); memcpy(cm_info->loc_addr, child_listen_node->loc_addr, sizeof(cm_info->loc_addr)); ret = irdma_manage_qhash(iwdev, cm_info, IRDMA_QHASH_TYPE_TCP_SYN, IRDMA_QHASH_MANAGE_TYPE_ADD, NULL, true); if (ret) { kfree(child_listen_node); cm_parent_listen_node->cm_core ->stats_listen_nodes_created--; continue; } child_listen_node->qhash_set = true; spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags); list_add(&child_listen_node->child_listen_list, &cm_parent_listen_node->child_listen_list); spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags); } if_addr_runlock(ip_dev); } exit: IFNET_RUNLOCK(); return ret; } /** * irdma_add_mqh - Adds multiple qhashes * @iwdev: iWarp device * @cm_info: CM info for parent listen node * @cm_listen_node: The parent listen node */ static int irdma_add_mqh(struct irdma_device *iwdev, struct irdma_cm_info *cm_info, struct irdma_cm_listener *cm_listen_node) { int err; VNET_ITERATOR_DECL(vnet_iter); VNET_LIST_RLOCK(); VNET_FOREACH(vnet_iter) { IFNET_RLOCK(); CURVNET_SET_QUIET(vnet_iter); if (cm_info->ipv4) err = irdma_add_mqh_4(iwdev, cm_info, cm_listen_node); else err = irdma_add_mqh_6(iwdev, cm_info, cm_listen_node); CURVNET_RESTORE(); IFNET_RUNLOCK(); } VNET_LIST_RUNLOCK(); return err; } /** * irdma_reset_list_prep - add connection nodes slated for reset to list * @cm_core: cm's core * @listener: pointer to listener node * @reset_list: a list to which cm_node will be selected */ static void irdma_reset_list_prep(struct irdma_cm_core *cm_core, struct irdma_cm_listener *listener, struct list_head *reset_list) { struct irdma_cm_node *cm_node; int bkt; HASH_FOR_EACH_RCU(cm_core->cm_hash_tbl, bkt, cm_node, list) { if (cm_node->listener == listener && !cm_node->accelerated && atomic_inc_not_zero(&cm_node->refcnt)) list_add(&cm_node->reset_entry, reset_list); } } /** * irdma_dec_refcnt_listen - delete listener and associated cm nodes * @cm_core: cm's core * @listener: pointer to listener node * @free_hanging_nodes: to free associated cm_nodes * @apbvt_del: flag to delete the apbvt */ static int irdma_dec_refcnt_listen(struct irdma_cm_core *cm_core, struct irdma_cm_listener *listener, int free_hanging_nodes, bool apbvt_del) { struct list_head *list_pos; struct list_head *list_temp; struct irdma_cm_node *cm_node; struct list_head reset_list; struct irdma_cm_info nfo; enum irdma_cm_node_state old_state; unsigned long flags; int err; /* free non-accelerated child nodes for this listener */ INIT_LIST_HEAD(&reset_list); if (free_hanging_nodes) { rcu_read_lock(); irdma_reset_list_prep(cm_core, listener, &reset_list); rcu_read_unlock(); } list_for_each_safe(list_pos, list_temp, &reset_list) { cm_node = container_of(list_pos, struct irdma_cm_node, reset_entry); if (cm_node->state >= IRDMA_CM_STATE_FIN_WAIT1) { irdma_rem_ref_cm_node(cm_node); continue; } irdma_cleanup_retrans_entry(cm_node); err = irdma_send_reset(cm_node); if (err) { cm_node->state = IRDMA_CM_STATE_CLOSED; irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "send reset failed\n"); } else { old_state = cm_node->state; cm_node->state = IRDMA_CM_STATE_LISTENER_DESTROYED; if (old_state != IRDMA_CM_STATE_MPAREQ_RCVD) irdma_rem_ref_cm_node(cm_node); } } if (atomic_dec_and_test(&listener->refcnt)) { spin_lock_irqsave(&cm_core->listen_list_lock, flags); list_del(&listener->list); spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); if (apbvt_del) irdma_del_apbvt(listener->iwdev, listener->apbvt_entry); memcpy(nfo.loc_addr, listener->loc_addr, sizeof(nfo.loc_addr)); nfo.loc_port = listener->loc_port; nfo.ipv4 = listener->ipv4; nfo.vlan_id = listener->vlan_id; nfo.user_pri = listener->user_pri; nfo.qh_qpid = listener->iwdev->vsi.ilq->qp_id; if (!list_empty(&listener->child_listen_list)) { irdma_del_multiple_qhash(listener->iwdev, &nfo, listener); } else { if (listener->qhash_set) irdma_manage_qhash(listener->iwdev, &nfo, IRDMA_QHASH_TYPE_TCP_SYN, IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL, false); } cm_core->stats_listen_destroyed++; cm_core->stats_listen_nodes_destroyed++; irdma_debug(iwdev_to_idev(listener->iwdev), IRDMA_DEBUG_CM, "loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n", listener->loc_port, listener->loc_addr, listener, listener->cm_id, listener->qhash_set, listener->vlan_id, apbvt_del); kfree(listener); listener = NULL; return 0; } return -EINVAL; } /** * irdma_cm_del_listen - delete a listener * @cm_core: cm's core * @listener: passive connection's listener * @apbvt_del: flag to delete apbvt */ static int irdma_cm_del_listen(struct irdma_cm_core *cm_core, struct irdma_cm_listener *listener, bool apbvt_del) { listener->listener_state = IRDMA_CM_LISTENER_PASSIVE_STATE; listener->cm_id = NULL; return irdma_dec_refcnt_listen(cm_core, listener, 1, apbvt_del); } /** * irdma_find_node - find a cm node that matches the reference cm node * @cm_core: cm's core * @rem_port: remote tcp port num * @rem_addr: remote ip addr * @loc_port: local tcp port num * @loc_addr: local ip addr * @vlan_id: local VLAN ID */ struct irdma_cm_node * irdma_find_node(struct irdma_cm_core *cm_core, u16 rem_port, u32 *rem_addr, u16 loc_port, u32 *loc_addr, u16 vlan_id) { struct irdma_cm_node *cm_node; u32 key = (rem_port << 16) | loc_port; rcu_read_lock(); HASH_FOR_EACH_POSSIBLE_RCU(cm_core->cm_hash_tbl, cm_node, list, key) { if (cm_node->vlan_id == vlan_id && cm_node->loc_port == loc_port && cm_node->rem_port == rem_port && !memcmp(cm_node->loc_addr, loc_addr, sizeof(cm_node->loc_addr)) && !memcmp(cm_node->rem_addr, rem_addr, sizeof(cm_node->rem_addr))) { if (!atomic_inc_not_zero(&cm_node->refcnt)) goto exit; rcu_read_unlock(); return cm_node; } } exit: rcu_read_unlock(); /* no owner node */ return NULL; } /** * irdma_add_hte_node - add a cm node to the hash table * @cm_core: cm's core * @cm_node: connection's node */ static void irdma_add_hte_node(struct irdma_cm_core *cm_core, struct irdma_cm_node *cm_node) { unsigned long flags; u32 key = (cm_node->rem_port << 16) | cm_node->loc_port; spin_lock_irqsave(&cm_core->ht_lock, flags); HASH_ADD_RCU(cm_core->cm_hash_tbl, &cm_node->list, key); spin_unlock_irqrestore(&cm_core->ht_lock, flags); } /** * irdma_ipv4_is_lpb - check if loopback * @loc_addr: local addr to compare * @rem_addr: remote address */ bool -irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr) +irdma_ipv4_is_lpb(struct vnet *vnet, u32 loc_addr, u32 rem_addr) { - return ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr); + bool ret; + + CURVNET_SET_QUIET(vnet); + ret = ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr); + CURVNET_RESTORE(); + + return (ret); } /** * irdma_ipv6_is_lpb - check if loopback * @loc_addr: local addr to compare * @rem_addr: remote address */ bool irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr) { struct in6_addr raddr6; irdma_copy_ip_htonl(raddr6.__u6_addr.__u6_addr32, rem_addr); return !memcmp(loc_addr, rem_addr, 16) || ipv6_addr_loopback(&raddr6); } /** * irdma_cm_create_ah - create a cm address handle * @cm_node: The connection manager node to create AH for * @wait: Provides option to wait for ah creation or not */ static int irdma_cm_create_ah(struct irdma_cm_node *cm_node, bool wait) { + struct rdma_cm_id *rdma_id = (struct rdma_cm_id *)cm_node->cm_id->context; + struct vnet *vnet = rdma_id->route.addr.dev_addr.net; struct irdma_ah_info ah_info = {0}; struct irdma_device *iwdev = cm_node->iwdev; ether_addr_copy(ah_info.mac_addr, IF_LLADDR(iwdev->netdev)); ah_info.hop_ttl = 0x40; ah_info.tc_tos = cm_node->tos; ah_info.vsi = &iwdev->vsi; if (cm_node->ipv4) { ah_info.ipv4_valid = true; ah_info.dest_ip_addr[0] = cm_node->rem_addr[0]; ah_info.src_ip_addr[0] = cm_node->loc_addr[0]; - ah_info.do_lpbk = irdma_ipv4_is_lpb(ah_info.src_ip_addr[0], + ah_info.do_lpbk = irdma_ipv4_is_lpb(vnet, + ah_info.src_ip_addr[0], ah_info.dest_ip_addr[0]); } else { memcpy(ah_info.dest_ip_addr, cm_node->rem_addr, sizeof(ah_info.dest_ip_addr)); memcpy(ah_info.src_ip_addr, cm_node->loc_addr, sizeof(ah_info.src_ip_addr)); ah_info.do_lpbk = irdma_ipv6_is_lpb(ah_info.src_ip_addr, ah_info.dest_ip_addr); } ah_info.vlan_tag = cm_node->vlan_id; if (cm_node->vlan_id < VLAN_N_VID) { ah_info.insert_vlan_tag = 1; ah_info.vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT; } ah_info.dst_arpindex = irdma_arp_table(iwdev->rf, ah_info.dest_ip_addr, NULL, IRDMA_ARP_RESOLVE); if (irdma_puda_create_ah(&iwdev->rf->sc_dev, &ah_info, wait, IRDMA_PUDA_RSRC_TYPE_ILQ, cm_node, &cm_node->ah)) return -ENOMEM; return 0; } /** * irdma_cm_free_ah - free a cm address handle * @cm_node: The connection manager node to create AH for */ static void irdma_cm_free_ah(struct irdma_cm_node *cm_node) { struct irdma_device *iwdev = cm_node->iwdev; irdma_puda_free_ah(&iwdev->rf->sc_dev, cm_node->ah); cm_node->ah = NULL; } /** * irdma_make_cm_node - create a new instance of a cm node * @cm_core: cm's core * @iwdev: iwarp device structure * @cm_info: quad info for connection * @listener: passive connection's listener */ static struct irdma_cm_node * irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, struct irdma_cm_info *cm_info, struct irdma_cm_listener *listener) { struct irdma_cm_node *cm_node; int arpindex; struct ifnet *netdev = iwdev->netdev; /* create an hte and cm_node for this instance */ cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC); if (!cm_node) return NULL; /* set our node specific transport info */ cm_node->ipv4 = cm_info->ipv4; cm_node->vlan_id = cm_info->vlan_id; if (cm_node->vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode) cm_node->vlan_id = 0; cm_node->tos = cm_info->tos; cm_node->user_pri = cm_info->user_pri; if (listener) { if (listener->tos != cm_info->tos) irdma_dev_warn( &iwdev->rf->sc_dev, "application TOS[%d] and remote client TOS[%d] mismatch\n", listener->tos, cm_info->tos); if (iwdev->vsi.dscp_mode) { cm_node->user_pri = listener->user_pri; } else { cm_node->tos = max(listener->tos, cm_info->tos); cm_node->user_pri = rt_tos2priority(cm_node->tos); } irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_DCB, "listener: TOS:[%d] UP:[%d]\n", cm_node->tos, cm_node->user_pri); } memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr)); memcpy(cm_node->rem_addr, cm_info->rem_addr, sizeof(cm_node->rem_addr)); cm_node->loc_port = cm_info->loc_port; cm_node->rem_port = cm_info->rem_port; cm_node->mpa_frame_rev = IRDMA_CM_DEFAULT_MPA_VER; cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; cm_node->iwdev = iwdev; cm_node->dev = &iwdev->rf->sc_dev; cm_node->ird_size = cm_node->dev->hw_attrs.max_hw_ird; cm_node->ord_size = cm_node->dev->hw_attrs.max_hw_ord; cm_node->listener = listener; cm_node->cm_id = cm_info->cm_id; ether_addr_copy(cm_node->loc_mac, IF_LLADDR(netdev)); spin_lock_init(&cm_node->retrans_list_lock); cm_node->ack_rcvd = false; init_completion(&cm_node->establish_comp); atomic_set(&cm_node->refcnt, 1); /* associate our parent CM core */ cm_node->cm_core = cm_core; cm_node->tcp_cntxt.loc_id = IRDMA_CM_DEFAULT_LOCAL_ID; cm_node->tcp_cntxt.rcv_wscale = iwdev->rcv_wscale; cm_node->tcp_cntxt.rcv_wnd = iwdev->rcv_wnd >> cm_node->tcp_cntxt.rcv_wscale; kc_set_loc_seq_num_mss(cm_node); arpindex = irdma_resolve_neigh_lpb_chk(iwdev, cm_node, cm_info); if (arpindex < 0) goto err; ether_addr_copy(cm_node->rem_mac, iwdev->rf->arp_table[arpindex].mac_addr); irdma_add_hte_node(cm_core, cm_node); cm_core->stats_nodes_created++; return cm_node; err: kfree(cm_node); return NULL; } static void irdma_cm_node_free_cb(struct rcu_head *rcu_head) { struct irdma_cm_node *cm_node = container_of(rcu_head, struct irdma_cm_node, rcu_head); struct irdma_cm_core *cm_core = cm_node->cm_core; struct irdma_qp *iwqp; struct irdma_cm_info nfo; /* if the node is destroyed before connection was accelerated */ if (!cm_node->accelerated && cm_node->accept_pend) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "node destroyed before established\n"); atomic_dec(&cm_node->listener->pend_accepts_cnt); } if (cm_node->close_entry) irdma_handle_close_entry(cm_node, 0); if (cm_node->listener) { irdma_dec_refcnt_listen(cm_core, cm_node->listener, 0, true); } else { if (cm_node->apbvt_set) { irdma_del_apbvt(cm_node->iwdev, cm_node->apbvt_entry); cm_node->apbvt_set = 0; } irdma_get_addr_info(cm_node, &nfo); if (cm_node->qhash_set) { nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id; irdma_manage_qhash(cm_node->iwdev, &nfo, IRDMA_QHASH_TYPE_TCP_ESTABLISHED, IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL, false); cm_node->qhash_set = 0; } } iwqp = cm_node->iwqp; if (iwqp) { cm_node->cm_id->rem_ref(cm_node->cm_id); cm_node->cm_id = NULL; iwqp->cm_id = NULL; irdma_qp_rem_ref(&iwqp->ibqp); cm_node->iwqp = NULL; } else if (cm_node->qhash_set) { irdma_get_addr_info(cm_node, &nfo); nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id; irdma_manage_qhash(cm_node->iwdev, &nfo, IRDMA_QHASH_TYPE_TCP_ESTABLISHED, IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL, false); cm_node->qhash_set = 0; } cm_core->cm_free_ah(cm_node); kfree(cm_node); } /** * irdma_rem_ref_cm_node - destroy an instance of a cm node * @cm_node: connection's node */ void irdma_rem_ref_cm_node(struct irdma_cm_node *cm_node) { struct irdma_cm_core *cm_core = cm_node->cm_core; unsigned long flags; spin_lock_irqsave(&cm_core->ht_lock, flags); if (!atomic_dec_and_test(&cm_node->refcnt)) { spin_unlock_irqrestore(&cm_core->ht_lock, flags); return; } if (cm_node->iwqp) { cm_node->iwqp->cm_node = NULL; cm_node->iwqp->cm_id = NULL; } HASH_DEL_RCU(cm_core->cm_hash_tbl, &cm_node->list); cm_node->cm_core->stats_nodes_destroyed++; spin_unlock_irqrestore(&cm_core->ht_lock, flags); /* wait for all list walkers to exit their grace period */ call_rcu(&cm_node->rcu_head, irdma_cm_node_free_cb); } /** * irdma_handle_fin_pkt - FIN packet received * @cm_node: connection's node */ static void irdma_handle_fin_pkt(struct irdma_cm_node *cm_node) { switch (cm_node->state) { case IRDMA_CM_STATE_SYN_RCVD: case IRDMA_CM_STATE_SYN_SENT: case IRDMA_CM_STATE_ESTABLISHED: case IRDMA_CM_STATE_MPAREJ_RCVD: cm_node->tcp_cntxt.rcv_nxt++; irdma_cleanup_retrans_entry(cm_node); cm_node->state = IRDMA_CM_STATE_LAST_ACK; irdma_send_fin(cm_node); break; case IRDMA_CM_STATE_MPAREQ_SENT: irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED); cm_node->tcp_cntxt.rcv_nxt++; irdma_cleanup_retrans_entry(cm_node); cm_node->state = IRDMA_CM_STATE_CLOSED; atomic_inc(&cm_node->refcnt); irdma_send_reset(cm_node); break; case IRDMA_CM_STATE_FIN_WAIT1: cm_node->tcp_cntxt.rcv_nxt++; irdma_cleanup_retrans_entry(cm_node); cm_node->state = IRDMA_CM_STATE_CLOSING; irdma_send_ack(cm_node); /* * Wait for ACK as this is simultaneous close. After we receive ACK, do not send anything. Just rm the * node. */ break; case IRDMA_CM_STATE_FIN_WAIT2: cm_node->tcp_cntxt.rcv_nxt++; irdma_cleanup_retrans_entry(cm_node); cm_node->state = IRDMA_CM_STATE_TIME_WAIT; irdma_send_ack(cm_node); irdma_schedule_cm_timer(cm_node, NULL, IRDMA_TIMER_TYPE_CLOSE, 1, 0); break; case IRDMA_CM_STATE_TIME_WAIT: cm_node->tcp_cntxt.rcv_nxt++; irdma_cleanup_retrans_entry(cm_node); cm_node->state = IRDMA_CM_STATE_CLOSED; irdma_rem_ref_cm_node(cm_node); break; case IRDMA_CM_STATE_OFFLOADED: default: irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "bad state node state = %d\n", cm_node->state); break; } } /** * irdma_handle_rst_pkt - process received RST packet * @cm_node: connection's node * @rbuf: receive buffer */ static void irdma_handle_rst_pkt(struct irdma_cm_node *cm_node, struct irdma_puda_buf *rbuf) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "caller: %pS cm_node=%p state=%d rem_port=0x%04x loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n", __builtin_return_address(0), cm_node, cm_node->state, cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr); irdma_cleanup_retrans_entry(cm_node); switch (cm_node->state) { case IRDMA_CM_STATE_SYN_SENT: case IRDMA_CM_STATE_MPAREQ_SENT: switch (cm_node->mpa_frame_rev) { case IETF_MPA_V2: /* Drop down to MPA_V1 */ cm_node->mpa_frame_rev = IETF_MPA_V1; /* send a syn and goto syn sent state */ cm_node->state = IRDMA_CM_STATE_SYN_SENT; if (irdma_send_syn(cm_node, 0)) irdma_active_open_err(cm_node, false); break; case IETF_MPA_V1: default: irdma_active_open_err(cm_node, false); break; } break; case IRDMA_CM_STATE_MPAREQ_RCVD: atomic_inc(&cm_node->passive_state); break; case IRDMA_CM_STATE_ESTABLISHED: case IRDMA_CM_STATE_SYN_RCVD: case IRDMA_CM_STATE_LISTENING: irdma_passive_open_err(cm_node, false); break; case IRDMA_CM_STATE_OFFLOADED: irdma_active_open_err(cm_node, false); break; case IRDMA_CM_STATE_CLOSED: break; case IRDMA_CM_STATE_FIN_WAIT2: case IRDMA_CM_STATE_FIN_WAIT1: case IRDMA_CM_STATE_LAST_ACK: case IRDMA_CM_STATE_TIME_WAIT: cm_node->state = IRDMA_CM_STATE_CLOSED; irdma_rem_ref_cm_node(cm_node); break; default: break; } } /** * irdma_handle_rcv_mpa - Process a recv'd mpa buffer * @cm_node: connection's node * @rbuf: receive buffer */ static void irdma_handle_rcv_mpa(struct irdma_cm_node *cm_node, struct irdma_puda_buf *rbuf) { int err; int datasize = rbuf->datalen; u8 *dataloc = rbuf->data; enum irdma_cm_event_type type = IRDMA_CM_EVENT_UNKNOWN; u32 res_type; err = irdma_parse_mpa(cm_node, dataloc, &res_type, datasize); if (err) { if (cm_node->state == IRDMA_CM_STATE_MPAREQ_SENT) irdma_active_open_err(cm_node, true); else irdma_passive_open_err(cm_node, true); return; } switch (cm_node->state) { case IRDMA_CM_STATE_ESTABLISHED: if (res_type == IRDMA_MPA_REQUEST_REJECT) irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "state for reject\n"); cm_node->state = IRDMA_CM_STATE_MPAREQ_RCVD; type = IRDMA_CM_EVENT_MPA_REQ; irdma_send_ack(cm_node); /* ACK received MPA request */ atomic_set(&cm_node->passive_state, IRDMA_PASSIVE_STATE_INDICATED); break; case IRDMA_CM_STATE_MPAREQ_SENT: irdma_cleanup_retrans_entry(cm_node); if (res_type == IRDMA_MPA_REQUEST_REJECT) { type = IRDMA_CM_EVENT_MPA_REJECT; cm_node->state = IRDMA_CM_STATE_MPAREJ_RCVD; } else { type = IRDMA_CM_EVENT_CONNECTED; cm_node->state = IRDMA_CM_STATE_OFFLOADED; } irdma_send_ack(cm_node); break; default: irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "wrong cm_node state =%d\n", cm_node->state); break; } irdma_create_event(cm_node, type); } /** * irdma_check_syn - Check for error on received syn ack * @cm_node: connection's node * @tcph: pointer tcp header */ static int irdma_check_syn(struct irdma_cm_node *cm_node, struct tcphdr *tcph) { if (ntohl(tcph->th_ack) != cm_node->tcp_cntxt.loc_seq_num) { irdma_active_open_err(cm_node, true); return 1; } return 0; } /** * irdma_check_seq - check seq numbers if OK * @cm_node: connection's node * @tcph: pointer tcp header */ static int irdma_check_seq(struct irdma_cm_node *cm_node, struct tcphdr *tcph) { u32 seq; u32 ack_seq; u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num; u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt; u32 rcv_wnd; int err = 0; seq = ntohl(tcph->th_seq); ack_seq = ntohl(tcph->th_ack); rcv_wnd = cm_node->tcp_cntxt.rcv_wnd; if (ack_seq != loc_seq_num || !between(seq, rcv_nxt, (rcv_nxt + rcv_wnd))) err = -1; if (err) irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "seq number err\n"); return err; } void irdma_add_conn_est_qh(struct irdma_cm_node *cm_node) { struct irdma_cm_info nfo; irdma_get_addr_info(cm_node, &nfo); nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id; irdma_manage_qhash(cm_node->iwdev, &nfo, IRDMA_QHASH_TYPE_TCP_ESTABLISHED, IRDMA_QHASH_MANAGE_TYPE_ADD, cm_node, false); cm_node->qhash_set = true; } /** * irdma_handle_syn_pkt - is for Passive node * @cm_node: connection's node * @rbuf: receive buffer */ static void irdma_handle_syn_pkt(struct irdma_cm_node *cm_node, struct irdma_puda_buf *rbuf) { struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; int err; u32 inc_sequence; int optionsize; optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr); inc_sequence = ntohl(tcph->th_seq); switch (cm_node->state) { case IRDMA_CM_STATE_SYN_SENT: case IRDMA_CM_STATE_MPAREQ_SENT: /* Rcvd syn on active open connection */ irdma_active_open_err(cm_node, 1); break; case IRDMA_CM_STATE_LISTENING: /* Passive OPEN */ if (atomic_read(&cm_node->listener->pend_accepts_cnt) > cm_node->listener->backlog) { cm_node->cm_core->stats_backlog_drops++; irdma_passive_open_err(cm_node, false); break; } err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1); if (err) { irdma_passive_open_err(cm_node, false); /* drop pkt */ break; } err = cm_node->cm_core->cm_create_ah(cm_node, false); if (err) { irdma_passive_open_err(cm_node, false); /* drop pkt */ break; } cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1; cm_node->accept_pend = 1; atomic_inc(&cm_node->listener->pend_accepts_cnt); cm_node->state = IRDMA_CM_STATE_SYN_RCVD; break; case IRDMA_CM_STATE_CLOSED: irdma_cleanup_retrans_entry(cm_node); atomic_inc(&cm_node->refcnt); irdma_send_reset(cm_node); break; case IRDMA_CM_STATE_OFFLOADED: case IRDMA_CM_STATE_ESTABLISHED: case IRDMA_CM_STATE_FIN_WAIT1: case IRDMA_CM_STATE_FIN_WAIT2: case IRDMA_CM_STATE_MPAREQ_RCVD: case IRDMA_CM_STATE_LAST_ACK: case IRDMA_CM_STATE_CLOSING: case IRDMA_CM_STATE_UNKNOWN: default: break; } } /** * irdma_handle_synack_pkt - Process SYN+ACK packet (active side) * @cm_node: connection's node * @rbuf: receive buffer */ static void irdma_handle_synack_pkt(struct irdma_cm_node *cm_node, struct irdma_puda_buf *rbuf) { struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; int err; u32 inc_sequence; int optionsize; optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr); inc_sequence = ntohl(tcph->th_seq); switch (cm_node->state) { case IRDMA_CM_STATE_SYN_SENT: irdma_cleanup_retrans_entry(cm_node); /* active open */ if (irdma_check_syn(cm_node, tcph)) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "check syn fail\n"); return; } cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack); /* setup options */ err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 0); if (err) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "cm_node=%p tcp_options failed\n", cm_node); break; } irdma_cleanup_retrans_entry(cm_node); cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1; irdma_send_ack(cm_node); /* ACK for the syn_ack */ err = irdma_send_mpa_request(cm_node); if (err) { irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "cm_node=%p irdma_send_mpa_request failed\n", cm_node); break; } cm_node->state = IRDMA_CM_STATE_MPAREQ_SENT; break; case IRDMA_CM_STATE_MPAREQ_RCVD: irdma_passive_open_err(cm_node, true); break; case IRDMA_CM_STATE_LISTENING: cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack); irdma_cleanup_retrans_entry(cm_node); cm_node->state = IRDMA_CM_STATE_CLOSED; irdma_send_reset(cm_node); break; case IRDMA_CM_STATE_CLOSED: cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack); irdma_cleanup_retrans_entry(cm_node); atomic_inc(&cm_node->refcnt); irdma_send_reset(cm_node); break; case IRDMA_CM_STATE_ESTABLISHED: case IRDMA_CM_STATE_FIN_WAIT1: case IRDMA_CM_STATE_FIN_WAIT2: case IRDMA_CM_STATE_LAST_ACK: case IRDMA_CM_STATE_OFFLOADED: case IRDMA_CM_STATE_CLOSING: case IRDMA_CM_STATE_UNKNOWN: case IRDMA_CM_STATE_MPAREQ_SENT: default: break; } } /** * irdma_handle_ack_pkt - process packet with ACK * @cm_node: connection's node * @rbuf: receive buffer */ static int irdma_handle_ack_pkt(struct irdma_cm_node *cm_node, struct irdma_puda_buf *rbuf) { struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; u32 inc_sequence; int ret; int optionsize; u32 datasize = rbuf->datalen; optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr); if (irdma_check_seq(cm_node, tcph)) return -EINVAL; inc_sequence = ntohl(tcph->th_seq); switch (cm_node->state) { case IRDMA_CM_STATE_SYN_RCVD: irdma_cleanup_retrans_entry(cm_node); ret = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1); if (ret) return ret; cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack); cm_node->state = IRDMA_CM_STATE_ESTABLISHED; if (datasize) { cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; irdma_handle_rcv_mpa(cm_node, rbuf); } break; case IRDMA_CM_STATE_ESTABLISHED: irdma_cleanup_retrans_entry(cm_node); if (datasize) { cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; irdma_handle_rcv_mpa(cm_node, rbuf); } break; case IRDMA_CM_STATE_MPAREQ_SENT: cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack); if (datasize) { cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; cm_node->ack_rcvd = false; irdma_handle_rcv_mpa(cm_node, rbuf); } else { cm_node->ack_rcvd = true; } break; case IRDMA_CM_STATE_LISTENING: irdma_cleanup_retrans_entry(cm_node); cm_node->state = IRDMA_CM_STATE_CLOSED; irdma_send_reset(cm_node); break; case IRDMA_CM_STATE_CLOSED: irdma_cleanup_retrans_entry(cm_node); atomic_inc(&cm_node->refcnt); irdma_send_reset(cm_node); break; case IRDMA_CM_STATE_LAST_ACK: case IRDMA_CM_STATE_CLOSING: irdma_cleanup_retrans_entry(cm_node); cm_node->state = IRDMA_CM_STATE_CLOSED; irdma_rem_ref_cm_node(cm_node); break; case IRDMA_CM_STATE_FIN_WAIT1: irdma_cleanup_retrans_entry(cm_node); cm_node->state = IRDMA_CM_STATE_FIN_WAIT2; break; case IRDMA_CM_STATE_SYN_SENT: case IRDMA_CM_STATE_FIN_WAIT2: case IRDMA_CM_STATE_OFFLOADED: case IRDMA_CM_STATE_MPAREQ_RCVD: case IRDMA_CM_STATE_UNKNOWN: default: irdma_cleanup_retrans_entry(cm_node); break; } return 0; } /** * irdma_process_pkt - process cm packet * @cm_node: connection's node * @rbuf: receive buffer */ static void irdma_process_pkt(struct irdma_cm_node *cm_node, struct irdma_puda_buf *rbuf) { enum irdma_tcpip_pkt_type pkt_type = IRDMA_PKT_TYPE_UNKNOWN; struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; u32 fin_set = 0; int err; if (tcph->th_flags & TH_RST) { pkt_type = IRDMA_PKT_TYPE_RST; } else if (tcph->th_flags & TH_SYN) { pkt_type = IRDMA_PKT_TYPE_SYN; if (tcph->th_flags & TH_ACK) pkt_type = IRDMA_PKT_TYPE_SYNACK; } else if (tcph->th_flags & TH_ACK) { pkt_type = IRDMA_PKT_TYPE_ACK; } if (tcph->th_flags & TH_FIN) fin_set = 1; switch (pkt_type) { case IRDMA_PKT_TYPE_SYN: irdma_handle_syn_pkt(cm_node, rbuf); break; case IRDMA_PKT_TYPE_SYNACK: irdma_handle_synack_pkt(cm_node, rbuf); break; case IRDMA_PKT_TYPE_ACK: err = irdma_handle_ack_pkt(cm_node, rbuf); if (fin_set && !err) irdma_handle_fin_pkt(cm_node); break; case IRDMA_PKT_TYPE_RST: irdma_handle_rst_pkt(cm_node, rbuf); break; default: if (fin_set && (!irdma_check_seq(cm_node, (struct tcphdr *)rbuf->tcph))) irdma_handle_fin_pkt(cm_node); break; } } /** * irdma_make_listen_node - create a listen node with params * @cm_core: cm's core * @iwdev: iwarp device structure * @cm_info: quad info for connection */ static struct irdma_cm_listener * irdma_make_listen_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, struct irdma_cm_info *cm_info) { struct irdma_cm_listener *listener; unsigned long flags; /* cannot have multiple matching listeners */ listener = irdma_find_listener(cm_core, cm_info->loc_addr, cm_info->loc_port, cm_info->vlan_id, IRDMA_CM_LISTENER_EITHER_STATE); if (listener && listener->listener_state == IRDMA_CM_LISTENER_ACTIVE_STATE) { atomic_dec(&listener->refcnt); return NULL; } if (!listener) { /* * create a CM listen node 1/2 node to compare incoming traffic to */ listener = kzalloc(sizeof(*listener), GFP_KERNEL); if (!listener) return NULL; cm_core->stats_listen_nodes_created++; memcpy(listener->loc_addr, cm_info->loc_addr, sizeof(listener->loc_addr)); listener->loc_port = cm_info->loc_port; INIT_LIST_HEAD(&listener->child_listen_list); atomic_set(&listener->refcnt, 1); } else { listener->reused_node = 1; } listener->cm_id = cm_info->cm_id; listener->ipv4 = cm_info->ipv4; listener->vlan_id = cm_info->vlan_id; atomic_set(&listener->pend_accepts_cnt, 0); listener->cm_core = cm_core; listener->iwdev = iwdev; listener->backlog = cm_info->backlog; listener->listener_state = IRDMA_CM_LISTENER_ACTIVE_STATE; if (!listener->reused_node) { spin_lock_irqsave(&cm_core->listen_list_lock, flags); list_add(&listener->list, &cm_core->listen_list); spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); } return listener; } /** * irdma_create_cm_node - make a connection node with params * @cm_core: cm's core * @iwdev: iwarp device structure * @conn_param: connection parameters * @cm_info: quad info for connection * @caller_cm_node: pointer to cm_node structure to return */ static int irdma_create_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, struct iw_cm_conn_param *conn_param, struct irdma_cm_info *cm_info, struct irdma_cm_node **caller_cm_node) { struct irdma_cm_node *cm_node; u16 private_data_len = conn_param->private_data_len; const void *private_data = conn_param->private_data; /* create a CM connection node */ cm_node = irdma_make_cm_node(cm_core, iwdev, cm_info, NULL); if (!cm_node) return -ENOMEM; /* set our node side to client (active) side */ cm_node->tcp_cntxt.client = 1; cm_node->tcp_cntxt.rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE; irdma_record_ird_ord(cm_node, conn_param->ird, conn_param->ord); cm_node->pdata.size = private_data_len; cm_node->pdata.addr = cm_node->pdata_buf; memcpy(cm_node->pdata_buf, private_data, private_data_len); *caller_cm_node = cm_node; return 0; } /** * irdma_cm_reject - reject and teardown a connection * @cm_node: connection's node * @pdata: ptr to private data for reject * @plen: size of private data */ static int irdma_cm_reject(struct irdma_cm_node *cm_node, const void *pdata, u8 plen) { int ret; int passive_state; if (cm_node->tcp_cntxt.client) return 0; irdma_cleanup_retrans_entry(cm_node); passive_state = atomic_add_return(1, &cm_node->passive_state); if (passive_state == IRDMA_SEND_RESET_EVENT) { cm_node->state = IRDMA_CM_STATE_CLOSED; irdma_rem_ref_cm_node(cm_node); return 0; } if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) { irdma_rem_ref_cm_node(cm_node); return 0; } ret = irdma_send_mpa_reject(cm_node, pdata, plen); if (!ret) return 0; cm_node->state = IRDMA_CM_STATE_CLOSED; if (irdma_send_reset(cm_node)) irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "send reset failed\n"); return ret; } /** * irdma_cm_close - close of cm connection * @cm_node: connection's node */ static int irdma_cm_close(struct irdma_cm_node *cm_node) { switch (cm_node->state) { case IRDMA_CM_STATE_SYN_RCVD: case IRDMA_CM_STATE_SYN_SENT: case IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED: case IRDMA_CM_STATE_ESTABLISHED: case IRDMA_CM_STATE_ACCEPTING: case IRDMA_CM_STATE_MPAREQ_SENT: case IRDMA_CM_STATE_MPAREQ_RCVD: irdma_cleanup_retrans_entry(cm_node); irdma_send_reset(cm_node); break; case IRDMA_CM_STATE_CLOSE_WAIT: cm_node->state = IRDMA_CM_STATE_LAST_ACK; irdma_send_fin(cm_node); break; case IRDMA_CM_STATE_FIN_WAIT1: case IRDMA_CM_STATE_FIN_WAIT2: case IRDMA_CM_STATE_LAST_ACK: case IRDMA_CM_STATE_TIME_WAIT: case IRDMA_CM_STATE_CLOSING: return -EINVAL; case IRDMA_CM_STATE_LISTENING: irdma_cleanup_retrans_entry(cm_node); irdma_send_reset(cm_node); break; case IRDMA_CM_STATE_MPAREJ_RCVD: case IRDMA_CM_STATE_UNKNOWN: case IRDMA_CM_STATE_INITED: case IRDMA_CM_STATE_CLOSED: case IRDMA_CM_STATE_LISTENER_DESTROYED: irdma_rem_ref_cm_node(cm_node); break; case IRDMA_CM_STATE_OFFLOADED: if (cm_node->send_entry) irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "CM send_entry in OFFLOADED state\n"); irdma_rem_ref_cm_node(cm_node); break; } return 0; } /** * irdma_receive_ilq - recv an ETHERNET packet, and process it * through CM * @vsi: VSI structure of dev * @rbuf: receive buffer */ void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf) { struct irdma_cm_node *cm_node; struct irdma_cm_listener *listener; struct ip *iph; struct ip6_hdr *ip6h; struct tcphdr *tcph; struct irdma_cm_info cm_info = {0}; struct irdma_device *iwdev = vsi->back_vsi; struct irdma_cm_core *cm_core = &iwdev->cm_core; struct ether_vlan_header *ethh; u16 vtag; /* if vlan, then maclen = 18 else 14 */ iph = (struct ip *)rbuf->iph; irdma_debug_buf(vsi->dev, IRDMA_DEBUG_ILQ, "RECEIVE ILQ BUFFER", rbuf->mem.va, rbuf->totallen); if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { if (rbuf->vlan_valid) { vtag = rbuf->vlan_id; cm_info.user_pri = (vtag & EVL_PRI_MASK) >> VLAN_PRIO_SHIFT; cm_info.vlan_id = vtag & EVL_VLID_MASK; } else { cm_info.vlan_id = 0xFFFF; } } else { ethh = rbuf->mem.va; if (ethh->evl_proto == htons(ETH_P_8021Q)) { vtag = ntohs(ethh->evl_tag); cm_info.user_pri = (vtag & EVL_PRI_MASK) >> VLAN_PRIO_SHIFT; cm_info.vlan_id = vtag & EVL_VLID_MASK; irdma_debug(iwdev_to_idev(cm_core->iwdev), IRDMA_DEBUG_CM, "vlan_id=%d\n", cm_info.vlan_id); } else { cm_info.vlan_id = 0xFFFF; } } tcph = (struct tcphdr *)rbuf->tcph; if (rbuf->ipv4) { cm_info.loc_addr[0] = ntohl(iph->ip_dst.s_addr); cm_info.rem_addr[0] = ntohl(iph->ip_src.s_addr); cm_info.ipv4 = true; cm_info.tos = iph->ip_tos; } else { ip6h = (struct ip6_hdr *)rbuf->iph; irdma_copy_ip_ntohl(cm_info.loc_addr, ip6h->ip6_dst.__u6_addr.__u6_addr32); irdma_copy_ip_ntohl(cm_info.rem_addr, ip6h->ip6_src.__u6_addr.__u6_addr32); cm_info.ipv4 = false; cm_info.tos = (ip6h->ip6_vfc << 4) | ip6h->ip6_flow; } cm_info.loc_port = ntohs(tcph->th_dport); cm_info.rem_port = ntohs(tcph->th_sport); cm_node = irdma_find_node(cm_core, cm_info.rem_port, cm_info.rem_addr, cm_info.loc_port, cm_info.loc_addr, cm_info.vlan_id); if (!cm_node) { /* * Only type of packet accepted are for the PASSIVE open (syn only) */ if (!(tcph->th_flags & TH_SYN) || tcph->th_flags & TH_ACK) return; listener = irdma_find_listener(cm_core, cm_info.loc_addr, cm_info.loc_port, cm_info.vlan_id, IRDMA_CM_LISTENER_ACTIVE_STATE); if (!listener) { cm_info.cm_id = NULL; irdma_debug(iwdev_to_idev(cm_core->iwdev), IRDMA_DEBUG_CM, "no listener found\n"); return; } cm_info.cm_id = listener->cm_id; cm_node = irdma_make_cm_node(cm_core, iwdev, &cm_info, listener); if (!cm_node) { irdma_debug(iwdev_to_idev(cm_core->iwdev), IRDMA_DEBUG_CM, "allocate node failed\n"); atomic_dec(&listener->refcnt); return; } if (!(tcph->th_flags & (TH_RST | TH_FIN))) { cm_node->state = IRDMA_CM_STATE_LISTENING; } else { irdma_rem_ref_cm_node(cm_node); return; } atomic_inc(&cm_node->refcnt); } else if (cm_node->state == IRDMA_CM_STATE_OFFLOADED) { irdma_rem_ref_cm_node(cm_node); return; } irdma_process_pkt(cm_node, rbuf); irdma_rem_ref_cm_node(cm_node); } static int irdma_add_qh(struct irdma_cm_node *cm_node, bool active) { if (!active) irdma_add_conn_est_qh(cm_node); return 0; } static void irdma_cm_free_ah_nop(struct irdma_cm_node *cm_node) { } /** * irdma_setup_cm_core - setup top level instance of a cm core * @iwdev: iwarp device structure * @rdma_ver: HW version */ int irdma_setup_cm_core(struct irdma_device *iwdev, u8 rdma_ver) { struct irdma_cm_core *cm_core = &iwdev->cm_core; cm_core->iwdev = iwdev; cm_core->dev = &iwdev->rf->sc_dev; /* Handles CM event work items send to Iwarp core */ cm_core->event_wq = alloc_ordered_workqueue("iwarp-event-wq", 0); if (!cm_core->event_wq) return -ENOMEM; INIT_LIST_HEAD(&cm_core->listen_list); timer_setup(&cm_core->tcp_timer, irdma_cm_timer_tick, 0); spin_lock_init(&cm_core->ht_lock); spin_lock_init(&cm_core->listen_list_lock); spin_lock_init(&cm_core->apbvt_lock); switch (rdma_ver) { case IRDMA_GEN_1: cm_core->form_cm_frame = irdma_form_uda_cm_frame; cm_core->cm_create_ah = irdma_add_qh; cm_core->cm_free_ah = irdma_cm_free_ah_nop; break; case IRDMA_GEN_2: default: cm_core->form_cm_frame = irdma_form_ah_cm_frame; cm_core->cm_create_ah = irdma_cm_create_ah; cm_core->cm_free_ah = irdma_cm_free_ah; } return 0; } /** * irdma_cleanup_cm_core - deallocate a top level instance of a * cm core * @cm_core: cm's core */ void irdma_cleanup_cm_core(struct irdma_cm_core *cm_core) { unsigned long flags; if (!cm_core) return; spin_lock_irqsave(&cm_core->ht_lock, flags); if (timer_pending(&cm_core->tcp_timer)) del_timer_sync(&cm_core->tcp_timer); spin_unlock_irqrestore(&cm_core->ht_lock, flags); destroy_workqueue(cm_core->event_wq); cm_core->dev->ws_reset(&cm_core->iwdev->vsi); } /** * irdma_init_tcp_ctx - setup qp context * @cm_node: connection's node * @tcp_info: offload info for tcp * @iwqp: associate qp for the connection */ static void irdma_init_tcp_ctx(struct irdma_cm_node *cm_node, struct irdma_tcp_offload_info *tcp_info, struct irdma_qp *iwqp) { tcp_info->ipv4 = cm_node->ipv4; tcp_info->drop_ooo_seg = !iwqp->iwdev->iw_ooo; tcp_info->wscale = true; tcp_info->ignore_tcp_opt = true; tcp_info->ignore_tcp_uns_opt = true; tcp_info->no_nagle = false; tcp_info->ttl = IRDMA_DEFAULT_TTL; tcp_info->rtt_var = IRDMA_DEFAULT_RTT_VAR; tcp_info->ss_thresh = IRDMA_DEFAULT_SS_THRESH; tcp_info->rexmit_thresh = IRDMA_DEFAULT_REXMIT_THRESH; tcp_info->tcp_state = IRDMA_TCP_STATE_ESTABLISHED; tcp_info->snd_wscale = cm_node->tcp_cntxt.snd_wscale; tcp_info->rcv_wscale = cm_node->tcp_cntxt.rcv_wscale; tcp_info->snd_nxt = cm_node->tcp_cntxt.loc_seq_num; tcp_info->snd_wnd = cm_node->tcp_cntxt.snd_wnd; tcp_info->rcv_nxt = cm_node->tcp_cntxt.rcv_nxt; tcp_info->snd_max = cm_node->tcp_cntxt.loc_seq_num; tcp_info->snd_una = cm_node->tcp_cntxt.loc_seq_num; tcp_info->cwnd = 2 * cm_node->tcp_cntxt.mss; tcp_info->snd_wl1 = cm_node->tcp_cntxt.rcv_nxt; tcp_info->snd_wl2 = cm_node->tcp_cntxt.loc_seq_num; tcp_info->max_snd_window = cm_node->tcp_cntxt.max_snd_wnd; tcp_info->rcv_wnd = cm_node->tcp_cntxt.rcv_wnd << cm_node->tcp_cntxt.rcv_wscale; tcp_info->flow_label = 0; tcp_info->snd_mss = (u32)cm_node->tcp_cntxt.mss; tcp_info->tos = cm_node->tos; if (cm_node->vlan_id < VLAN_N_VID) { tcp_info->insert_vlan_tag = true; tcp_info->vlan_tag = cm_node->vlan_id; tcp_info->vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT; } tcp_info->src_port = cm_node->loc_port; tcp_info->dst_port = cm_node->rem_port; tcp_info->arp_idx = (u16)irdma_arp_table(iwqp->iwdev->rf, cm_node->rem_addr, NULL, IRDMA_ARP_RESOLVE); if (cm_node->ipv4) { tcp_info->dest_ip_addr[3] = cm_node->rem_addr[0]; tcp_info->local_ipaddr[3] = cm_node->loc_addr[0]; } else { memcpy(tcp_info->dest_ip_addr, cm_node->rem_addr, sizeof(tcp_info->dest_ip_addr)); memcpy(tcp_info->local_ipaddr, cm_node->loc_addr, sizeof(tcp_info->local_ipaddr)); } } /** * irdma_cm_init_tsa_conn - setup qp for RTS * @iwqp: associate qp for the connection * @cm_node: connection's node */ static void irdma_cm_init_tsa_conn(struct irdma_qp *iwqp, struct irdma_cm_node *cm_node) { struct irdma_iwarp_offload_info *iwarp_info; struct irdma_qp_host_ctx_info *ctx_info; iwarp_info = &iwqp->iwarp_info; ctx_info = &iwqp->ctx_info; ctx_info->tcp_info = &iwqp->tcp_info; ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; iwarp_info->ord_size = cm_node->ord_size; iwarp_info->ird_size = cm_node->ird_size; iwarp_info->rd_en = true; iwarp_info->rdmap_ver = 1; iwarp_info->ddp_ver = 1; iwarp_info->pd_id = iwqp->iwpd->sc_pd.pd_id; ctx_info->tcp_info_valid = true; ctx_info->iwarp_info_valid = true; ctx_info->user_pri = cm_node->user_pri; irdma_init_tcp_ctx(cm_node, &iwqp->tcp_info, iwqp); if (cm_node->snd_mark_en) { iwarp_info->snd_mark_en = true; iwarp_info->snd_mark_offset = (iwqp->tcp_info.snd_nxt & SNDMARKER_SEQNMASK) + cm_node->lsmm_size; } cm_node->state = IRDMA_CM_STATE_OFFLOADED; iwqp->tcp_info.tcp_state = IRDMA_TCP_STATE_ESTABLISHED; iwqp->tcp_info.src_mac_addr_idx = iwqp->iwdev->mac_ip_table_idx; if (cm_node->rcv_mark_en) { iwarp_info->rcv_mark_en = true; iwarp_info->align_hdrs = true; } irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); /* once tcp_info is set, no need to do it again */ ctx_info->tcp_info_valid = false; ctx_info->iwarp_info_valid = false; } /** * irdma_cm_disconn - when a connection is being closed * @iwqp: associated qp for the connection */ void irdma_cm_disconn(struct irdma_qp *iwqp) { struct irdma_device *iwdev = iwqp->iwdev; struct disconn_work *work; unsigned long flags; work = kzalloc(sizeof(*work), GFP_ATOMIC); if (!work) return; spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); if (!iwdev->rf->qp_table[iwqp->ibqp.qp_num]) { spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "qp_id %d is already freed\n", iwqp->ibqp.qp_num); kfree(work); return; } irdma_qp_add_ref(&iwqp->ibqp); spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); work->iwqp = iwqp; INIT_WORK(&work->work, irdma_disconnect_worker); queue_work(iwdev->cleanup_wq, &work->work); } /** * irdma_qp_disconnect - free qp and close cm * @iwqp: associate qp for the connection */ static void irdma_qp_disconnect(struct irdma_qp *iwqp) { struct irdma_device *iwdev = iwqp->iwdev; iwqp->active_conn = 0; /* close the CM node down if it is still active */ irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "Call close API\n"); irdma_cm_close(iwqp->cm_node); } /** * irdma_cm_disconn_true - called by worker thread to disconnect qp * @iwqp: associate qp for the connection */ static void irdma_cm_disconn_true(struct irdma_qp *iwqp) { struct iw_cm_id *cm_id; struct irdma_device *iwdev; struct irdma_sc_qp *qp = &iwqp->sc_qp; u16 last_ae; u8 original_hw_tcp_state; u8 original_ibqp_state; int disconn_status = 0; int issue_disconn = 0; int issue_close = 0; int issue_flush = 0; unsigned long flags; int err; iwdev = iwqp->iwdev; spin_lock_irqsave(&iwqp->lock, flags); if (rdma_protocol_roce(&iwdev->ibdev, 1)) { struct ib_qp_attr attr; if (iwqp->flush_issued || iwqp->sc_qp.qp_uk.destroy_pending) { spin_unlock_irqrestore(&iwqp->lock, flags); return; } spin_unlock_irqrestore(&iwqp->lock, flags); attr.qp_state = IB_QPS_ERR; irdma_modify_qp_roce(&iwqp->ibqp, &attr, IB_QP_STATE, NULL); irdma_ib_qp_event(iwqp, qp->event_type); return; } cm_id = iwqp->cm_id; /* make sure we havent already closed this connection */ if (!cm_id) { spin_unlock_irqrestore(&iwqp->lock, flags); return; } original_hw_tcp_state = iwqp->hw_tcp_state; original_ibqp_state = iwqp->ibqp_state; last_ae = iwqp->last_aeq; if (qp->term_flags) { issue_disconn = 1; issue_close = 1; iwqp->cm_id = NULL; irdma_terminate_del_timer(qp); if (!iwqp->flush_issued) { iwqp->flush_issued = 1; issue_flush = 1; } } else if ((original_hw_tcp_state == IRDMA_TCP_STATE_CLOSE_WAIT) || ((original_ibqp_state == IB_QPS_RTS) && (last_ae == IRDMA_AE_LLP_CONNECTION_RESET))) { issue_disconn = 1; if (last_ae == IRDMA_AE_LLP_CONNECTION_RESET) disconn_status = -ECONNRESET; } if ((original_hw_tcp_state == IRDMA_TCP_STATE_CLOSED || original_hw_tcp_state == IRDMA_TCP_STATE_TIME_WAIT || last_ae == IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE || last_ae == IRDMA_AE_BAD_CLOSE || last_ae == IRDMA_AE_LLP_CONNECTION_RESET || iwdev->rf->reset)) { issue_close = 1; iwqp->cm_id = NULL; qp->term_flags = 0; if (!iwqp->flush_issued) { iwqp->flush_issued = 1; issue_flush = 1; } } spin_unlock_irqrestore(&iwqp->lock, flags); if (issue_flush && !iwqp->sc_qp.qp_uk.destroy_pending) { if (!iwqp->user_mode) queue_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | IRDMA_FLUSH_RQ | IRDMA_FLUSH_WAIT); if (qp->term_flags) irdma_ib_qp_event(iwqp, qp->event_type); } if (!cm_id || !cm_id->event_handler) return; spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags); if (!iwqp->cm_node) { spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); return; } atomic_inc(&iwqp->cm_node->refcnt); spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); if (issue_disconn) { err = irdma_send_cm_event(iwqp->cm_node, cm_id, IW_CM_EVENT_DISCONNECT, disconn_status); if (err) irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "disconnect event failed: - cm_id = %p\n", cm_id); } if (issue_close) { cm_id->provider_data = iwqp; err = irdma_send_cm_event(iwqp->cm_node, cm_id, IW_CM_EVENT_CLOSE, 0); if (err) irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "close event failed: - cm_id = %p\n", cm_id); irdma_qp_disconnect(iwqp); } irdma_rem_ref_cm_node(iwqp->cm_node); } /** * irdma_disconnect_worker - worker for connection close * @work: points or disconn structure */ static void irdma_disconnect_worker(struct work_struct *work) { struct disconn_work *dwork = container_of(work, struct disconn_work, work); struct irdma_qp *iwqp = dwork->iwqp; kfree(dwork); irdma_cm_disconn_true(iwqp); irdma_qp_rem_ref(&iwqp->ibqp); } /** * irdma_free_lsmm_rsrc - free lsmm memory and deregister * @iwqp: associate qp for the connection */ void irdma_free_lsmm_rsrc(struct irdma_qp *iwqp) { struct irdma_device *iwdev; iwdev = iwqp->iwdev; if (iwqp->ietf_mem.va) { if (iwqp->lsmm_mr) kc_free_lsmm_dereg_mr(iwdev, iwqp); irdma_free_dma_mem(iwdev->rf->sc_dev.hw, &iwqp->ietf_mem); iwqp->ietf_mem.va = NULL; } } /** * irdma_accept - registered call for connection to be accepted * @cm_id: cm information for passive connection * @conn_param: accpet parameters */ int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) { struct ib_qp *ibqp; struct irdma_qp *iwqp; struct irdma_device *iwdev; struct irdma_sc_dev *dev; struct irdma_cm_node *cm_node; struct ib_qp_attr attr = {0}; int passive_state; struct ib_mr *ibmr; struct irdma_pd *iwpd; u16 buf_len = 0; struct irdma_kmem_info accept; u64 tagged_offset; int wait_ret; int ret = 0; ibqp = irdma_get_qp(cm_id->device, conn_param->qpn); if (!ibqp) return -EINVAL; iwqp = to_iwqp(ibqp); iwdev = iwqp->iwdev; dev = &iwdev->rf->sc_dev; cm_node = cm_id->provider_data; if (((struct sockaddr_in *)&cm_id->local_addr)->sin_family == AF_INET) { cm_node->ipv4 = true; cm_node->vlan_id = irdma_get_vlan_ipv4(cm_node->loc_addr); } else { cm_node->ipv4 = false; irdma_netdev_vlan_ipv6(cm_node->loc_addr, &cm_node->vlan_id, NULL); } irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "Accept vlan_id=%d\n", cm_node->vlan_id); if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) { ret = -EINVAL; goto error; } passive_state = atomic_add_return(1, &cm_node->passive_state); if (passive_state == IRDMA_SEND_RESET_EVENT) { ret = -ECONNRESET; goto error; } buf_len = conn_param->private_data_len + IRDMA_MAX_IETF_SIZE; iwqp->ietf_mem.size = buf_len; iwqp->ietf_mem.va = irdma_allocate_dma_mem(dev->hw, &iwqp->ietf_mem, iwqp->ietf_mem.size, 1); if (!iwqp->ietf_mem.va) { ret = -ENOMEM; goto error; } cm_node->pdata.size = conn_param->private_data_len; accept.addr = iwqp->ietf_mem.va; accept.size = irdma_cm_build_mpa_frame(cm_node, &accept, MPA_KEY_REPLY); memcpy((u8 *)accept.addr + accept.size, conn_param->private_data, conn_param->private_data_len); if (cm_node->dev->ws_add(iwqp->sc_qp.vsi, cm_node->user_pri)) { ret = -ENOMEM; goto error; } iwqp->sc_qp.user_pri = cm_node->user_pri; irdma_qp_add_qos(&iwqp->sc_qp); if (cm_node->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) iwdev->rf->check_fc(&iwdev->vsi, &iwqp->sc_qp); /* setup our first outgoing iWarp send WQE (the IETF frame response) */ iwpd = iwqp->iwpd; tagged_offset = (uintptr_t)iwqp->ietf_mem.va; ibmr = irdma_reg_phys_mr(&iwpd->ibpd, iwqp->ietf_mem.pa, buf_len, IB_ACCESS_LOCAL_WRITE, &tagged_offset); if (IS_ERR(ibmr)) { ret = -ENOMEM; goto error; } ibmr->pd = &iwpd->ibpd; ibmr->device = iwpd->ibpd.device; iwqp->lsmm_mr = ibmr; if (iwqp->page) iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(iwqp->page); cm_node->lsmm_size = accept.size + conn_param->private_data_len; irdma_sc_send_lsmm(&iwqp->sc_qp, iwqp->ietf_mem.va, cm_node->lsmm_size, ibmr->lkey); if (iwqp->page) kunmap_local(iwqp->sc_qp.qp_uk.sq_base); iwqp->cm_id = cm_id; cm_node->cm_id = cm_id; cm_id->provider_data = iwqp; iwqp->active_conn = 0; iwqp->cm_node = cm_node; cm_node->iwqp = iwqp; irdma_cm_init_tsa_conn(iwqp, cm_node); irdma_qp_add_ref(&iwqp->ibqp); cm_id->add_ref(cm_id); attr.qp_state = IB_QPS_RTS; cm_node->qhash_set = false; cm_node->cm_core->cm_free_ah(cm_node); irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL); if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) { wait_ret = wait_event_interruptible_timeout(iwqp->waitq, iwqp->rts_ae_rcvd, IRDMA_MAX_TIMEOUT); if (!wait_ret) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n", cm_node, cm_node->loc_port, cm_node->rem_port, cm_node->cm_id); ret = -ECONNRESET; goto error; } } irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_ESTABLISHED, 0); cm_node->accelerated = true; complete(&cm_node->establish_comp); if (cm_node->accept_pend) { atomic_dec(&cm_node->listener->pend_accepts_cnt); cm_node->accept_pend = 0; } irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n", cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num); cm_node->cm_core->stats_accepts++; return 0; error: irdma_free_lsmm_rsrc(iwqp); irdma_rem_ref_cm_node(cm_node); return ret; } /** * irdma_reject - registered call for connection to be rejected * @cm_id: cm information for passive connection * @pdata: private data to be sent * @pdata_len: private data length */ int irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) { struct irdma_device *iwdev; struct irdma_cm_node *cm_node; cm_node = cm_id->provider_data; cm_node->pdata.size = pdata_len; iwdev = to_iwdev(cm_id->device); if (!iwdev) return -EINVAL; cm_node->cm_core->stats_rejects++; if (pdata_len + sizeof(struct ietf_mpa_v2) > IRDMA_MAX_CM_BUF) return -EINVAL; return irdma_cm_reject(cm_node, pdata, pdata_len); } /** * irdma_connect - registered call for connection to be established * @cm_id: cm information for passive connection * @conn_param: Information about the connection */ int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) { struct ib_qp *ibqp; struct irdma_qp *iwqp; struct irdma_device *iwdev; struct irdma_cm_node *cm_node; struct irdma_cm_info cm_info; struct sockaddr_in *laddr; struct sockaddr_in *raddr; struct sockaddr_in6 *laddr6; struct sockaddr_in6 *raddr6; int ret = 0; ibqp = irdma_get_qp(cm_id->device, conn_param->qpn); if (!ibqp) return -EINVAL; iwqp = to_iwqp(ibqp); if (!iwqp) return -EINVAL; iwdev = iwqp->iwdev; if (!iwdev) return -EINVAL; laddr = (struct sockaddr_in *)&cm_id->m_local_addr; raddr = (struct sockaddr_in *)&cm_id->m_remote_addr; laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr; raddr6 = (struct sockaddr_in6 *)&cm_id->m_remote_addr; if (!(laddr->sin_port) || !(raddr->sin_port)) return -EINVAL; iwqp->active_conn = 1; iwqp->cm_id = NULL; cm_id->provider_data = iwqp; /* set up the connection params for the node */ if (cm_id->remote_addr.ss_family == AF_INET) { if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4) return -EINVAL; cm_info.ipv4 = true; memset(cm_info.loc_addr, 0, sizeof(cm_info.loc_addr)); memset(cm_info.rem_addr, 0, sizeof(cm_info.rem_addr)); cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr); cm_info.rem_addr[0] = ntohl(raddr->sin_addr.s_addr); cm_info.loc_port = ntohs(laddr->sin_port); cm_info.rem_port = ntohs(raddr->sin_port); cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr); } else { if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6) return -EINVAL; cm_info.ipv4 = false; irdma_copy_ip_ntohl(cm_info.loc_addr, laddr6->sin6_addr.__u6_addr.__u6_addr32); irdma_copy_ip_ntohl(cm_info.rem_addr, raddr6->sin6_addr.__u6_addr.__u6_addr32); cm_info.loc_port = ntohs(laddr6->sin6_port); cm_info.rem_port = ntohs(raddr6->sin6_port); irdma_netdev_vlan_ipv6(cm_info.loc_addr, &cm_info.vlan_id, NULL); } cm_info.cm_id = cm_id; cm_info.qh_qpid = iwdev->vsi.ilq->qp_id; cm_info.tos = cm_id->tos; if (iwdev->vsi.dscp_mode) cm_info.user_pri = iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(cm_info.tos)]; else cm_info.user_pri = rt_tos2priority(cm_id->tos); if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, cm_info.user_pri)) return -ENOMEM; iwqp->sc_qp.user_pri = cm_info.user_pri; irdma_qp_add_qos(&iwqp->sc_qp); if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) iwdev->rf->check_fc(&iwdev->vsi, &iwqp->sc_qp); irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_DCB, "TOS:[%d] UP:[%d]\n", cm_id->tos, cm_info.user_pri); ret = irdma_create_cm_node(&iwdev->cm_core, iwdev, conn_param, &cm_info, &cm_node); if (ret) return ret; ret = cm_node->cm_core->cm_create_ah(cm_node, true); if (ret) goto err; if (irdma_manage_qhash(iwdev, &cm_info, IRDMA_QHASH_TYPE_TCP_ESTABLISHED, IRDMA_QHASH_MANAGE_TYPE_ADD, NULL, true)) { ret = -EINVAL; goto err; } cm_node->qhash_set = true; cm_node->apbvt_entry = irdma_add_apbvt(iwdev, cm_info.loc_port); if (!cm_node->apbvt_entry) { ret = -EINVAL; goto err; } cm_node->apbvt_set = true; iwqp->cm_node = cm_node; cm_node->iwqp = iwqp; iwqp->cm_id = cm_id; irdma_qp_add_ref(&iwqp->ibqp); cm_id->add_ref(cm_id); if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) { cm_node->state = IRDMA_CM_STATE_SYN_SENT; ret = irdma_send_syn(cm_node, 0); if (ret) goto err; } irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n", cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num); return 0; err: if (cm_info.ipv4) irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "connect() FAILED: dest addr=%pI4", cm_info.rem_addr); else irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "connect() FAILED: dest addr=%pI6", cm_info.rem_addr); irdma_rem_ref_cm_node(cm_node); iwdev->cm_core.stats_connect_errs++; return ret; } /** * irdma_create_listen - registered call creating listener * @cm_id: cm information for passive connection * @backlog: to max accept pending count */ int irdma_create_listen(struct iw_cm_id *cm_id, int backlog) { struct irdma_device *iwdev; struct irdma_cm_listener *cm_listen_node; struct irdma_cm_info cm_info = {0}; struct sockaddr_in *laddr; struct sockaddr_in6 *laddr6; bool wildcard = false; int err; iwdev = to_iwdev(cm_id->device); if (!iwdev) return -EINVAL; laddr = (struct sockaddr_in *)&cm_id->m_local_addr; laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr; cm_info.qh_qpid = iwdev->vsi.ilq->qp_id; if (laddr->sin_family == AF_INET) { if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4) return -EINVAL; cm_info.ipv4 = true; cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr); cm_info.loc_port = ntohs(laddr->sin_port); if (laddr->sin_addr.s_addr != htonl(INADDR_ANY)) { cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr); } else { cm_info.vlan_id = 0xFFFF; wildcard = true; } } else { if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6) return -EINVAL; cm_info.ipv4 = false; irdma_copy_ip_ntohl(cm_info.loc_addr, laddr6->sin6_addr.__u6_addr.__u6_addr32); cm_info.loc_port = ntohs(laddr6->sin6_port); if (!IN6_IS_ADDR_UNSPECIFIED(&laddr6->sin6_addr)) { irdma_netdev_vlan_ipv6(cm_info.loc_addr, &cm_info.vlan_id, NULL); } else { cm_info.vlan_id = 0xFFFF; wildcard = true; } } if (cm_info.vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode) cm_info.vlan_id = 0; cm_info.backlog = backlog; cm_info.cm_id = cm_id; cm_listen_node = irdma_make_listen_node(&iwdev->cm_core, iwdev, &cm_info); if (!cm_listen_node) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "cm_listen_node == NULL\n"); return -ENOMEM; } cm_id->provider_data = cm_listen_node; cm_listen_node->tos = cm_id->tos; if (iwdev->vsi.dscp_mode) cm_listen_node->user_pri = iwdev->vsi.dscp_map[irdma_tos2dscp(cm_id->tos)]; else cm_listen_node->user_pri = rt_tos2priority(cm_id->tos); cm_info.user_pri = cm_listen_node->user_pri; if (!cm_listen_node->reused_node) { if (wildcard) { err = irdma_add_mqh(iwdev, &cm_info, cm_listen_node); if (err) goto error; } else { err = irdma_manage_qhash(iwdev, &cm_info, IRDMA_QHASH_TYPE_TCP_SYN, IRDMA_QHASH_MANAGE_TYPE_ADD, NULL, true); if (err) goto error; cm_listen_node->qhash_set = true; } cm_listen_node->apbvt_entry = irdma_add_apbvt(iwdev, cm_info.loc_port); if (!cm_listen_node->apbvt_entry) goto error; } cm_id->add_ref(cm_id); cm_listen_node->cm_core->stats_listen_created++; irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d\n", cm_listen_node->loc_port, cm_listen_node->loc_addr, cm_listen_node, cm_listen_node->cm_id, cm_listen_node->qhash_set, cm_listen_node->vlan_id); return 0; error: irdma_cm_del_listen(&iwdev->cm_core, cm_listen_node, false); return -EINVAL; } /** * irdma_destroy_listen - registered call to destroy listener * @cm_id: cm information for passive connection */ int irdma_destroy_listen(struct iw_cm_id *cm_id) { struct irdma_device *iwdev; iwdev = to_iwdev(cm_id->device); if (cm_id->provider_data) irdma_cm_del_listen(&iwdev->cm_core, cm_id->provider_data, true); else irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "cm_id->provider_data was NULL\n"); cm_id->rem_ref(cm_id); return 0; } /** * irdma_teardown_list_prep - add conn nodes slated for tear down to list * @cm_core: cm's core * @teardown_list: a list to which cm_node will be selected * @ipaddr: pointer to ip address * @nfo: pointer to cm_info structure instance * @disconnect_all: flag indicating disconnect all QPs */ static void irdma_teardown_list_prep(struct irdma_cm_core *cm_core, struct list_head *teardown_list, u32 *ipaddr, struct irdma_cm_info *nfo, bool disconnect_all) { struct irdma_cm_node *cm_node; int bkt; HASH_FOR_EACH_RCU(cm_core->cm_hash_tbl, bkt, cm_node, list) { if ((disconnect_all || (nfo->vlan_id == cm_node->vlan_id && !memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16))) && atomic_inc_not_zero(&cm_node->refcnt)) list_add(&cm_node->teardown_entry, teardown_list); } } /** * irdma_cm_event_connected - handle connected active node * @event: the info for cm_node of connection */ static void irdma_cm_event_connected(struct irdma_cm_event *event) { struct irdma_qp *iwqp; struct irdma_device *iwdev; struct irdma_cm_node *cm_node; struct irdma_sc_dev *dev; struct ib_qp_attr attr = {0}; struct iw_cm_id *cm_id; int status; bool read0; int wait_ret = 0; cm_node = event->cm_node; cm_id = cm_node->cm_id; iwqp = cm_id->provider_data; iwdev = iwqp->iwdev; dev = &iwdev->rf->sc_dev; if (iwqp->sc_qp.qp_uk.destroy_pending) { status = -ETIMEDOUT; goto error; } irdma_cm_init_tsa_conn(iwqp, cm_node); read0 = (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO); if (iwqp->page) iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(iwqp->page); irdma_sc_send_rtt(&iwqp->sc_qp, read0); if (iwqp->page) kunmap_local(iwqp->sc_qp.qp_uk.sq_base); attr.qp_state = IB_QPS_RTS; cm_node->qhash_set = false; irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL); if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) { wait_ret = wait_event_interruptible_timeout(iwqp->waitq, iwqp->rts_ae_rcvd, IRDMA_MAX_TIMEOUT); if (!wait_ret) irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, "Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n", cm_node, cm_node->loc_port, cm_node->rem_port, cm_node->cm_id); } irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, 0); cm_node->accelerated = true; complete(&cm_node->establish_comp); cm_node->cm_core->cm_free_ah(cm_node); return; error: iwqp->cm_id = NULL; cm_id->provider_data = NULL; irdma_send_cm_event(event->cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, status); irdma_rem_ref_cm_node(event->cm_node); } /** * irdma_cm_event_reset - handle reset * @event: the info for cm_node of connection */ static void irdma_cm_event_reset(struct irdma_cm_event *event) { struct irdma_cm_node *cm_node = event->cm_node; struct iw_cm_id *cm_id = cm_node->cm_id; struct irdma_qp *iwqp; if (!cm_id) return; iwqp = cm_id->provider_data; if (!iwqp) return; irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "reset event %p - cm_id = %p\n", event->cm_node, cm_id); iwqp->cm_id = NULL; irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_DISCONNECT, -ECONNRESET); irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CLOSE, 0); } /** * irdma_cm_event_handler - send event to cm upper layer * @work: pointer of cm event info. */ static void irdma_cm_event_handler(struct work_struct *work) { struct irdma_cm_event *event = container_of(work, struct irdma_cm_event, event_work); struct irdma_cm_node *cm_node; if (!event || !event->cm_node || !event->cm_node->cm_core) return; cm_node = event->cm_node; switch (event->type) { case IRDMA_CM_EVENT_MPA_REQ: irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CONNECT_REQUEST, 0); break; case IRDMA_CM_EVENT_RESET: irdma_cm_event_reset(event); break; case IRDMA_CM_EVENT_CONNECTED: if (!event->cm_node->cm_id || event->cm_node->state != IRDMA_CM_STATE_OFFLOADED) break; irdma_cm_event_connected(event); break; case IRDMA_CM_EVENT_MPA_REJECT: if (!event->cm_node->cm_id || cm_node->state == IRDMA_CM_STATE_OFFLOADED) break; irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED); break; case IRDMA_CM_EVENT_ABORTED: if (!event->cm_node->cm_id || event->cm_node->state == IRDMA_CM_STATE_OFFLOADED) break; irdma_event_connect_error(event); break; default: irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, "bad event type = %d\n", event->type); break; } irdma_rem_ref_cm_node(event->cm_node); kfree(event); } /** * irdma_cm_post_event - queue event request for worker thread * @event: cm node's info for up event call */ static void irdma_cm_post_event(struct irdma_cm_event *event) { atomic_inc(&event->cm_node->refcnt); INIT_WORK(&event->event_work, irdma_cm_event_handler); queue_work(event->cm_node->cm_core->event_wq, &event->event_work); } /** * irdma_cm_teardown_connections - teardown QPs * @iwdev: device pointer * @ipaddr: Pointer to IPv4 or IPv6 address * @nfo: Connection info * @disconnect_all: flag indicating disconnect all QPs * * teardown QPs where source or destination addr matches ip addr */ void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr, struct irdma_cm_info *nfo, bool disconnect_all) { struct irdma_cm_core *cm_core = &iwdev->cm_core; struct list_head *list_core_temp; struct list_head *list_node; struct irdma_cm_node *cm_node; struct list_head teardown_list; struct ib_qp_attr attr; struct irdma_sc_vsi *vsi = &iwdev->vsi; struct irdma_sc_qp *sc_qp; struct irdma_qp *qp; int i; INIT_LIST_HEAD(&teardown_list); rcu_read_lock(); irdma_teardown_list_prep(cm_core, &teardown_list, ipaddr, nfo, disconnect_all); rcu_read_unlock(); list_for_each_safe(list_node, list_core_temp, &teardown_list) { cm_node = container_of(list_node, struct irdma_cm_node, teardown_entry); attr.qp_state = IB_QPS_ERR; irdma_modify_qp(&cm_node->iwqp->ibqp, &attr, IB_QP_STATE, NULL); if (iwdev->rf->reset) irdma_cm_disconn(cm_node->iwqp); irdma_rem_ref_cm_node(cm_node); } if (!iwdev->roce_mode) return; INIT_LIST_HEAD(&teardown_list); for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { mutex_lock(&vsi->qos[i].qos_mutex); list_for_each_safe(list_node, list_core_temp, &vsi->qos[i].qplist) { u32 qp_ip[4]; sc_qp = container_of(list_node, struct irdma_sc_qp, list); if (sc_qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_RC) continue; qp = sc_qp->qp_uk.back_qp; if (!disconnect_all) { if (nfo->ipv4) qp_ip[0] = qp->udp_info.local_ipaddr[3]; else memcpy(qp_ip, &qp->udp_info.local_ipaddr[0], sizeof(qp_ip)); } if (disconnect_all || (nfo->vlan_id == (qp->udp_info.vlan_tag & EVL_VLID_MASK) && !memcmp(qp_ip, ipaddr, nfo->ipv4 ? 4 : 16))) { spin_lock(&iwdev->rf->qptable_lock); if (iwdev->rf->qp_table[sc_qp->qp_uk.qp_id]) { irdma_qp_add_ref(&qp->ibqp); list_add(&qp->teardown_entry, &teardown_list); } spin_unlock(&iwdev->rf->qptable_lock); } } mutex_unlock(&vsi->qos[i].qos_mutex); } list_for_each_safe(list_node, list_core_temp, &teardown_list) { qp = container_of(list_node, struct irdma_qp, teardown_entry); attr.qp_state = IB_QPS_ERR; irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); irdma_qp_rem_ref(&qp->ibqp); } } diff --git a/sys/dev/irdma/irdma_cm.h b/sys/dev/irdma/irdma_cm.h index e66db15a6fff..d3d55977cf3c 100644 --- a/sys/dev/irdma/irdma_cm.h +++ b/sys/dev/irdma/irdma_cm.h @@ -1,453 +1,453 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * * Copyright (c) 2015 - 2021 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenFabrics.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /*$FreeBSD$*/ #ifndef IRDMA_CM_H #define IRDMA_CM_H #define IRDMA_MPA_REQUEST_ACCEPT 1 #define IRDMA_MPA_REQUEST_REJECT 2 /* IETF MPA -- defines */ #define IEFT_MPA_KEY_REQ "MPA ID Req Frame" #define IEFT_MPA_KEY_REP "MPA ID Rep Frame" #define IETF_MPA_KEY_SIZE 16 #define IETF_MPA_VER 1 #define IETF_MAX_PRIV_DATA_LEN 512 #define IETF_MPA_FRAME_SIZE 20 #define IETF_RTR_MSG_SIZE 4 #define IETF_MPA_V2_FLAG 0x10 #define SNDMARKER_SEQNMASK 0x000001ff #define IRDMA_MAX_IETF_SIZE 32 /* IETF RTR MSG Fields */ #define IETF_PEER_TO_PEER 0x8000 #define IETF_FLPDU_ZERO_LEN 0x4000 #define IETF_RDMA0_WRITE 0x8000 #define IETF_RDMA0_READ 0x4000 #define IETF_NO_IRD_ORD 0x3fff #define MAX_PORTS 65536 #define IRDMA_PASSIVE_STATE_INDICATED 0 #define IRDMA_DO_NOT_SEND_RESET_EVENT 1 #define IRDMA_SEND_RESET_EVENT 2 #define MAX_IRDMA_IFS 4 #define SET_ACK 1 #define SET_SYN 2 #define SET_FIN 4 #define SET_RST 8 #define TCP_OPTIONS_PADDING 3 #define IRDMA_DEFAULT_RETRYS 64 #define IRDMA_DEFAULT_RETRANS 8 #define IRDMA_DEFAULT_TTL 0x40 #define IRDMA_DEFAULT_RTT_VAR 6 #define IRDMA_DEFAULT_SS_THRESH 0x3fffffff #define IRDMA_DEFAULT_REXMIT_THRESH 8 #define IRDMA_RETRY_TIMEOUT HZ #define IRDMA_SHORT_TIME 10 #define IRDMA_LONG_TIME (2 * HZ) #define IRDMA_MAX_TIMEOUT ((unsigned long)(12 * HZ)) #define IRDMA_CM_HASHTABLE_SIZE 1024 #define IRDMA_CM_TCP_TIMER_INTERVAL 3000 #define IRDMA_CM_DEFAULT_MTU 1540 #define IRDMA_CM_DEFAULT_FRAME_CNT 10 #define IRDMA_CM_THREAD_STACK_SIZE 256 #define IRDMA_CM_DEFAULT_RCV_WND 64240 #define IRDMA_CM_DEFAULT_RCV_WND_SCALED 0x3FFFC #define IRDMA_CM_DEFAULT_RCV_WND_SCALE 2 #define IRDMA_CM_DEFAULT_FREE_PKTS 10 #define IRDMA_CM_FREE_PKT_LO_WATERMARK 2 #define IRDMA_CM_DEFAULT_MSS 536 #define IRDMA_CM_DEFAULT_MPA_VER 2 #define IRDMA_CM_DEFAULT_SEQ 0x159bf75f #define IRDMA_CM_DEFAULT_LOCAL_ID 0x3b47 #define IRDMA_CM_DEFAULT_SEQ2 0x18ed5740 #define IRDMA_CM_DEFAULT_LOCAL_ID2 0xb807 #define IRDMA_MAX_CM_BUF (IRDMA_MAX_IETF_SIZE + IETF_MAX_PRIV_DATA_LEN) enum ietf_mpa_flags { IETF_MPA_FLAGS_REJECT = 0x20, IETF_MPA_FLAGS_CRC = 0x40, IETF_MPA_FLAGS_MARKERS = 0x80, }; enum irdma_timer_type { IRDMA_TIMER_TYPE_SEND, IRDMA_TIMER_TYPE_CLOSE, }; enum option_nums { OPTION_NUM_EOL, OPTION_NUM_NONE, OPTION_NUM_MSS, OPTION_NUM_WINDOW_SCALE, OPTION_NUM_SACK_PERM, OPTION_NUM_SACK, OPTION_NUM_WRITE0 = 0xbc, }; /* cm node transition states */ enum irdma_cm_node_state { IRDMA_CM_STATE_UNKNOWN, IRDMA_CM_STATE_INITED, IRDMA_CM_STATE_LISTENING, IRDMA_CM_STATE_SYN_RCVD, IRDMA_CM_STATE_SYN_SENT, IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED, IRDMA_CM_STATE_ESTABLISHED, IRDMA_CM_STATE_ACCEPTING, IRDMA_CM_STATE_MPAREQ_SENT, IRDMA_CM_STATE_MPAREQ_RCVD, IRDMA_CM_STATE_MPAREJ_RCVD, IRDMA_CM_STATE_OFFLOADED, IRDMA_CM_STATE_FIN_WAIT1, IRDMA_CM_STATE_FIN_WAIT2, IRDMA_CM_STATE_CLOSE_WAIT, IRDMA_CM_STATE_TIME_WAIT, IRDMA_CM_STATE_LAST_ACK, IRDMA_CM_STATE_CLOSING, IRDMA_CM_STATE_LISTENER_DESTROYED, IRDMA_CM_STATE_CLOSED, }; enum mpa_frame_ver { IETF_MPA_V1 = 1, IETF_MPA_V2 = 2, }; enum mpa_frame_key { MPA_KEY_REQUEST, MPA_KEY_REPLY, }; enum send_rdma0 { SEND_RDMA_READ_ZERO = 1, SEND_RDMA_WRITE_ZERO = 2, }; enum irdma_tcpip_pkt_type { IRDMA_PKT_TYPE_UNKNOWN, IRDMA_PKT_TYPE_SYN, IRDMA_PKT_TYPE_SYNACK, IRDMA_PKT_TYPE_ACK, IRDMA_PKT_TYPE_FIN, IRDMA_PKT_TYPE_RST, }; enum irdma_cm_listener_state { IRDMA_CM_LISTENER_PASSIVE_STATE = 1, IRDMA_CM_LISTENER_ACTIVE_STATE = 2, IRDMA_CM_LISTENER_EITHER_STATE = 3, }; /* CM event codes */ enum irdma_cm_event_type { IRDMA_CM_EVENT_UNKNOWN, IRDMA_CM_EVENT_ESTABLISHED, IRDMA_CM_EVENT_MPA_REQ, IRDMA_CM_EVENT_MPA_CONNECT, IRDMA_CM_EVENT_MPA_ACCEPT, IRDMA_CM_EVENT_MPA_REJECT, IRDMA_CM_EVENT_MPA_ESTABLISHED, IRDMA_CM_EVENT_CONNECTED, IRDMA_CM_EVENT_RESET, IRDMA_CM_EVENT_ABORTED, }; struct irdma_bth { /* Base Trasnport Header */ u8 opcode; u8 flags; __be16 pkey; __be32 qpn; __be32 apsn; }; struct ietf_mpa_v1 { u8 key[IETF_MPA_KEY_SIZE]; u8 flags; u8 rev; __be16 priv_data_len; u8 priv_data[]; }; struct ietf_rtr_msg { __be16 ctrl_ird; __be16 ctrl_ord; }; struct ietf_mpa_v2 { u8 key[IETF_MPA_KEY_SIZE]; u8 flags; u8 rev; __be16 priv_data_len; struct ietf_rtr_msg rtr_msg; u8 priv_data[]; }; struct option_base { u8 optionnum; u8 len; }; struct option_mss { u8 optionnum; u8 len; __be16 mss; }; struct option_windowscale { u8 optionnum; u8 len; u8 shiftcount; }; union all_known_options { char eol; struct option_base base; struct option_mss mss; struct option_windowscale windowscale; }; struct irdma_timer_entry { struct list_head list; unsigned long timetosend; /* jiffies */ struct irdma_puda_buf *sqbuf; u32 type; u32 retrycount; u32 retranscount; u32 context; u32 send_retrans; int close_when_complete; }; /* CM context params */ struct irdma_cm_tcp_context { u8 client; u32 loc_seq_num; u32 loc_ack_num; u32 rem_ack_num; u32 rcv_nxt; u32 loc_id; u32 rem_id; u32 snd_wnd; u32 max_snd_wnd; u32 rcv_wnd; u32 mss; u8 snd_wscale; u8 rcv_wscale; }; struct irdma_apbvt_entry { struct hlist_node hlist; u32 use_cnt; u16 port; }; struct irdma_cm_listener { struct list_head list; struct iw_cm_id *cm_id; struct irdma_cm_core *cm_core; struct irdma_device *iwdev; struct list_head child_listen_list; struct irdma_apbvt_entry *apbvt_entry; enum irdma_cm_listener_state listener_state; atomic_t refcnt; atomic_t pend_accepts_cnt; u32 loc_addr[4]; u32 reused_node; int backlog; u16 loc_port; u16 vlan_id; u8 loc_mac[ETH_ALEN]; u8 user_pri; u8 tos; bool qhash_set:1; bool ipv4:1; }; struct irdma_kmem_info { void *addr; u32 size; }; struct irdma_mpa_priv_info { const void *addr; u32 size; }; struct irdma_cm_node { struct irdma_qp *iwqp; struct irdma_device *iwdev; struct irdma_sc_dev *dev; struct irdma_cm_tcp_context tcp_cntxt; struct irdma_cm_core *cm_core; struct irdma_timer_entry *send_entry; struct irdma_timer_entry *close_entry; struct irdma_cm_listener *listener; struct list_head timer_entry; struct list_head reset_entry; struct list_head teardown_entry; struct irdma_apbvt_entry *apbvt_entry; struct rcu_head rcu_head; struct irdma_mpa_priv_info pdata; struct irdma_sc_ah *ah; struct irdma_kmem_info mpa_hdr; struct iw_cm_id *cm_id; struct hlist_node list; struct completion establish_comp; spinlock_t retrans_list_lock; /* protect CM node rexmit updates*/ atomic_t passive_state; atomic_t refcnt; enum irdma_cm_node_state state; enum send_rdma0 send_rdma0_op; enum mpa_frame_ver mpa_frame_rev; u32 loc_addr[4], rem_addr[4]; u16 loc_port, rem_port; int apbvt_set; int accept_pend; u16 vlan_id; u16 ird_size; u16 ord_size; u16 mpav2_ird_ord; u16 lsmm_size; u8 pdata_buf[IETF_MAX_PRIV_DATA_LEN]; u8 loc_mac[ETH_ALEN]; u8 rem_mac[ETH_ALEN]; u8 user_pri; u8 tos; bool ack_rcvd:1; bool qhash_set:1; bool ipv4:1; bool snd_mark_en:1; bool rcv_mark_en:1; bool do_lpb:1; bool accelerated:1; struct ietf_mpa_v2 mpa_v2_frame; }; /* Used by internal CM APIs to pass CM information*/ struct irdma_cm_info { struct iw_cm_id *cm_id; u16 loc_port; u16 rem_port; u32 loc_addr[4]; u32 rem_addr[4]; u32 qh_qpid; u16 vlan_id; int backlog; u8 user_pri; u8 tos; bool ipv4; }; struct irdma_cm_event { enum irdma_cm_event_type type; struct irdma_cm_info cm_info; struct work_struct event_work; struct irdma_cm_node *cm_node; }; struct irdma_cm_core { struct irdma_device *iwdev; struct irdma_sc_dev *dev; struct list_head listen_list; DECLARE_HASHTABLE(cm_hash_tbl, 8); DECLARE_HASHTABLE(apbvt_hash_tbl, 8); struct timer_list tcp_timer; struct workqueue_struct *event_wq; spinlock_t ht_lock; /* protect CM node (active side) list */ spinlock_t listen_list_lock; /* protect listener list */ spinlock_t apbvt_lock; /*serialize apbvt add/del entries*/ u64 stats_nodes_created; u64 stats_nodes_destroyed; u64 stats_listen_created; u64 stats_listen_destroyed; u64 stats_listen_nodes_created; u64 stats_listen_nodes_destroyed; u64 stats_lpbs; u64 stats_accepts; u64 stats_rejects; u64 stats_connect_errs; u64 stats_passive_errs; u64 stats_pkt_retrans; u64 stats_backlog_drops; struct irdma_puda_buf *(*form_cm_frame)(struct irdma_cm_node *cm_node, struct irdma_kmem_info *options, struct irdma_kmem_info *hdr, struct irdma_mpa_priv_info *pdata, u8 flags); int (*cm_create_ah)(struct irdma_cm_node *cm_node, bool wait); void (*cm_free_ah)(struct irdma_cm_node *cm_node); }; int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node, struct irdma_puda_buf *sqbuf, enum irdma_timer_type type, int send_retrans, int close_when_complete); static inline u8 irdma_tos2dscp(u8 tos) { #define IRDMA_DSCP_S 2 #define IRDMA_DSCP_M (0x3f << IRDMA_DSCP_S) return RS_32(tos, IRDMA_DSCP); } int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param); int irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len); int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param); int irdma_create_listen(struct iw_cm_id *cm_id, int backlog); int irdma_destroy_listen(struct iw_cm_id *cm_id); int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, u8 *mac); void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr, struct irdma_cm_info *nfo, bool disconnect_all); int irdma_cm_start(struct irdma_device *dev); int irdma_cm_stop(struct irdma_device *dev); -bool irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr); +bool irdma_ipv4_is_lpb(struct vnet *, u32 loc_addr, u32 rem_addr); bool irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr); int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, u8 *mac_addr, u32 action); bool irdma_port_in_use(struct irdma_cm_core *cm_core, u16 port); void irdma_send_ack(struct irdma_cm_node *cm_node); void irdma_lpb_nop(struct irdma_sc_qp *qp); void irdma_rem_ref_cm_node(struct irdma_cm_node *cm_node); void irdma_add_conn_est_qh(struct irdma_cm_node *cm_node); #endif /* IRDMA_CM_H */ diff --git a/sys/dev/irdma/irdma_kcompat.c b/sys/dev/irdma/irdma_kcompat.c index beff04f7dc6c..3d88187cb9ae 100644 --- a/sys/dev/irdma/irdma_kcompat.c +++ b/sys/dev/irdma/irdma_kcompat.c @@ -1,1568 +1,1571 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * * Copyright (c) 2018 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenFabrics.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /*$FreeBSD$*/ #include "irdma_main.h" void irdma_get_dev_fw_str(struct ib_device *dev, char *str, size_t str_len) { struct irdma_device *iwdev = to_iwdev(dev); snprintf(str, str_len, "%u.%u", irdma_fw_major_ver(&iwdev->rf->sc_dev), irdma_fw_minor_ver(&iwdev->rf->sc_dev)); } int irdma_add_gid(struct ib_device *device, u8 port_num, unsigned int index, const union ib_gid *gid, const struct ib_gid_attr *attr, void **context) { return 0; } int irdma_del_gid(struct ib_device *device, u8 port_num, unsigned int index, void **context) { return 0; } /** * irdma_alloc_mr - register stag for fast memory registration * @pd: ibpd pointer * @mr_type: memory for stag registrion * @max_num_sg: man number of pages * @udata: user data */ struct ib_mr * irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg, struct ib_udata *udata) { struct irdma_device *iwdev = to_iwdev(pd->device); struct irdma_pble_alloc *palloc; struct irdma_pbl *iwpbl; struct irdma_mr *iwmr; int status; u32 stag; int err_code = -ENOMEM; iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); if (!iwmr) return ERR_PTR(-ENOMEM); stag = irdma_create_stag(iwdev); if (!stag) { err_code = -ENOMEM; goto err; } iwmr->stag = stag; iwmr->ibmr.rkey = stag; iwmr->ibmr.lkey = stag; iwmr->ibmr.pd = pd; iwmr->ibmr.device = pd->device; iwpbl = &iwmr->iwpbl; iwpbl->iwmr = iwmr; iwmr->type = IRDMA_MEMREG_TYPE_MEM; palloc = &iwpbl->pble_alloc; iwmr->page_cnt = max_num_sg; status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt, true); if (status) goto err_get_pble; err_code = irdma_hw_alloc_stag(iwdev, iwmr); if (err_code) goto err_alloc_stag; iwpbl->pbl_allocated = true; return &iwmr->ibmr; err_alloc_stag: irdma_free_pble(iwdev->rf->pble_rsrc, palloc); err_get_pble: irdma_free_stag(iwdev, stag); err: kfree(iwmr); return ERR_PTR(err_code); } /** * irdma_alloc_ucontext - Allocate the user context data structure * @uctx: context * @udata: user data * * This keeps track of all objects associated with a particular * user-mode client. */ int irdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) { struct ib_device *ibdev = uctx->device; struct irdma_device *iwdev = to_iwdev(ibdev); struct irdma_alloc_ucontext_req req; struct irdma_alloc_ucontext_resp uresp = {0}; struct irdma_ucontext *ucontext = to_ucontext(uctx); struct irdma_uk_attrs *uk_attrs; if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) return -EINVAL; if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) goto ver_error; ucontext->iwdev = iwdev; ucontext->abi_ver = req.userspace_ver; uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; /* GEN_1 support for libi40iw */ if (udata->outlen < sizeof(uresp)) { if (uk_attrs->hw_rev != IRDMA_GEN_1) return -EOPNOTSUPP; ucontext->legacy_mode = true; uresp.max_qps = iwdev->rf->max_qp; uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; uresp.kernel_ver = req.userspace_ver; if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) return -EFAULT; } else { u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; ucontext->db_mmap_entry = irdma_user_mmap_entry_insert(ucontext, bar_off, IRDMA_MMAP_IO_NC, &uresp.db_mmap_key); if (!ucontext->db_mmap_entry) { return -ENOMEM; } uresp.kernel_ver = IRDMA_ABI_VER; uresp.feature_flags = uk_attrs->feature_flags; uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; uresp.max_hw_inline = uk_attrs->max_hw_inline; uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; uresp.hw_rev = uk_attrs->hw_rev; if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) { rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); return -EFAULT; } } INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); spin_lock_init(&ucontext->cq_reg_mem_list_lock); INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); spin_lock_init(&ucontext->qp_reg_mem_list_lock); INIT_LIST_HEAD(&ucontext->vma_list); mutex_init(&ucontext->vma_list_mutex); return 0; ver_error: irdma_dev_err(&iwdev->rf->sc_dev, "Invalid userspace driver version detected. Detected version %d, should be %d\n", req.userspace_ver, IRDMA_ABI_VER); return -EINVAL; } /** * irdma_dealloc_ucontext - deallocate the user context data structure * @context: user context created during alloc */ void irdma_dealloc_ucontext(struct ib_ucontext *context) { struct irdma_ucontext *ucontext = to_ucontext(context); rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); return; } /** * irdma_alloc_pd - allocate protection domain * @pd: protection domain * @udata: user data */ int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) { struct irdma_pd *iwpd = to_iwpd(pd); struct irdma_device *iwdev = to_iwdev(pd->device); struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; struct irdma_pci_f *rf = iwdev->rf; struct irdma_alloc_pd_resp uresp = {0}; struct irdma_sc_pd *sc_pd; u32 pd_id = 0; int err; err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, &rf->next_pd); if (err) return err; sc_pd = &iwpd->sc_pd; if (udata) { struct irdma_ucontext *ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); uresp.pd_id = pd_id; if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) { err = -EFAULT; goto error; } } else { irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); } return 0; error: irdma_free_rsrc(rf, rf->allocated_pds, pd_id); return err; } void irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct irdma_pd *iwpd = to_iwpd(ibpd); struct irdma_device *iwdev = to_iwdev(ibpd->device); irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); } static void -irdma_fill_ah_info(struct irdma_ah_info *ah_info, +irdma_fill_ah_info(struct vnet *vnet, + struct irdma_ah_info *ah_info, const struct ib_gid_attr *sgid_attr, struct sockaddr *sgid_addr, struct sockaddr *dgid_addr, u8 *dmac, u8 net_type) { if (net_type == RDMA_NETWORK_IPV4) { ah_info->ipv4_valid = true; ah_info->dest_ip_addr[0] = ntohl(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr); ah_info->src_ip_addr[0] = ntohl(((struct sockaddr_in *)sgid_addr)->sin_addr.s_addr); - ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], + ah_info->do_lpbk = irdma_ipv4_is_lpb(vnet, + ah_info->src_ip_addr[0], ah_info->dest_ip_addr[0]); if (ipv4_is_multicast(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr)) { irdma_mcast_mac_v4(ah_info->dest_ip_addr, dmac); } } else { irdma_copy_ip_ntohl(ah_info->dest_ip_addr, ((struct sockaddr_in6 *)dgid_addr)->sin6_addr.__u6_addr.__u6_addr32); irdma_copy_ip_ntohl(ah_info->src_ip_addr, ((struct sockaddr_in6 *)sgid_addr)->sin6_addr.__u6_addr.__u6_addr32); ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, ah_info->dest_ip_addr); if (rdma_is_multicast_addr(&((struct sockaddr_in6 *)dgid_addr)->sin6_addr)) { irdma_mcast_mac_v6(ah_info->dest_ip_addr, dmac); } } } static int irdma_create_ah_vlan_tag(struct irdma_device *iwdev, struct irdma_ah_info *ah_info, const struct ib_gid_attr *sgid_attr, u8 *dmac) { if (sgid_attr->ndev && is_vlan_dev(sgid_attr->ndev)) ah_info->vlan_tag = vlan_dev_vlan_id(sgid_attr->ndev); else ah_info->vlan_tag = VLAN_N_VID; ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, dmac); if (ah_info->dst_arpindex == -1) return -EINVAL; if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode) ah_info->vlan_tag = 0; if (ah_info->vlan_tag < VLAN_N_VID) { ah_info->insert_vlan_tag = true; ah_info->vlan_tag |= rt_tos2priority(ah_info->tc_tos) << VLAN_PRIO_SHIFT; } return 0; } static int irdma_create_ah_wait(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, bool sleep) { if (!sleep) { int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD; do { irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); mdelay(1); } while (!sc_ah->ah_info.ah_valid && --cnt); if (!cnt) return -ETIMEDOUT; } return 0; } /** * irdma_create_ah - create address handle * @ib_ah: ptr to AH * @attr: address handle attributes * @flags: AH flags to wait * @udata: user data * * returns 0 on success, error otherwise */ int irdma_create_ah(struct ib_ah *ib_ah, struct ib_ah_attr *attr, u32 flags, struct ib_udata *udata) { struct irdma_pd *pd = to_iwpd(ib_ah->pd); struct irdma_ah *ah = container_of(ib_ah, struct irdma_ah, ibah); struct irdma_device *iwdev = to_iwdev(ib_ah->pd->device); union ib_gid sgid; struct ib_gid_attr sgid_attr; struct irdma_pci_f *rf = iwdev->rf; struct irdma_sc_ah *sc_ah; u32 ah_id = 0; struct irdma_ah_info *ah_info; struct irdma_create_ah_resp uresp; union { struct sockaddr saddr; struct sockaddr_in saddr_in; struct sockaddr_in6 saddr_in6; } sgid_addr, dgid_addr; int err; u8 dmac[ETH_ALEN]; bool sleep; err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, &ah_id, &rf->next_ah); if (err) return err; ah->pd = pd; sc_ah = &ah->sc_ah; sc_ah->ah_info.ah_idx = ah_id; sc_ah->ah_info.vsi = &iwdev->vsi; irdma_sc_init_ah(&rf->sc_dev, sc_ah); ah->sgid_index = attr->grh.sgid_index; memcpy(&ah->dgid, &attr->grh.dgid, sizeof(ah->dgid)); rcu_read_lock(); err = ib_get_cached_gid(&iwdev->ibdev, attr->port_num, attr->grh.sgid_index, &sgid, &sgid_attr); rcu_read_unlock(); if (err) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "GID lookup at idx=%d with port=%d failed\n", attr->grh.sgid_index, attr->port_num); err = -EINVAL; goto error; } rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid); rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid); ah->av.attrs = *attr; ah->av.net_type = kc_rdma_gid_attr_network_type(sgid_attr, sgid_attr.gid_type, &sgid); if (sgid_attr.ndev) dev_put(sgid_attr.ndev); ah->av.sgid_addr.saddr = sgid_addr.saddr; ah->av.dgid_addr.saddr = dgid_addr.saddr; ah_info = &sc_ah->ah_info; ah_info->ah_idx = ah_id; ah_info->pd_idx = pd->sc_pd.pd_id; ether_addr_copy(ah_info->mac_addr, IF_LLADDR(iwdev->netdev)); if (attr->ah_flags & IB_AH_GRH) { ah_info->flow_label = attr->grh.flow_label; ah_info->hop_ttl = attr->grh.hop_limit; ah_info->tc_tos = attr->grh.traffic_class; } ether_addr_copy(dmac, attr->dmac); - irdma_fill_ah_info(ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr, + irdma_fill_ah_info(iwdev->netdev->if_vnet, + ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr, dmac, ah->av.net_type); err = irdma_create_ah_vlan_tag(iwdev, ah_info, &sgid_attr, dmac); if (err) goto error; sleep = flags & RDMA_CREATE_AH_SLEEPABLE; err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE, sleep, irdma_gsi_ud_qp_ah_cb, sc_ah); if (err) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "CQP-OP Create AH fail"); goto error; } err = irdma_create_ah_wait(rf, sc_ah, sleep); if (err) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "CQP create AH timed out"); goto error; } if (udata) { uresp.ah_id = ah->sc_ah.ah_info.ah_idx; err = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); } return 0; error: irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id); return err; } void irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr) { ether_addr_copy(dmac, attr->dmac); } int irdma_create_ah_stub(struct ib_ah *ib_ah, struct ib_ah_attr *attr, u32 flags, struct ib_udata *udata) { return -ENOSYS; } void irdma_destroy_ah_stub(struct ib_ah *ibah, u32 flags) { return; } /** * irdma_free_qp_rsrc - free up memory resources for qp * @iwqp: qp ptr (user or kernel) */ void irdma_free_qp_rsrc(struct irdma_qp *iwqp) { struct irdma_device *iwdev = iwqp->iwdev; struct irdma_pci_f *rf = iwdev->rf; u32 qp_num = iwqp->ibqp.qp_num; irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); irdma_dealloc_push_page(rf, &iwqp->sc_qp); if (iwqp->sc_qp.vsi) { irdma_qp_rem_qos(&iwqp->sc_qp); iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, iwqp->sc_qp.user_pri); } if (qp_num > 2) irdma_free_rsrc(rf, rf->allocated_qps, qp_num); irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->q2_ctx_mem); irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->kqp.dma_mem); kfree(iwqp->kqp.sig_trk_mem); iwqp->kqp.sig_trk_mem = NULL; kfree(iwqp->kqp.sq_wrid_mem); kfree(iwqp->kqp.rq_wrid_mem); kfree(iwqp->sg_list); kfree(iwqp); } /** * irdma_create_qp - create qp * @ibpd: ptr of pd * @init_attr: attributes for qp * @udata: user data for create qp */ struct ib_qp * irdma_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { struct irdma_pd *iwpd = to_iwpd(ibpd); struct irdma_device *iwdev = to_iwdev(ibpd->device); struct irdma_pci_f *rf = iwdev->rf; struct irdma_qp *iwqp; struct irdma_create_qp_req req; struct irdma_create_qp_resp uresp = {0}; u32 qp_num = 0; int ret; int err_code; int sq_size; int rq_size; struct irdma_sc_qp *qp; struct irdma_sc_dev *dev = &rf->sc_dev; struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; struct irdma_qp_init_info init_info = {{0}}; struct irdma_qp_host_ctx_info *ctx_info; unsigned long flags; err_code = irdma_validate_qp_attrs(init_attr, iwdev); if (err_code) return ERR_PTR(err_code); sq_size = init_attr->cap.max_send_wr; rq_size = init_attr->cap.max_recv_wr; init_info.vsi = &iwdev->vsi; init_info.qp_uk_init_info.uk_attrs = uk_attrs; init_info.qp_uk_init_info.sq_size = sq_size; init_info.qp_uk_init_info.rq_size = rq_size; init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL); if (!iwqp) return ERR_PTR(-ENOMEM); iwqp->sg_list = kcalloc(uk_attrs->max_hw_wq_frags, sizeof(*iwqp->sg_list), GFP_KERNEL); if (!iwqp->sg_list) { kfree(iwqp); return ERR_PTR(-ENOMEM); } qp = &iwqp->sc_qp; qp->qp_uk.back_qp = iwqp; qp->qp_uk.lock = &iwqp->lock; qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; iwqp->iwdev = iwdev; iwqp->q2_ctx_mem.size = IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE; iwqp->q2_ctx_mem.va = irdma_allocate_dma_mem(dev->hw, &iwqp->q2_ctx_mem, iwqp->q2_ctx_mem.size, 256); if (!iwqp->q2_ctx_mem.va) { kfree(iwqp->sg_list); kfree(iwqp); return ERR_PTR(-ENOMEM); } init_info.q2 = iwqp->q2_ctx_mem.va; init_info.q2_pa = iwqp->q2_ctx_mem.pa; init_info.host_ctx = (__le64 *) (init_info.q2 + IRDMA_Q2_BUF_SIZE); init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE; if (init_attr->qp_type == IB_QPT_GSI) qp_num = 1; else err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, &qp_num, &rf->next_qp); if (err_code) goto error; iwqp->iwpd = iwpd; iwqp->ibqp.qp_num = qp_num; qp = &iwqp->sc_qp; iwqp->iwscq = to_iwcq(init_attr->send_cq); iwqp->iwrcq = to_iwcq(init_attr->recv_cq); iwqp->host_ctx.va = init_info.host_ctx; iwqp->host_ctx.pa = init_info.host_ctx_pa; iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; init_info.pd = &iwpd->sc_pd; init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num; if (!rdma_protocol_roce(&iwdev->ibdev, 1)) init_info.qp_uk_init_info.first_sq_wq = 1; iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; init_waitqueue_head(&iwqp->waitq); init_waitqueue_head(&iwqp->mod_qp_waitq); if (udata) { err_code = ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen)); if (err_code) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "ib_copy_from_data fail\n"); goto error; } iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; iwqp->user_mode = 1; if (req.user_wqe_bufs) { struct irdma_ucontext *ucontext = to_ucontext(ibpd->uobject->context); init_info.qp_uk_init_info.legacy_mode = ucontext->legacy_mode; spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, &ucontext->qp_reg_mem_list); spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); if (!iwqp->iwpbl) { err_code = -ENODATA; irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "no pbl info\n"); goto error; } } init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; irdma_setup_virt_qp(iwdev, iwqp, &init_info); } else { INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker); init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); } if (err_code) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "setup qp failed\n"); goto error; } if (rdma_protocol_roce(&iwdev->ibdev, 1)) { if (init_attr->qp_type == IB_QPT_RC) { init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC; init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | IRDMA_WRITE_WITH_IMM | IRDMA_ROCE; } else { init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD; init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | IRDMA_ROCE; } } else { init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP; init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM; } ret = irdma_sc_qp_init(qp, &init_info); if (ret) { err_code = -EPROTO; irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "qp_init fail\n"); goto error; } ctx_info = &iwqp->ctx_info; ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; if (rdma_protocol_roce(&iwdev->ibdev, 1)) irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); else irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); err_code = irdma_cqp_create_qp_cmd(iwqp); if (err_code) goto error; atomic_set(&iwqp->refcnt, 1); spin_lock_init(&iwqp->lock); spin_lock_init(&iwqp->sc_qp.pfpdu.lock); iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; rf->qp_table[qp_num] = iwqp; iwqp->max_send_wr = sq_size; iwqp->max_recv_wr = rq_size; if (rdma_protocol_roce(&iwdev->ibdev, 1)) { if (dev->ws_add(&iwdev->vsi, 0)) { irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); err_code = -EINVAL; goto error; } irdma_qp_add_qos(&iwqp->sc_qp); } if (udata) { /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ if (udata->outlen < sizeof(uresp)) { uresp.lsmm = 1; uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; } else { if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) uresp.lsmm = 1; } uresp.actual_sq_size = sq_size; uresp.actual_rq_size = rq_size; uresp.qp_id = qp_num; uresp.qp_caps = qp->qp_uk.qp_caps; err_code = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)); if (err_code) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "copy_to_udata failed\n"); kc_irdma_destroy_qp(&iwqp->ibqp, udata); return ERR_PTR(err_code); } } init_completion(&iwqp->free_qp); return &iwqp->ibqp; error: irdma_free_qp_rsrc(iwqp); return ERR_PTR(err_code); } /** * irdma_destroy_qp - destroy qp * @ibqp: qp's ib pointer also to get to device's qp address * @udata: user data */ int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) { struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_device *iwdev = iwqp->iwdev; if (iwqp->sc_qp.qp_uk.destroy_pending) goto free_rsrc; iwqp->sc_qp.qp_uk.destroy_pending = true; if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) irdma_modify_qp_to_err(&iwqp->sc_qp); if (!iwqp->user_mode) cancel_delayed_work_sync(&iwqp->dwork_flush); irdma_qp_rem_ref(&iwqp->ibqp); wait_for_completion(&iwqp->free_qp); irdma_free_lsmm_rsrc(iwqp); if (!iwdev->rf->reset && irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp)) return -ENOTRECOVERABLE; free_rsrc: if (!iwqp->user_mode) { if (iwqp->iwscq) { irdma_clean_cqes(iwqp, iwqp->iwscq); if (iwqp->iwrcq != iwqp->iwscq) irdma_clean_cqes(iwqp, iwqp->iwrcq); } } irdma_remove_push_mmap_entries(iwqp); irdma_free_qp_rsrc(iwqp); return 0; } /** * irdma_create_cq - create cq * @ibcq: CQ allocated * @attr: attributes for cq * @udata: user data */ int irdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata) { struct ib_device *ibdev = ibcq->device; struct irdma_device *iwdev = to_iwdev(ibdev); struct irdma_pci_f *rf = iwdev->rf; struct irdma_cq *iwcq = to_iwcq(ibcq); u32 cq_num = 0; struct irdma_sc_cq *cq; struct irdma_sc_dev *dev = &rf->sc_dev; struct irdma_cq_init_info info = {0}; int status; struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; unsigned long flags; int err_code; int entries = attr->cqe; err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); if (err_code) return err_code; err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, &rf->next_cq); if (err_code) return err_code; cq = &iwcq->sc_cq; cq->back_cq = iwcq; atomic_set(&iwcq->refcnt, 1); spin_lock_init(&iwcq->lock); INIT_LIST_HEAD(&iwcq->resize_list); INIT_LIST_HEAD(&iwcq->cmpl_generated); info.dev = dev; ukinfo->cq_size = max(entries, 4); ukinfo->cq_id = cq_num; iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; if (attr->comp_vector < rf->ceqs_count) info.ceq_id = attr->comp_vector; info.ceq_id_valid = true; info.ceqe_mask = 1; info.type = IRDMA_CQ_TYPE_IWARP; info.vsi = &iwdev->vsi; if (udata) { struct irdma_ucontext *ucontext; struct irdma_create_cq_req req = {0}; struct irdma_cq_mr *cqmr; struct irdma_pbl *iwpbl; struct irdma_pbl *iwpbl_shadow; struct irdma_cq_mr *cqmr_shadow; iwcq->user_mode = true; ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) { err_code = -EFAULT; goto cq_free_rsrc; } spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, &ucontext->cq_reg_mem_list); spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); if (!iwpbl) { err_code = -EPROTO; goto cq_free_rsrc; } iwcq->iwpbl = iwpbl; iwcq->cq_mem_size = 0; cqmr = &iwpbl->cq_mr; if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); iwpbl_shadow = irdma_get_pbl((unsigned long)req.user_shadow_area, &ucontext->cq_reg_mem_list); spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); if (!iwpbl_shadow) { err_code = -EPROTO; goto cq_free_rsrc; } iwcq->iwpbl_shadow = iwpbl_shadow; cqmr_shadow = &iwpbl_shadow->cq_mr; info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; cqmr->split = true; } else { info.shadow_area_pa = cqmr->shadow; } if (iwpbl->pbl_allocated) { info.virtual_map = true; info.pbl_chunk_size = 1; info.first_pm_pbl_idx = cqmr->cq_pbl.idx; } else { info.cq_base_pa = cqmr->cq_pbl.addr; } } else { /* Kmode allocations */ int rsize; if (entries < 1 || entries > rf->max_cqe) { err_code = -EINVAL; goto cq_free_rsrc; } entries++; if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) entries *= 2; ukinfo->cq_size = entries; rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe); iwcq->kmem.size = round_up(rsize, 256); iwcq->kmem.va = irdma_allocate_dma_mem(dev->hw, &iwcq->kmem, iwcq->kmem.size, 256); if (!iwcq->kmem.va) { err_code = -ENOMEM; goto cq_free_rsrc; } iwcq->kmem_shadow.size = IRDMA_SHADOW_AREA_SIZE << 3; iwcq->kmem_shadow.va = irdma_allocate_dma_mem(dev->hw, &iwcq->kmem_shadow, iwcq->kmem_shadow.size, 64); if (!iwcq->kmem_shadow.va) { err_code = -ENOMEM; goto cq_free_rsrc; } info.shadow_area_pa = iwcq->kmem_shadow.pa; ukinfo->shadow_area = iwcq->kmem_shadow.va; ukinfo->cq_base = iwcq->kmem.va; info.cq_base_pa = iwcq->kmem.pa; } if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, (u32)IRDMA_MAX_CQ_READ_THRESH); if (irdma_sc_cq_init(cq, &info)) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "init cq fail\n"); err_code = -EPROTO; goto cq_free_rsrc; } cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); if (!cqp_request) { err_code = -ENOMEM; goto cq_free_rsrc; } cqp_info = &cqp_request->info; cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; cqp_info->post_sq = 1; cqp_info->in.u.cq_create.cq = cq; cqp_info->in.u.cq_create.check_overflow = true; cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; status = irdma_handle_cqp_op(rf, cqp_request); irdma_put_cqp_request(&rf->cqp, cqp_request); if (status) { err_code = -ENOMEM; goto cq_free_rsrc; } if (udata) { struct irdma_create_cq_resp resp = {0}; resp.cq_id = info.cq_uk_init_info.cq_id; resp.cq_size = info.cq_uk_init_info.cq_size; if (ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen))) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "copy to user data\n"); err_code = -EPROTO; goto cq_destroy; } } rf->cq_table[cq_num] = iwcq; init_completion(&iwcq->free_cq); return 0; cq_destroy: irdma_cq_wq_destroy(rf, cq); cq_free_rsrc: irdma_cq_free_rsrc(rf, iwcq); return err_code; } /** * irdma_copy_user_pgaddrs - copy user page address to pble's os locally * @iwmr: iwmr for IB's user page addresses * @pbl: ple pointer to save 1 level or 0 level pble * @level: indicated level 0, 1 or 2 */ void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, enum irdma_pble_level level) { struct ib_umem *region = iwmr->region; struct irdma_pbl *iwpbl = &iwmr->iwpbl; int chunk_pages, entry, i; struct scatterlist *sg; u64 pg_addr = 0; struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; struct irdma_pble_info *pinfo; u32 idx = 0; u32 pbl_cnt = 0; pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) { chunk_pages = DIV_ROUND_UP(sg_dma_len(sg), iwmr->page_size); if (iwmr->type == IRDMA_MEMREG_TYPE_QP && !iwpbl->qp_mr.sq_page) iwpbl->qp_mr.sq_page = sg_page(sg); for (i = 0; i < chunk_pages; i++) { pg_addr = sg_dma_address(sg) + (i * iwmr->page_size); if ((entry + i) == 0) *pbl = pg_addr & iwmr->page_msk; else if (!(pg_addr & ~iwmr->page_msk)) *pbl = pg_addr; else continue; if (++pbl_cnt == palloc->total_cnt) break; pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); } } } /** * irdma_destroy_ah - Destroy address handle * @ibah: pointer to address handle * @ah_flags: destroy flags */ void irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags) { struct irdma_device *iwdev = to_iwdev(ibah->device); struct irdma_ah *ah = to_iwah(ibah); irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, false, NULL, ah); irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah->sc_ah.ah_info.ah_idx); } int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) { struct ib_pd *ibpd = ib_mr->pd; struct irdma_pd *iwpd = to_iwpd(ibpd); struct irdma_mr *iwmr = to_iwmr(ib_mr); struct irdma_device *iwdev = to_iwdev(ib_mr->device); struct irdma_dealloc_stag_info *info; struct irdma_pbl *iwpbl = &iwmr->iwpbl; struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; int status; if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { if (iwmr->region) { struct irdma_ucontext *ucontext; ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); irdma_del_memlist(iwmr, ucontext); } goto done; } cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); if (!cqp_request) return -ENOMEM; cqp_info = &cqp_request->info; info = &cqp_info->in.u.dealloc_stag.info; memset(info, 0, sizeof(*info)); info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff; info->stag_idx = RS_64_1(ib_mr->rkey, IRDMA_CQPSQ_STAG_IDX_S); info->mr = true; if (iwpbl->pbl_allocated) info->dealloc_pbl = true; cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; cqp_info->post_sq = 1; cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; status = irdma_handle_cqp_op(iwdev->rf, cqp_request); irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); if (status) return status; irdma_free_stag(iwdev, iwmr->stag); done: if (iwpbl->pbl_allocated) irdma_free_pble(iwdev->rf->pble_rsrc, palloc); if (iwmr->region) ib_umem_release(iwmr->region); kfree(iwmr); return 0; } int kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp, struct ib_qp_attr *attr, u16 *vlan_id) { int ret; union ib_gid sgid; struct ib_gid_attr sgid_attr; struct irdma_av *av = &iwqp->roce_ah.av; ret = ib_get_cached_gid(iwqp->ibqp.device, attr->ah_attr.port_num, attr->ah_attr.grh.sgid_index, &sgid, &sgid_attr); if (ret) return ret; if (sgid_attr.ndev) { *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr.ndev); ether_addr_copy(iwqp->ctx_info.roce_info->mac_addr, IF_LLADDR(sgid_attr.ndev)); } rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid); dev_put(sgid_attr.ndev); return 0; } /** * irdma_destroy_cq - destroy cq * @ib_cq: cq pointer * @udata: user data */ void irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct irdma_device *iwdev = to_iwdev(ib_cq->device); struct irdma_cq *iwcq = to_iwcq(ib_cq); struct irdma_sc_cq *cq = &iwcq->sc_cq; struct irdma_sc_dev *dev = cq->dev; struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); unsigned long flags; spin_lock_irqsave(&iwcq->lock, flags); if (!list_empty(&iwcq->cmpl_generated)) irdma_remove_cmpls_list(iwcq); if (!list_empty(&iwcq->resize_list)) irdma_process_resize_list(iwcq, iwdev, NULL); spin_unlock_irqrestore(&iwcq->lock, flags); irdma_cq_rem_ref(ib_cq); wait_for_completion(&iwcq->free_cq); irdma_cq_wq_destroy(iwdev->rf, cq); irdma_cq_free_rsrc(iwdev->rf, iwcq); spin_lock_irqsave(&iwceq->ce_lock, flags); irdma_sc_cleanup_ceqes(cq, ceq); spin_unlock_irqrestore(&iwceq->ce_lock, flags); } /** * irdma_alloc_mw - Allocate memory window * @pd: Protection domain * @type: Window type * @udata: user data pointer */ struct ib_mw * irdma_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, struct ib_udata *udata) { struct irdma_device *iwdev = to_iwdev(pd->device); struct irdma_mr *iwmr; int err_code; u32 stag; iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); if (!iwmr) return ERR_PTR(-ENOMEM); stag = irdma_create_stag(iwdev); if (!stag) { kfree(iwmr); return ERR_PTR(-ENOMEM); } iwmr->stag = stag; iwmr->ibmw.rkey = stag; iwmr->ibmw.pd = pd; iwmr->ibmw.type = type; iwmr->ibmw.device = pd->device; err_code = irdma_hw_alloc_mw(iwdev, iwmr); if (err_code) { irdma_free_stag(iwdev, stag); kfree(iwmr); return ERR_PTR(err_code); } return &iwmr->ibmw; } /** * kc_set_loc_seq_num_mss - Set local seq number and mss * @cm_node: cm node info */ void kc_set_loc_seq_num_mss(struct irdma_cm_node *cm_node) { struct timespec ts; getnanotime(&ts); cm_node->tcp_cntxt.loc_seq_num = ts.tv_nsec; if (cm_node->iwdev->vsi.mtu > 1500 && 2 * cm_node->iwdev->vsi.mtu > cm_node->iwdev->rcv_wnd) cm_node->tcp_cntxt.mss = (cm_node->ipv4) ? (1500 - IRDMA_MTU_TO_MSS_IPV4) : (1500 - IRDMA_MTU_TO_MSS_IPV6); else cm_node->tcp_cntxt.mss = (cm_node->ipv4) ? (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV4) : (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6); } /** * irdma_disassociate_ucontext - Disassociate user context * @context: ib user context */ void irdma_disassociate_ucontext(struct ib_ucontext *context) { } struct ib_device * ib_device_get_by_netdev(struct ifnet *netdev, int driver_id) { struct irdma_device *iwdev; struct irdma_handler *hdl; unsigned long flags; spin_lock_irqsave(&irdma_handler_lock, flags); list_for_each_entry(hdl, &irdma_handlers, list) { iwdev = hdl->iwdev; if (netdev == iwdev->netdev) { spin_unlock_irqrestore(&irdma_handler_lock, flags); return &iwdev->ibdev; } } spin_unlock_irqrestore(&irdma_handler_lock, flags); return NULL; } void ib_unregister_device_put(struct ib_device *device) { ib_unregister_device(device); } /** * irdma_query_gid_roce - Query port GID for Roce * @ibdev: device pointer from stack * @port: port number * @index: Entry index * @gid: Global ID */ int irdma_query_gid_roce(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid) { int ret; ret = rdma_query_gid(ibdev, port, index, gid); if (ret == -EAGAIN) { memcpy(gid, &zgid, sizeof(*gid)); return 0; } return ret; } /** * irdma_modify_port - modify port attributes * @ibdev: device pointer from stack * @port: port number for query * @mask: Property mask * @props: returning device attributes */ int irdma_modify_port(struct ib_device *ibdev, u8 port, int mask, struct ib_port_modify *props) { if (port > 1) return -EINVAL; return 0; } /** * irdma_query_pkey - Query partition key * @ibdev: device pointer from stack * @port: port number * @index: index of pkey * @pkey: pointer to store the pkey */ int irdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) { if (index >= IRDMA_PKEY_TBL_SZ) return -EINVAL; *pkey = IRDMA_DEFAULT_PKEY; return 0; } int irdma_roce_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; int err; immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; err = ib_query_port(ibdev, port_num, &attr); if (err) return err; immutable->max_mad_size = IB_MGMT_MAD_SIZE; immutable->pkey_tbl_len = attr.pkey_tbl_len; immutable->gid_tbl_len = attr.gid_tbl_len; return 0; } int irdma_iw_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; int err; immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; err = ib_query_port(ibdev, port_num, &attr); if (err) return err; immutable->gid_tbl_len = 1; return 0; } /** * irdma_get_eth_speed_and_width - Get IB port speed and width from netdev speed * @link_speed: netdev phy link speed * @active_speed: IB port speed * @active_width: IB port width */ void irdma_get_eth_speed_and_width(u32 link_speed, u8 *active_speed, u8 *active_width) { if (link_speed <= SPEED_1000) { *active_width = IB_WIDTH_1X; *active_speed = IB_SPEED_SDR; } else if (link_speed <= SPEED_10000) { *active_width = IB_WIDTH_1X; *active_speed = IB_SPEED_FDR10; } else if (link_speed <= SPEED_20000) { *active_width = IB_WIDTH_4X; *active_speed = IB_SPEED_DDR; } else if (link_speed <= SPEED_25000) { *active_width = IB_WIDTH_1X; *active_speed = IB_SPEED_EDR; } else if (link_speed <= SPEED_40000) { *active_width = IB_WIDTH_4X; *active_speed = IB_SPEED_FDR10; } else { *active_width = IB_WIDTH_4X; *active_speed = IB_SPEED_EDR; } } /** * irdma_query_port - get port attributes * @ibdev: device pointer from stack * @port: port number for query * @props: returning device attributes */ int irdma_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) { struct irdma_device *iwdev = to_iwdev(ibdev); struct ifnet *netdev = iwdev->netdev; /* no need to zero out pros here. done by caller */ props->max_mtu = IB_MTU_4096; props->active_mtu = ib_mtu_int_to_enum(netdev->if_mtu); props->lid = 1; props->lmc = 0; props->sm_lid = 0; props->sm_sl = 0; if ((netdev->if_link_state == LINK_STATE_UP) && (netdev->if_drv_flags & IFF_DRV_RUNNING)) { props->state = IB_PORT_ACTIVE; props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; } else { props->state = IB_PORT_DOWN; props->phys_state = IB_PORT_PHYS_STATE_DISABLED; } irdma_get_eth_speed_and_width(SPEED_100000, &props->active_speed, &props->active_width); if (rdma_protocol_roce(ibdev, 1)) { props->gid_tbl_len = 32; kc_set_props_ip_gid_caps(props); props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; } else { props->gid_tbl_len = 1; } props->qkey_viol_cntr = 0; props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP; props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size; return 0; } extern const char *const irdma_hw_stat_names[]; /** * irdma_alloc_hw_stats - Allocate a hw stats structure * @ibdev: device pointer from stack * @port_num: port number */ struct rdma_hw_stats * irdma_alloc_hw_stats(struct ib_device *ibdev, u8 port_num) { struct irdma_device *iwdev = to_iwdev(ibdev); struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; int num_counters = dev->hw_attrs.max_stat_idx; unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; return rdma_alloc_hw_stats_struct(irdma_hw_stat_names, num_counters, lifespan); } /** * irdma_get_hw_stats - Populates the rdma_hw_stats structure * @ibdev: device pointer from stack * @stats: stats pointer from stack * @port_num: port number * @index: which hw counter the stack is requesting we update */ int irdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, u8 port_num, int index) { struct irdma_device *iwdev = to_iwdev(ibdev); struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats; if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true); memcpy(&stats->value[0], hw_stats, sizeof(u64)* stats->num_counters); return stats->num_counters; } /** * irdma_query_gid - Query port GID * @ibdev: device pointer from stack * @port: port number * @index: Entry index * @gid: Global ID */ int irdma_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid) { struct irdma_device *iwdev = to_iwdev(ibdev); memset(gid->raw, 0, sizeof(gid->raw)); ether_addr_copy(gid->raw, IF_LLADDR(iwdev->netdev)); return 0; } enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, u8 port_num) { return IB_LINK_LAYER_ETHERNET; } inline enum ib_mtu ib_mtu_int_to_enum(int mtu) { if (mtu >= 4096) return IB_MTU_4096; else if (mtu >= 2048) return IB_MTU_2048; else if (mtu >= 1024) return IB_MTU_1024; else if (mtu >= 512) return IB_MTU_512; else return IB_MTU_256; } inline void kc_set_roce_uverbs_cmd_mask(struct irdma_device *iwdev) { iwdev->ibdev.uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST) | BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH) | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH) | BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST); } inline void kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev) { iwdev->ibdev.uverbs_cmd_mask = BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) | BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE) | BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT) | BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) | BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) | BIT_ULL(IB_USER_VERBS_CMD_REG_MR) | BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) | BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) | BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ) | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ) | BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP) | BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP) | BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) | BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ) | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) | BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) | BIT_ULL(IB_USER_VERBS_CMD_BIND_MW) | BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) | BIT_ULL(IB_USER_VERBS_CMD_POST_RECV) | BIT_ULL(IB_USER_VERBS_CMD_POST_SEND); iwdev->ibdev.uverbs_ex_cmd_mask = BIT_ULL(IB_USER_VERBS_EX_CMD_MODIFY_QP) | BIT_ULL(IB_USER_VERBS_EX_CMD_QUERY_DEVICE); if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) iwdev->ibdev.uverbs_ex_cmd_mask |= BIT_ULL(IB_USER_VERBS_EX_CMD_CREATE_CQ); } diff --git a/sys/ofed/drivers/infiniband/core/ib_cma.c b/sys/ofed/drivers/infiniband/core/ib_cma.c index 10dedf96f0e6..85af3330f08a 100644 --- a/sys/ofed/drivers/infiniband/core/ib_cma.c +++ b/sys/ofed/drivers/infiniband/core/ib_cma.c @@ -1,4676 +1,4703 @@ /*- * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0 * * Copyright (c) 2005 Voltaire Inc. All rights reserved. * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved. * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #define LINUXKPI_PARAM_PREFIX ibcore_ #include "opt_inet.h" #include "opt_inet6.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "core_priv.h" MODULE_AUTHOR("Sean Hefty"); MODULE_DESCRIPTION("Generic RDMA CM Agent"); MODULE_LICENSE("Dual BSD/GPL"); #define CMA_CM_RESPONSE_TIMEOUT 20 #define CMA_QUERY_CLASSPORT_INFO_TIMEOUT 3000 #define CMA_MAX_CM_RETRIES 15 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24) #define CMA_IBOE_PACKET_LIFETIME 18 static const char * const cma_events[] = { [RDMA_CM_EVENT_ADDR_RESOLVED] = "address resolved", [RDMA_CM_EVENT_ADDR_ERROR] = "address error", [RDMA_CM_EVENT_ROUTE_RESOLVED] = "route resolved ", [RDMA_CM_EVENT_ROUTE_ERROR] = "route error", [RDMA_CM_EVENT_CONNECT_REQUEST] = "connect request", [RDMA_CM_EVENT_CONNECT_RESPONSE] = "connect response", [RDMA_CM_EVENT_CONNECT_ERROR] = "connect error", [RDMA_CM_EVENT_UNREACHABLE] = "unreachable", [RDMA_CM_EVENT_REJECTED] = "rejected", [RDMA_CM_EVENT_ESTABLISHED] = "established", [RDMA_CM_EVENT_DISCONNECTED] = "disconnected", [RDMA_CM_EVENT_DEVICE_REMOVAL] = "device removal", [RDMA_CM_EVENT_MULTICAST_JOIN] = "multicast join", [RDMA_CM_EVENT_MULTICAST_ERROR] = "multicast error", [RDMA_CM_EVENT_ADDR_CHANGE] = "address change", [RDMA_CM_EVENT_TIMEWAIT_EXIT] = "timewait exit", }; const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event) { size_t index = event; return (index < ARRAY_SIZE(cma_events) && cma_events[index]) ? cma_events[index] : "unrecognized event"; } EXPORT_SYMBOL(rdma_event_msg); const char *__attribute_const__ rdma_reject_msg(struct rdma_cm_id *id, int reason) { if (rdma_ib_or_roce(id->device, id->port_num)) return ibcm_reject_msg(reason); if (rdma_protocol_iwarp(id->device, id->port_num)) return iwcm_reject_msg(reason); WARN_ON_ONCE(1); return "unrecognized transport"; } EXPORT_SYMBOL(rdma_reject_msg); static int cma_check_linklocal(struct rdma_dev_addr *, struct sockaddr *); static void cma_add_one(struct ib_device *device); static void cma_remove_one(struct ib_device *device, void *client_data); static enum rdma_port_space rdma_ps_from_service_id(__be64 service_id); static struct ib_client cma_client = { .name = "cma", .add = cma_add_one, .remove = cma_remove_one }; static struct ib_sa_client sa_client; static struct rdma_addr_client addr_client; static LIST_HEAD(dev_list); static LIST_HEAD(listen_any_list); static DEFINE_MUTEX(lock); static struct workqueue_struct *cma_wq; struct cma_pernet { struct idr tcp_ps; struct idr udp_ps; struct idr ipoib_ps; struct idr ib_ps; struct idr sdp_ps; }; VNET_DEFINE(struct cma_pernet, cma_pernet); static struct cma_pernet *cma_pernet_ptr(struct vnet *vnet) { struct cma_pernet *retval; CURVNET_SET_QUIET(vnet); retval = &VNET(cma_pernet); CURVNET_RESTORE(); return (retval); } static struct idr *cma_pernet_idr(struct vnet *net, enum rdma_port_space ps) { struct cma_pernet *pernet = cma_pernet_ptr(net); switch (ps) { case RDMA_PS_TCP: return &pernet->tcp_ps; case RDMA_PS_UDP: return &pernet->udp_ps; case RDMA_PS_IPOIB: return &pernet->ipoib_ps; case RDMA_PS_IB: return &pernet->ib_ps; case RDMA_PS_SDP: return &pernet->sdp_ps; default: return NULL; } } struct cma_device { struct list_head list; struct ib_device *device; struct completion comp; atomic_t refcount; struct list_head id_list; struct sysctl_ctx_list sysctl_ctx; enum ib_gid_type *default_gid_type; }; struct rdma_bind_list { enum rdma_port_space ps; struct hlist_head owners; unsigned short port; }; struct class_port_info_context { struct ib_class_port_info *class_port_info; struct ib_device *device; struct completion done; struct ib_sa_query *sa_query; u8 port_num; }; static int cma_ps_alloc(struct vnet *vnet, enum rdma_port_space ps, struct rdma_bind_list *bind_list, int snum) { struct idr *idr = cma_pernet_idr(vnet, ps); return idr_alloc(idr, bind_list, snum, snum + 1, GFP_KERNEL); } static struct rdma_bind_list *cma_ps_find(struct vnet *net, enum rdma_port_space ps, int snum) { struct idr *idr = cma_pernet_idr(net, ps); return idr_find(idr, snum); } static void cma_ps_remove(struct vnet *net, enum rdma_port_space ps, int snum) { struct idr *idr = cma_pernet_idr(net, ps); idr_remove(idr, snum); } enum { CMA_OPTION_AFONLY, }; void cma_ref_dev(struct cma_device *cma_dev) { atomic_inc(&cma_dev->refcount); } struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter filter, void *cookie) { struct cma_device *cma_dev; struct cma_device *found_cma_dev = NULL; mutex_lock(&lock); list_for_each_entry(cma_dev, &dev_list, list) if (filter(cma_dev->device, cookie)) { found_cma_dev = cma_dev; break; } if (found_cma_dev) cma_ref_dev(found_cma_dev); mutex_unlock(&lock); return found_cma_dev; } int cma_get_default_gid_type(struct cma_device *cma_dev, unsigned int port) { if (!rdma_is_port_valid(cma_dev->device, port)) return -EINVAL; return cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)]; } int cma_set_default_gid_type(struct cma_device *cma_dev, unsigned int port, enum ib_gid_type default_gid_type) { unsigned long supported_gids; if (!rdma_is_port_valid(cma_dev->device, port)) return -EINVAL; supported_gids = roce_gid_type_mask_support(cma_dev->device, port); if (!(supported_gids & 1 << default_gid_type)) return -EINVAL; cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)] = default_gid_type; return 0; } struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev) { return cma_dev->device; } /* * Device removal can occur at anytime, so we need extra handling to * serialize notifying the user of device removal with other callbacks. * We do this by disabling removal notification while a callback is in process, * and reporting it after the callback completes. */ struct rdma_id_private { struct rdma_cm_id id; struct rdma_bind_list *bind_list; struct hlist_node node; struct list_head list; /* listen_any_list or cma_device.list */ struct list_head listen_list; /* per device listens */ struct cma_device *cma_dev; struct list_head mc_list; int internal_id; enum rdma_cm_state state; spinlock_t lock; struct mutex qp_mutex; struct completion comp; atomic_t refcount; struct mutex handler_mutex; int backlog; int timeout_ms; struct ib_sa_query *query; int query_id; union { struct ib_cm_id *ib; struct iw_cm_id *iw; } cm_id; u32 seq_num; u32 qkey; u32 qp_num; pid_t owner; u32 options; u8 srq; u8 tos; u8 timeout_set:1; u8 reuseaddr; u8 afonly; u8 timeout; enum ib_gid_type gid_type; }; struct cma_multicast { struct rdma_id_private *id_priv; union { struct ib_sa_multicast *ib; } multicast; struct list_head list; void *context; struct sockaddr_storage addr; struct kref mcref; bool igmp_joined; u8 join_state; }; struct cma_work { struct work_struct work; struct rdma_id_private *id; enum rdma_cm_state old_state; enum rdma_cm_state new_state; struct rdma_cm_event event; }; struct cma_ndev_work { struct work_struct work; struct rdma_id_private *id; struct rdma_cm_event event; }; struct iboe_mcast_work { struct work_struct work; struct rdma_id_private *id; struct cma_multicast *mc; }; struct cma_hdr { u8 cma_version; u8 ip_version; /* IP version: 7:4 */ __be16 port; union cma_ip_addr src_addr; union cma_ip_addr dst_addr; }; #define CMA_VERSION 0x00 #define SDP_MAJ_VERSION 0x2 struct cma_req_info { struct ib_device *device; int port; union ib_gid local_gid; __be64 service_id; u16 pkey; bool has_gid:1; }; static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp) { unsigned long flags; int ret; spin_lock_irqsave(&id_priv->lock, flags); ret = (id_priv->state == comp); spin_unlock_irqrestore(&id_priv->lock, flags); return ret; } static int cma_comp_exch(struct rdma_id_private *id_priv, enum rdma_cm_state comp, enum rdma_cm_state exch) { unsigned long flags; int ret; spin_lock_irqsave(&id_priv->lock, flags); if ((ret = (id_priv->state == comp))) id_priv->state = exch; spin_unlock_irqrestore(&id_priv->lock, flags); return ret; } static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv, enum rdma_cm_state exch) { unsigned long flags; enum rdma_cm_state old; spin_lock_irqsave(&id_priv->lock, flags); old = id_priv->state; id_priv->state = exch; spin_unlock_irqrestore(&id_priv->lock, flags); return old; } static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr) { return hdr->ip_version >> 4; } static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver) { hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF); } static inline u8 sdp_get_majv(u8 sdp_version) { return sdp_version >> 4; } static inline u8 sdp_get_ip_ver(const struct sdp_hh *hh) { return hh->ipv_cap >> 4; } static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver) { hh->ipv_cap = (ip_ver << 4) | (hh->ipv_cap & 0xF); } static int cma_igmp_send(struct ifnet *ndev, const union ib_gid *mgid, bool join) { int retval; if (ndev) { union rdma_sockaddr addr; rdma_gid2ip(&addr._sockaddr, mgid); CURVNET_SET_QUIET(ndev->if_vnet); if (join) retval = -if_addmulti(ndev, &addr._sockaddr, NULL); else retval = -if_delmulti(ndev, &addr._sockaddr); CURVNET_RESTORE(); } else { retval = -ENODEV; } return retval; } static void _cma_attach_to_dev(struct rdma_id_private *id_priv, struct cma_device *cma_dev) { cma_ref_dev(cma_dev); id_priv->cma_dev = cma_dev; id_priv->gid_type = 0; id_priv->id.device = cma_dev->device; id_priv->id.route.addr.dev_addr.transport = rdma_node_get_transport(cma_dev->device->node_type); list_add_tail(&id_priv->list, &cma_dev->id_list); } static void cma_attach_to_dev(struct rdma_id_private *id_priv, struct cma_device *cma_dev) { _cma_attach_to_dev(id_priv, cma_dev); id_priv->gid_type = cma_dev->default_gid_type[id_priv->id.port_num - rdma_start_port(cma_dev->device)]; } void cma_deref_dev(struct cma_device *cma_dev) { if (atomic_dec_and_test(&cma_dev->refcount)) complete(&cma_dev->comp); } static inline void release_mc(struct kref *kref) { struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref); kfree(mc->multicast.ib); kfree(mc); } static void cma_release_dev(struct rdma_id_private *id_priv) { mutex_lock(&lock); list_del(&id_priv->list); cma_deref_dev(id_priv->cma_dev); id_priv->cma_dev = NULL; mutex_unlock(&lock); } static inline struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv) { return (struct sockaddr *) &id_priv->id.route.addr.src_addr; } static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv) { return (struct sockaddr *) &id_priv->id.route.addr.dst_addr; } static inline unsigned short cma_family(struct rdma_id_private *id_priv) { return id_priv->id.route.addr.src_addr.ss_family; } static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey) { struct ib_sa_mcmember_rec rec; int ret = 0; if (id_priv->qkey) { if (qkey && id_priv->qkey != qkey) return -EINVAL; return 0; } if (qkey) { id_priv->qkey = qkey; return 0; } switch (id_priv->id.ps) { case RDMA_PS_UDP: case RDMA_PS_IB: id_priv->qkey = RDMA_UDP_QKEY; break; case RDMA_PS_IPOIB: ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid); ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, &rec.mgid, &rec); if (!ret) id_priv->qkey = be32_to_cpu(rec.qkey); break; default: break; } return ret; } static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr) { dev_addr->dev_type = ARPHRD_INFINIBAND; rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr); ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey)); } static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr) { int ret; if (addr->sa_family != AF_IB) { ret = rdma_translate_ip(addr, dev_addr); } else { cma_translate_ib((struct sockaddr_ib *) addr, dev_addr); ret = 0; } return ret; } static inline int cma_validate_port(struct ib_device *device, u8 port, enum ib_gid_type gid_type, union ib_gid *gid, const struct rdma_dev_addr *dev_addr) { const int dev_type = dev_addr->dev_type; struct ifnet *ndev; int ret = -ENODEV; if ((dev_type == ARPHRD_INFINIBAND) && !rdma_protocol_ib(device, port)) return ret; if ((dev_type != ARPHRD_INFINIBAND) && rdma_protocol_ib(device, port)) return ret; if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) { ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); } else { ndev = NULL; gid_type = IB_GID_TYPE_IB; } ret = ib_find_cached_gid_by_port(device, gid, gid_type, port, ndev, NULL); if (ndev) dev_put(ndev); return ret; } static int cma_acquire_dev(struct rdma_id_private *id_priv, struct rdma_id_private *listen_id_priv) { struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; struct cma_device *cma_dev; union ib_gid gid, iboe_gid, *gidp; int ret = -ENODEV; u8 port; if (dev_addr->dev_type != ARPHRD_INFINIBAND && id_priv->id.ps == RDMA_PS_IPOIB) return -EINVAL; mutex_lock(&lock); rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, &iboe_gid); memcpy(&gid, dev_addr->src_dev_addr + rdma_addr_gid_offset(dev_addr), sizeof gid); if (listen_id_priv) { cma_dev = listen_id_priv->cma_dev; port = listen_id_priv->id.port_num; if (rdma_is_port_valid(cma_dev->device, port)) { gidp = rdma_protocol_roce(cma_dev->device, port) ? &iboe_gid : &gid; ret = cma_validate_port(cma_dev->device, port, rdma_protocol_ib(cma_dev->device, port) ? IB_GID_TYPE_IB : listen_id_priv->gid_type, gidp, dev_addr); if (!ret) { id_priv->id.port_num = port; goto out; } } } list_for_each_entry(cma_dev, &dev_list, list) { for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) { if (listen_id_priv && listen_id_priv->cma_dev == cma_dev && listen_id_priv->id.port_num == port) continue; gidp = rdma_protocol_roce(cma_dev->device, port) ? &iboe_gid : &gid; ret = cma_validate_port(cma_dev->device, port, rdma_protocol_ib(cma_dev->device, port) ? IB_GID_TYPE_IB : cma_dev->default_gid_type[port - 1], gidp, dev_addr); if (!ret) { id_priv->id.port_num = port; goto out; } } } out: if (!ret) cma_attach_to_dev(id_priv, cma_dev); mutex_unlock(&lock); return ret; } /* * Select the source IB device and address to reach the destination IB address. */ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv) { struct cma_device *cma_dev, *cur_dev; struct sockaddr_ib *addr; union ib_gid gid, sgid, *dgid; u16 pkey, index; u8 p; int i; cma_dev = NULL; addr = (struct sockaddr_ib *) cma_dst_addr(id_priv); dgid = (union ib_gid *) &addr->sib_addr; pkey = ntohs(addr->sib_pkey); list_for_each_entry(cur_dev, &dev_list, list) { for (p = 1; p <= cur_dev->device->phys_port_cnt; ++p) { if (!rdma_cap_af_ib(cur_dev->device, p)) continue; if (ib_find_cached_pkey(cur_dev->device, p, pkey, &index)) continue; for (i = 0; !ib_get_cached_gid(cur_dev->device, p, i, &gid, NULL); i++) { if (!memcmp(&gid, dgid, sizeof(gid))) { cma_dev = cur_dev; sgid = gid; id_priv->id.port_num = p; goto found; } if (!cma_dev && (gid.global.subnet_prefix == dgid->global.subnet_prefix)) { cma_dev = cur_dev; sgid = gid; id_priv->id.port_num = p; } } } } if (!cma_dev) return -ENODEV; found: cma_attach_to_dev(id_priv, cma_dev); addr = (struct sockaddr_ib *) cma_src_addr(id_priv); memcpy(&addr->sib_addr, &sgid, sizeof sgid); cma_translate_ib(addr, &id_priv->id.route.addr.dev_addr); return 0; } static void cma_deref_id(struct rdma_id_private *id_priv) { if (atomic_dec_and_test(&id_priv->refcount)) complete(&id_priv->comp); } struct rdma_cm_id *rdma_create_id(struct vnet *net, rdma_cm_event_handler event_handler, void *context, enum rdma_port_space ps, enum ib_qp_type qp_type) { struct rdma_id_private *id_priv; #ifdef VIMAGE if (net == NULL) return ERR_PTR(-EINVAL); #endif id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL); if (!id_priv) return ERR_PTR(-ENOMEM); id_priv->owner = task_pid_nr(current); id_priv->state = RDMA_CM_IDLE; id_priv->id.context = context; id_priv->id.event_handler = event_handler; id_priv->id.ps = ps; id_priv->id.qp_type = qp_type; id_priv->timeout_set = false; spin_lock_init(&id_priv->lock); mutex_init(&id_priv->qp_mutex); init_completion(&id_priv->comp); atomic_set(&id_priv->refcount, 1); mutex_init(&id_priv->handler_mutex); INIT_LIST_HEAD(&id_priv->listen_list); INIT_LIST_HEAD(&id_priv->mc_list); get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num); id_priv->id.route.addr.dev_addr.net = net; return &id_priv->id; } EXPORT_SYMBOL(rdma_create_id); static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) { struct ib_qp_attr qp_attr; int qp_attr_mask, ret; qp_attr.qp_state = IB_QPS_INIT; ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); if (ret) return ret; ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); if (ret) return ret; qp_attr.qp_state = IB_QPS_RTR; ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE); if (ret) return ret; qp_attr.qp_state = IB_QPS_RTS; qp_attr.sq_psn = 0; ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN); return ret; } static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) { struct ib_qp_attr qp_attr; int qp_attr_mask, ret; qp_attr.qp_state = IB_QPS_INIT; ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); if (ret) return ret; return ib_modify_qp(qp, &qp_attr, qp_attr_mask); } int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd, struct ib_qp_init_attr *qp_init_attr) { struct rdma_id_private *id_priv; struct ib_qp *qp; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (id->device != pd->device) return -EINVAL; qp_init_attr->port_num = id->port_num; qp = ib_create_qp(pd, qp_init_attr); if (IS_ERR(qp)) return PTR_ERR(qp); if (id->qp_type == IB_QPT_UD) ret = cma_init_ud_qp(id_priv, qp); else ret = cma_init_conn_qp(id_priv, qp); if (ret) goto err; id->qp = qp; id_priv->qp_num = qp->qp_num; id_priv->srq = (qp->srq != NULL); return 0; err: ib_destroy_qp(qp); return ret; } EXPORT_SYMBOL(rdma_create_qp); void rdma_destroy_qp(struct rdma_cm_id *id) { struct rdma_id_private *id_priv; id_priv = container_of(id, struct rdma_id_private, id); mutex_lock(&id_priv->qp_mutex); ib_destroy_qp(id_priv->id.qp); id_priv->id.qp = NULL; mutex_unlock(&id_priv->qp_mutex); } EXPORT_SYMBOL(rdma_destroy_qp); static int cma_modify_qp_rtr(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct ib_qp_attr qp_attr; int qp_attr_mask, ret; union ib_gid sgid; mutex_lock(&id_priv->qp_mutex); if (!id_priv->id.qp) { ret = 0; goto out; } /* Need to update QP attributes from default values. */ qp_attr.qp_state = IB_QPS_INIT; ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); if (ret) goto out; ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); if (ret) goto out; qp_attr.qp_state = IB_QPS_RTR; ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); if (ret) goto out; ret = ib_query_gid(id_priv->id.device, id_priv->id.port_num, qp_attr.ah_attr.grh.sgid_index, &sgid, NULL); if (ret) goto out; BUG_ON(id_priv->cma_dev->device != id_priv->id.device); if (conn_param) qp_attr.max_dest_rd_atomic = conn_param->responder_resources; ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); out: mutex_unlock(&id_priv->qp_mutex); return ret; } static int cma_modify_qp_rts(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct ib_qp_attr qp_attr; int qp_attr_mask, ret; mutex_lock(&id_priv->qp_mutex); if (!id_priv->id.qp) { ret = 0; goto out; } qp_attr.qp_state = IB_QPS_RTS; ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); if (ret) goto out; if (conn_param) qp_attr.max_rd_atomic = conn_param->initiator_depth; ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); out: mutex_unlock(&id_priv->qp_mutex); return ret; } static int cma_modify_qp_err(struct rdma_id_private *id_priv) { struct ib_qp_attr qp_attr; int ret; mutex_lock(&id_priv->qp_mutex); if (!id_priv->id.qp) { ret = 0; goto out; } qp_attr.qp_state = IB_QPS_ERR; ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE); out: mutex_unlock(&id_priv->qp_mutex); return ret; } static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; int ret; u16 pkey; if (rdma_cap_eth_ah(id_priv->id.device, id_priv->id.port_num)) pkey = 0xffff; else pkey = ib_addr_get_pkey(dev_addr); ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num, pkey, &qp_attr->pkey_index); if (ret) return ret; qp_attr->port_num = id_priv->id.port_num; *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT; if (id_priv->id.qp_type == IB_QPT_UD) { ret = cma_set_qkey(id_priv, 0); if (ret) return ret; qp_attr->qkey = id_priv->qkey; *qp_attr_mask |= IB_QP_QKEY; } else { qp_attr->qp_access_flags = 0; *qp_attr_mask |= IB_QP_ACCESS_FLAGS; } return 0; } int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { struct rdma_id_private *id_priv; int ret = 0; id_priv = container_of(id, struct rdma_id_private, id); if (rdma_cap_ib_cm(id->device, id->port_num)) { if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD)) ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask); else ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr, qp_attr_mask); if (qp_attr->qp_state == IB_QPS_RTR) qp_attr->rq_psn = id_priv->seq_num; } else if (rdma_cap_iw_cm(id->device, id->port_num)) { if (!id_priv->cm_id.iw) { qp_attr->qp_access_flags = 0; *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS; } else ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr, qp_attr_mask); qp_attr->port_num = id_priv->id.port_num; *qp_attr_mask |= IB_QP_PORT; } else ret = -ENOSYS; if ((*qp_attr_mask & IB_QP_TIMEOUT) && id_priv->timeout_set) qp_attr->timeout = id_priv->timeout; return ret; } EXPORT_SYMBOL(rdma_init_qp_attr); static inline int cma_zero_addr(struct sockaddr *addr) { switch (addr->sa_family) { case AF_INET: return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr); case AF_INET6: return ipv6_addr_any(&((struct sockaddr_in6 *) addr)->sin6_addr); case AF_IB: return ib_addr_any(&((struct sockaddr_ib *) addr)->sib_addr); default: return 0; } } static inline int cma_loopback_addr(struct sockaddr *addr) { switch (addr->sa_family) { #ifdef INET /* * ipv4_is_loopback() requires an inet variable via vnet, * not present if INET is not included. */ case AF_INET: return ipv4_is_loopback(((struct sockaddr_in *) addr)->sin_addr.s_addr); #endif #ifdef INET6 case AF_INET6: return ipv6_addr_loopback(&((struct sockaddr_in6 *) addr)->sin6_addr); #endif case AF_IB: return ib_addr_loopback(&((struct sockaddr_ib *) addr)->sib_addr); default: return 0; } } -static inline int cma_any_addr(struct sockaddr *addr) +static inline bool cma_any_addr(struct vnet *vnet, struct sockaddr *addr) { - return cma_zero_addr(addr) || cma_loopback_addr(addr); + bool ret; + + CURVNET_SET_QUIET(vnet); + ret = cma_zero_addr(addr) || cma_loopback_addr(addr); + CURVNET_RESTORE(); + + return (ret); } static int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst) { if (src->sa_family != dst->sa_family) return -1; switch (src->sa_family) { case AF_INET: return ((struct sockaddr_in *) src)->sin_addr.s_addr != ((struct sockaddr_in *) dst)->sin_addr.s_addr; case AF_INET6: return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr, &((struct sockaddr_in6 *) dst)->sin6_addr); default: return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr, &((struct sockaddr_ib *) dst)->sib_addr); } } static __be16 cma_port(struct sockaddr *addr) { struct sockaddr_ib *sib; switch (addr->sa_family) { case AF_INET: return ((struct sockaddr_in *) addr)->sin_port; case AF_INET6: return ((struct sockaddr_in6 *) addr)->sin6_port; case AF_IB: sib = (struct sockaddr_ib *) addr; return htons((u16) (be64_to_cpu(sib->sib_sid) & be64_to_cpu(sib->sib_sid_mask))); default: return 0; } } static inline int cma_any_port(struct sockaddr *addr) { return !cma_port(addr); } static void cma_save_ib_info(struct sockaddr *src_addr, struct sockaddr *dst_addr, struct rdma_cm_id *listen_id, struct ib_sa_path_rec *path) { struct sockaddr_ib *listen_ib, *ib; listen_ib = (struct sockaddr_ib *) &listen_id->route.addr.src_addr; if (src_addr) { ib = (struct sockaddr_ib *)src_addr; ib->sib_family = AF_IB; if (path) { ib->sib_pkey = path->pkey; ib->sib_flowinfo = path->flow_label; memcpy(&ib->sib_addr, &path->sgid, 16); ib->sib_sid = path->service_id; ib->sib_scope_id = 0; } else { ib->sib_pkey = listen_ib->sib_pkey; ib->sib_flowinfo = listen_ib->sib_flowinfo; ib->sib_addr = listen_ib->sib_addr; ib->sib_sid = listen_ib->sib_sid; ib->sib_scope_id = listen_ib->sib_scope_id; } ib->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL); } if (dst_addr) { ib = (struct sockaddr_ib *)dst_addr; ib->sib_family = AF_IB; if (path) { ib->sib_pkey = path->pkey; ib->sib_flowinfo = path->flow_label; memcpy(&ib->sib_addr, &path->dgid, 16); } } } static void cma_save_ip4_info(struct sockaddr_in *src_addr, struct sockaddr_in *dst_addr, struct cma_hdr *hdr, __be16 local_port) { if (src_addr) { *src_addr = (struct sockaddr_in) { .sin_len = sizeof(struct sockaddr_in), .sin_family = AF_INET, .sin_addr.s_addr = hdr->dst_addr.ip4.addr, .sin_port = local_port, }; } if (dst_addr) { *dst_addr = (struct sockaddr_in) { .sin_len = sizeof(struct sockaddr_in), .sin_family = AF_INET, .sin_addr.s_addr = hdr->src_addr.ip4.addr, .sin_port = hdr->port, }; } } static void cma_ip6_clear_scope_id(struct in6_addr *addr) { /* make sure link local scope ID gets zeroed */ if (IN6_IS_SCOPE_LINKLOCAL(addr) || IN6_IS_ADDR_MC_INTFACELOCAL(addr)) { /* use byte-access to be alignment safe */ addr->s6_addr[2] = 0; addr->s6_addr[3] = 0; } } static void cma_save_ip6_info(struct sockaddr_in6 *src_addr, struct sockaddr_in6 *dst_addr, struct cma_hdr *hdr, __be16 local_port) { if (src_addr) { *src_addr = (struct sockaddr_in6) { .sin6_len = sizeof(struct sockaddr_in6), .sin6_family = AF_INET6, .sin6_addr = hdr->dst_addr.ip6, .sin6_port = local_port, }; cma_ip6_clear_scope_id(&src_addr->sin6_addr); } if (dst_addr) { *dst_addr = (struct sockaddr_in6) { .sin6_len = sizeof(struct sockaddr_in6), .sin6_family = AF_INET6, .sin6_addr = hdr->src_addr.ip6, .sin6_port = hdr->port, }; cma_ip6_clear_scope_id(&dst_addr->sin6_addr); } } static u16 cma_port_from_service_id(__be64 service_id) { return (u16)be64_to_cpu(service_id); } static int sdp_save_ip_info(struct sockaddr *src_addr, struct sockaddr *dst_addr, const struct sdp_hh *hdr, __be64 service_id) { __be16 local_port; BUG_ON(src_addr == NULL || dst_addr == NULL); if (sdp_get_majv(hdr->majv_minv) != SDP_MAJ_VERSION) return -EINVAL; local_port = htons(cma_port_from_service_id(service_id)); switch (sdp_get_ip_ver(hdr)) { case 4: { struct sockaddr_in *s4, *d4; s4 = (void *)src_addr; d4 = (void *)dst_addr; *s4 = (struct sockaddr_in) { .sin_len = sizeof(*s4), .sin_family = AF_INET, .sin_addr.s_addr = hdr->dst_addr.ip4.addr, .sin_port = local_port, }; *d4 = (struct sockaddr_in) { .sin_len = sizeof(*d4), .sin_family = AF_INET, .sin_addr.s_addr = hdr->src_addr.ip4.addr, .sin_port = hdr->port, }; break; } case 6: { struct sockaddr_in6 *s6, *d6; s6 = (void *)src_addr; d6 = (void *)dst_addr; *s6 = (struct sockaddr_in6) { .sin6_len = sizeof(*s6), .sin6_family = AF_INET6, .sin6_addr = hdr->dst_addr.ip6, .sin6_port = local_port, }; *d6 = (struct sockaddr_in6) { .sin6_len = sizeof(*d6), .sin6_family = AF_INET6, .sin6_addr = hdr->src_addr.ip6, .sin6_port = hdr->port, }; cma_ip6_clear_scope_id(&s6->sin6_addr); cma_ip6_clear_scope_id(&d6->sin6_addr); break; } default: return -EAFNOSUPPORT; } return 0; } static int cma_save_ip_info(struct sockaddr *src_addr, struct sockaddr *dst_addr, struct ib_cm_event *ib_event, __be64 service_id) { struct cma_hdr *hdr; __be16 port; if (rdma_ps_from_service_id(service_id) == RDMA_PS_SDP) return sdp_save_ip_info(src_addr, dst_addr, ib_event->private_data, service_id); hdr = ib_event->private_data; if (hdr->cma_version != CMA_VERSION) return -EINVAL; port = htons(cma_port_from_service_id(service_id)); switch (cma_get_ip_ver(hdr)) { case 4: cma_save_ip4_info((struct sockaddr_in *)src_addr, (struct sockaddr_in *)dst_addr, hdr, port); break; case 6: cma_save_ip6_info((struct sockaddr_in6 *)src_addr, (struct sockaddr_in6 *)dst_addr, hdr, port); break; default: return -EAFNOSUPPORT; } return 0; } static int cma_save_net_info(struct sockaddr *src_addr, struct sockaddr *dst_addr, struct rdma_cm_id *listen_id, struct ib_cm_event *ib_event, sa_family_t sa_family, __be64 service_id) { if (sa_family == AF_IB) { if (ib_event->event == IB_CM_REQ_RECEIVED) cma_save_ib_info(src_addr, dst_addr, listen_id, ib_event->param.req_rcvd.primary_path); else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) cma_save_ib_info(src_addr, dst_addr, listen_id, NULL); return 0; } return cma_save_ip_info(src_addr, dst_addr, ib_event, service_id); } static int cma_save_req_info(const struct ib_cm_event *ib_event, struct cma_req_info *req) { const struct ib_cm_req_event_param *req_param = &ib_event->param.req_rcvd; const struct ib_cm_sidr_req_event_param *sidr_param = &ib_event->param.sidr_req_rcvd; switch (ib_event->event) { case IB_CM_REQ_RECEIVED: req->device = req_param->listen_id->device; req->port = req_param->port; memcpy(&req->local_gid, &req_param->primary_path->sgid, sizeof(req->local_gid)); req->has_gid = true; req->service_id = req_param->primary_path->service_id; req->pkey = be16_to_cpu(req_param->primary_path->pkey); if (req->pkey != req_param->bth_pkey) pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and primary path P_Key (0x%x)\n" "RDMA CMA: in the future this may cause the request to be dropped\n", req_param->bth_pkey, req->pkey); break; case IB_CM_SIDR_REQ_RECEIVED: req->device = sidr_param->listen_id->device; req->port = sidr_param->port; req->has_gid = false; req->service_id = sidr_param->service_id; req->pkey = sidr_param->pkey; if (req->pkey != sidr_param->bth_pkey) pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and SIDR request payload P_Key (0x%x)\n" "RDMA CMA: in the future this may cause the request to be dropped\n", sidr_param->bth_pkey, req->pkey); break; default: return -EINVAL; } return 0; } +#ifdef INET +static bool validate_ipv4_net_dev_addr(struct vnet *vnet, + const __be32 saddr, const __be32 daddr) +{ + bool ret; + CURVNET_SET(vnet); + ret = ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) || + ipv4_is_lbcast(daddr) || ipv4_is_zeronet(saddr) || + ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr) || + ipv4_is_loopback(saddr); + CURVNET_RESTORE(); + return (ret); +} +#endif + static bool validate_ipv4_net_dev(struct ifnet *net_dev, const struct sockaddr_in *dst_addr, const struct sockaddr_in *src_addr) { #ifdef INET __be32 daddr = dst_addr->sin_addr.s_addr, saddr = src_addr->sin_addr.s_addr; struct ifnet *dst_dev; struct nhop_object *nh; bool ret; - if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) || - ipv4_is_lbcast(daddr) || ipv4_is_zeronet(saddr) || - ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr) || - ipv4_is_loopback(saddr)) + if (validate_ipv4_net_dev_addr(net_dev->if_vnet, saddr, daddr)) return false; dst_dev = ip_ifp_find(net_dev->if_vnet, daddr); if (dst_dev != net_dev) { if (dst_dev != NULL) dev_put(dst_dev); return false; } dev_put(dst_dev); /* * Check for loopback. */ if (saddr == daddr) return true; CURVNET_SET(net_dev->if_vnet); nh = fib4_lookup(RT_DEFAULT_FIB, src_addr->sin_addr, 0, NHR_NONE, 0); if (nh != NULL) ret = (nh->nh_ifp == net_dev); else ret = false; CURVNET_RESTORE(); return ret; #else return false; #endif } static bool validate_ipv6_net_dev(struct ifnet *net_dev, const struct sockaddr_in6 *dst_addr, const struct sockaddr_in6 *src_addr) { #ifdef INET6 struct sockaddr_in6 src_tmp = *src_addr; struct sockaddr_in6 dst_tmp = *dst_addr; struct ifnet *dst_dev; struct nhop_object *nh; bool ret; dst_dev = ip6_ifp_find(net_dev->if_vnet, dst_tmp.sin6_addr, net_dev->if_index); if (dst_dev != net_dev) { if (dst_dev != NULL) dev_put(dst_dev); return false; } dev_put(dst_dev); CURVNET_SET(net_dev->if_vnet); /* * Make sure the scope ID gets embedded. */ src_tmp.sin6_scope_id = net_dev->if_index; sa6_embedscope(&src_tmp, 0); dst_tmp.sin6_scope_id = net_dev->if_index; sa6_embedscope(&dst_tmp, 0); /* * Check for loopback after scope ID * has been embedded: */ if (memcmp(&src_tmp.sin6_addr, &dst_tmp.sin6_addr, sizeof(dst_tmp.sin6_addr)) == 0) { ret = true; } else { /* non-loopback case */ nh = fib6_lookup(RT_DEFAULT_FIB, &src_addr->sin6_addr, net_dev->if_index, NHR_NONE, 0); if (nh != NULL) ret = (nh->nh_ifp == net_dev); else ret = false; } CURVNET_RESTORE(); return ret; #else return false; #endif } static bool validate_net_dev(struct ifnet *net_dev, const struct sockaddr *daddr, const struct sockaddr *saddr) { const struct sockaddr_in *daddr4 = (const struct sockaddr_in *)daddr; const struct sockaddr_in *saddr4 = (const struct sockaddr_in *)saddr; const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr; const struct sockaddr_in6 *saddr6 = (const struct sockaddr_in6 *)saddr; switch (daddr->sa_family) { case AF_INET: return saddr->sa_family == AF_INET && validate_ipv4_net_dev(net_dev, daddr4, saddr4); case AF_INET6: return saddr->sa_family == AF_INET6 && validate_ipv6_net_dev(net_dev, daddr6, saddr6); default: return false; } } static struct ifnet * roce_get_net_dev_by_cm_event(struct ib_device *device, u8 port_num, const struct ib_cm_event *ib_event) { struct ib_gid_attr sgid_attr; union ib_gid sgid; int err = -EINVAL; if (ib_event->event == IB_CM_REQ_RECEIVED) { err = ib_get_cached_gid(device, port_num, ib_event->param.req_rcvd.ppath_sgid_index, &sgid, &sgid_attr); } else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) { err = ib_get_cached_gid(device, port_num, ib_event->param.sidr_req_rcvd.sgid_index, &sgid, &sgid_attr); } if (err) return (NULL); return (sgid_attr.ndev); } static struct ifnet *cma_get_net_dev(struct ib_cm_event *ib_event, const struct cma_req_info *req) { struct sockaddr_storage listen_addr_storage, src_addr_storage; struct sockaddr *listen_addr = (struct sockaddr *)&listen_addr_storage, *src_addr = (struct sockaddr *)&src_addr_storage; struct ifnet *net_dev; const union ib_gid *gid = req->has_gid ? &req->local_gid : NULL; struct epoch_tracker et; int err; err = cma_save_ip_info(listen_addr, src_addr, ib_event, req->service_id); if (err) return ERR_PTR(err); if (rdma_protocol_roce(req->device, req->port)) { net_dev = roce_get_net_dev_by_cm_event(req->device, req->port, ib_event); } else { net_dev = ib_get_net_dev_by_params(req->device, req->port, req->pkey, gid, listen_addr); } if (!net_dev) return ERR_PTR(-ENODEV); NET_EPOCH_ENTER(et); if (!validate_net_dev(net_dev, listen_addr, src_addr)) { NET_EPOCH_EXIT(et); dev_put(net_dev); return ERR_PTR(-EHOSTUNREACH); } NET_EPOCH_EXIT(et); return net_dev; } static enum rdma_port_space rdma_ps_from_service_id(__be64 service_id) { return (be64_to_cpu(service_id) >> 16) & 0xffff; } static bool sdp_match_private_data(struct rdma_id_private *id_priv, const struct sdp_hh *hdr, struct sockaddr *addr) { __be32 ip4_addr; struct in6_addr ip6_addr; + struct vnet *vnet = id_priv->id.route.addr.dev_addr.net; switch (addr->sa_family) { case AF_INET: ip4_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr; if (sdp_get_ip_ver(hdr) != 4) return false; - if (!cma_any_addr(addr) && + if (!cma_any_addr(vnet, addr) && hdr->dst_addr.ip4.addr != ip4_addr) return false; break; case AF_INET6: ip6_addr = ((struct sockaddr_in6 *)addr)->sin6_addr; if (sdp_get_ip_ver(hdr) != 6) return false; cma_ip6_clear_scope_id(&ip6_addr); - if (!cma_any_addr(addr) && + if (!cma_any_addr(vnet, addr) && memcmp(&hdr->dst_addr.ip6, &ip6_addr, sizeof(ip6_addr))) return false; break; case AF_IB: return true; default: return false; } return true; } static bool cma_match_private_data(struct rdma_id_private *id_priv, const void *vhdr) { const struct cma_hdr *hdr = vhdr; struct sockaddr *addr = cma_src_addr(id_priv); + struct vnet *vnet = id_priv->id.route.addr.dev_addr.net; __be32 ip4_addr; struct in6_addr ip6_addr; - if (cma_any_addr(addr) && !id_priv->afonly) + if (cma_any_addr(vnet, addr) && !id_priv->afonly) return true; if (id_priv->id.ps == RDMA_PS_SDP) return sdp_match_private_data(id_priv, vhdr, addr); switch (addr->sa_family) { case AF_INET: ip4_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr; if (cma_get_ip_ver(hdr) != 4) return false; - if (!cma_any_addr(addr) && + if (!cma_any_addr(vnet, addr) && hdr->dst_addr.ip4.addr != ip4_addr) return false; break; case AF_INET6: ip6_addr = ((struct sockaddr_in6 *)addr)->sin6_addr; if (cma_get_ip_ver(hdr) != 6) return false; cma_ip6_clear_scope_id(&ip6_addr); - if (!cma_any_addr(addr) && + if (!cma_any_addr(vnet, addr) && memcmp(&hdr->dst_addr.ip6, &ip6_addr, sizeof(ip6_addr))) return false; break; case AF_IB: return true; default: return false; } return true; } static bool cma_protocol_roce_dev_port(struct ib_device *device, int port_num) { enum rdma_link_layer ll = rdma_port_get_link_layer(device, port_num); enum rdma_transport_type transport = rdma_node_get_transport(device->node_type); return ll == IB_LINK_LAYER_ETHERNET && transport == RDMA_TRANSPORT_IB; } static bool cma_protocol_roce(const struct rdma_cm_id *id) { struct ib_device *device = id->device; const int port_num = id->port_num ?: rdma_start_port(device); return cma_protocol_roce_dev_port(device, port_num); } static bool cma_match_net_dev(const struct rdma_cm_id *id, const struct ifnet *net_dev, u8 port_num) { const struct rdma_addr *addr = &id->route.addr; if (!net_dev) { if (id->port_num && id->port_num != port_num) return false; if (id->ps == RDMA_PS_SDP) { if (addr->src_addr.ss_family == AF_INET || addr->src_addr.ss_family == AF_INET6) return true; return false; } /* This request is an AF_IB request or a RoCE request */ return addr->src_addr.ss_family == AF_IB || cma_protocol_roce_dev_port(id->device, port_num); } return !addr->dev_addr.bound_dev_if || (net_eq(dev_net(net_dev), addr->dev_addr.net) && addr->dev_addr.bound_dev_if == net_dev->if_index); } static struct rdma_id_private *cma_find_listener( const struct rdma_bind_list *bind_list, const struct ib_cm_id *cm_id, const struct ib_cm_event *ib_event, const struct cma_req_info *req, const struct ifnet *net_dev) { struct rdma_id_private *id_priv, *id_priv_dev; if (!bind_list) return ERR_PTR(-EINVAL); hlist_for_each_entry(id_priv, &bind_list->owners, node) { if (cma_match_private_data(id_priv, ib_event->private_data)) { if (id_priv->id.device == cm_id->device && cma_match_net_dev(&id_priv->id, net_dev, req->port)) return id_priv; list_for_each_entry(id_priv_dev, &id_priv->listen_list, listen_list) { if (id_priv_dev->id.device == cm_id->device && cma_match_net_dev(&id_priv_dev->id, net_dev, req->port)) return id_priv_dev; } } } return ERR_PTR(-EINVAL); } static struct rdma_id_private *cma_id_from_event(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event, struct ifnet **net_dev) { struct cma_req_info req; struct rdma_bind_list *bind_list; struct rdma_id_private *id_priv; int err; err = cma_save_req_info(ib_event, &req); if (err) return ERR_PTR(err); if (rdma_ps_from_service_id(cm_id->service_id) == RDMA_PS_SDP) { *net_dev = NULL; goto there_is_no_net_dev; } *net_dev = cma_get_net_dev(ib_event, &req); if (IS_ERR(*net_dev)) { if (PTR_ERR(*net_dev) == -EAFNOSUPPORT) { /* Assuming the protocol is AF_IB */ *net_dev = NULL; } else { return ERR_CAST(*net_dev); } } there_is_no_net_dev: bind_list = cma_ps_find(*net_dev ? dev_net(*net_dev) : &init_net, rdma_ps_from_service_id(req.service_id), cma_port_from_service_id(req.service_id)); id_priv = cma_find_listener(bind_list, cm_id, ib_event, &req, *net_dev); if (IS_ERR(id_priv) && *net_dev) { dev_put(*net_dev); *net_dev = NULL; } return id_priv; } static inline int cma_user_data_offset(struct rdma_id_private *id_priv) { if (cma_family(id_priv) == AF_IB) return 0; if (id_priv->id.ps == RDMA_PS_SDP) return 0; return sizeof(struct cma_hdr); } static void cma_cancel_route(struct rdma_id_private *id_priv) { if (rdma_cap_ib_sa(id_priv->id.device, id_priv->id.port_num)) { if (id_priv->query) ib_sa_cancel_query(id_priv->query_id, id_priv->query); } } static void cma_cancel_listens(struct rdma_id_private *id_priv) { struct rdma_id_private *dev_id_priv; /* * Remove from listen_any_list to prevent added devices from spawning * additional listen requests. */ mutex_lock(&lock); list_del(&id_priv->list); while (!list_empty(&id_priv->listen_list)) { dev_id_priv = list_entry(id_priv->listen_list.next, struct rdma_id_private, listen_list); /* sync with device removal to avoid duplicate destruction */ list_del_init(&dev_id_priv->list); list_del(&dev_id_priv->listen_list); mutex_unlock(&lock); rdma_destroy_id(&dev_id_priv->id); mutex_lock(&lock); } mutex_unlock(&lock); } static void cma_cancel_operation(struct rdma_id_private *id_priv, enum rdma_cm_state state) { + struct vnet *vnet = id_priv->id.route.addr.dev_addr.net; + switch (state) { case RDMA_CM_ADDR_QUERY: rdma_addr_cancel(&id_priv->id.route.addr.dev_addr); break; case RDMA_CM_ROUTE_QUERY: cma_cancel_route(id_priv); break; case RDMA_CM_LISTEN: - if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev) + if (cma_any_addr(vnet, cma_src_addr(id_priv)) && !id_priv->cma_dev) cma_cancel_listens(id_priv); break; default: break; } } static void cma_release_port(struct rdma_id_private *id_priv) { struct rdma_bind_list *bind_list = id_priv->bind_list; struct vnet *net = id_priv->id.route.addr.dev_addr.net; if (!bind_list) return; mutex_lock(&lock); hlist_del(&id_priv->node); if (hlist_empty(&bind_list->owners)) { cma_ps_remove(net, bind_list->ps, bind_list->port); kfree(bind_list); } mutex_unlock(&lock); } static void cma_leave_mc_groups(struct rdma_id_private *id_priv) { struct cma_multicast *mc; while (!list_empty(&id_priv->mc_list)) { mc = container_of(id_priv->mc_list.next, struct cma_multicast, list); list_del(&mc->list); if (rdma_cap_ib_mcast(id_priv->cma_dev->device, id_priv->id.port_num)) { ib_sa_free_multicast(mc->multicast.ib); kfree(mc); } else { if (mc->igmp_joined) { struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; struct ifnet *ndev = NULL; if (dev_addr->bound_dev_if) ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); if (ndev) { cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid, false); dev_put(ndev); } } kref_put(&mc->mcref, release_mc); } } } void rdma_destroy_id(struct rdma_cm_id *id) { struct rdma_id_private *id_priv; enum rdma_cm_state state; id_priv = container_of(id, struct rdma_id_private, id); state = cma_exch(id_priv, RDMA_CM_DESTROYING); cma_cancel_operation(id_priv, state); /* * Wait for any active callback to finish. New callbacks will find * the id_priv state set to destroying and abort. */ mutex_lock(&id_priv->handler_mutex); mutex_unlock(&id_priv->handler_mutex); if (id_priv->cma_dev) { if (rdma_cap_ib_cm(id_priv->id.device, 1)) { if (id_priv->cm_id.ib) ib_destroy_cm_id(id_priv->cm_id.ib); } else if (rdma_cap_iw_cm(id_priv->id.device, 1)) { if (id_priv->cm_id.iw) iw_destroy_cm_id(id_priv->cm_id.iw); } cma_leave_mc_groups(id_priv); cma_release_dev(id_priv); } cma_release_port(id_priv); cma_deref_id(id_priv); wait_for_completion(&id_priv->comp); if (id_priv->internal_id) cma_deref_id(id_priv->id.context); kfree(id_priv->id.route.path_rec); kfree(id_priv); } EXPORT_SYMBOL(rdma_destroy_id); static int cma_rep_recv(struct rdma_id_private *id_priv) { int ret; ret = cma_modify_qp_rtr(id_priv, NULL); if (ret) goto reject; ret = cma_modify_qp_rts(id_priv, NULL); if (ret) goto reject; ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0); if (ret) goto reject; return 0; reject: cma_modify_qp_err(id_priv); ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0, NULL, 0); return ret; } static int sdp_verify_rep(const struct sdp_hah *data) { if (sdp_get_majv(data->majv_minv) != SDP_MAJ_VERSION) return -EINVAL; return 0; } static void cma_set_rep_event_data(struct rdma_cm_event *event, struct ib_cm_rep_event_param *rep_data, void *private_data) { event->param.conn.private_data = private_data; event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE; event->param.conn.responder_resources = rep_data->responder_resources; event->param.conn.initiator_depth = rep_data->initiator_depth; event->param.conn.flow_control = rep_data->flow_control; event->param.conn.rnr_retry_count = rep_data->rnr_retry_count; event->param.conn.srq = rep_data->srq; event->param.conn.qp_num = rep_data->remote_qpn; } static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) { struct rdma_id_private *id_priv = cm_id->context; struct rdma_cm_event event; int ret = 0; mutex_lock(&id_priv->handler_mutex); if ((ib_event->event != IB_CM_TIMEWAIT_EXIT && id_priv->state != RDMA_CM_CONNECT) || (ib_event->event == IB_CM_TIMEWAIT_EXIT && id_priv->state != RDMA_CM_DISCONNECT)) goto out; memset(&event, 0, sizeof event); switch (ib_event->event) { case IB_CM_REQ_ERROR: case IB_CM_REP_ERROR: event.event = RDMA_CM_EVENT_UNREACHABLE; event.status = -ETIMEDOUT; break; case IB_CM_REP_RECEIVED: if (id_priv->id.ps == RDMA_PS_SDP) { event.status = sdp_verify_rep(ib_event->private_data); if (event.status) event.event = RDMA_CM_EVENT_CONNECT_ERROR; else event.event = RDMA_CM_EVENT_CONNECT_RESPONSE; } else { if (id_priv->id.qp) { event.status = cma_rep_recv(id_priv); event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR : RDMA_CM_EVENT_ESTABLISHED; } else { event.event = RDMA_CM_EVENT_CONNECT_RESPONSE; } } cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd, ib_event->private_data); break; case IB_CM_RTU_RECEIVED: case IB_CM_USER_ESTABLISHED: event.event = RDMA_CM_EVENT_ESTABLISHED; break; case IB_CM_DREQ_ERROR: event.status = -ETIMEDOUT; /* fall through */ case IB_CM_DREQ_RECEIVED: case IB_CM_DREP_RECEIVED: if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_DISCONNECT)) goto out; event.event = RDMA_CM_EVENT_DISCONNECTED; break; case IB_CM_TIMEWAIT_EXIT: event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT; break; case IB_CM_MRA_RECEIVED: /* ignore event */ goto out; case IB_CM_REJ_RECEIVED: cma_modify_qp_err(id_priv); event.status = ib_event->param.rej_rcvd.reason; event.event = RDMA_CM_EVENT_REJECTED; event.param.conn.private_data = ib_event->private_data; event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE; break; default: pr_err("RDMA CMA: unexpected IB CM event: %d\n", ib_event->event); goto out; } ret = id_priv->id.event_handler(&id_priv->id, &event); if (ret) { /* Destroy the CM ID by returning a non-zero value. */ id_priv->cm_id.ib = NULL; cma_exch(id_priv, RDMA_CM_DESTROYING); mutex_unlock(&id_priv->handler_mutex); rdma_destroy_id(&id_priv->id); return ret; } out: mutex_unlock(&id_priv->handler_mutex); return ret; } static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id, struct ib_cm_event *ib_event, struct ifnet *net_dev) { struct rdma_id_private *id_priv; struct rdma_cm_id *id; struct rdma_route *rt; const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family; const __be64 service_id = ib_event->param.req_rcvd.primary_path->service_id; + struct vnet *vnet = listen_id->route.addr.dev_addr.net; int ret; - id = rdma_create_id(listen_id->route.addr.dev_addr.net, + id = rdma_create_id(vnet, listen_id->event_handler, listen_id->context, listen_id->ps, ib_event->param.req_rcvd.qp_type); if (IS_ERR(id)) return NULL; id_priv = container_of(id, struct rdma_id_private, id); if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr, (struct sockaddr *)&id->route.addr.dst_addr, listen_id, ib_event, ss_family, service_id)) goto err; rt = &id->route; rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1; rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths, GFP_KERNEL); if (!rt->path_rec) goto err; rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path; if (rt->num_paths == 2) rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path; if (net_dev) { ret = rdma_copy_addr(&rt->addr.dev_addr, net_dev, NULL); if (ret) goto err; } else { if (!cma_protocol_roce(listen_id) && - cma_any_addr(cma_src_addr(id_priv))) { + cma_any_addr(vnet, cma_src_addr(id_priv))) { rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND; rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid); ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey)); - } else if (!cma_any_addr(cma_src_addr(id_priv))) { + } else if (!cma_any_addr(vnet, cma_src_addr(id_priv))) { ret = cma_translate_addr(cma_src_addr(id_priv), &rt->addr.dev_addr); if (ret) goto err; } } rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid); id_priv->state = RDMA_CM_CONNECT; return id_priv; err: rdma_destroy_id(id); return NULL; } static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id, struct ib_cm_event *ib_event, struct ifnet *net_dev) { struct rdma_id_private *id_priv; struct rdma_cm_id *id; const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family; - struct vnet *net = listen_id->route.addr.dev_addr.net; + struct vnet *vnet = listen_id->route.addr.dev_addr.net; int ret; - id = rdma_create_id(net, listen_id->event_handler, listen_id->context, + id = rdma_create_id(vnet, listen_id->event_handler, listen_id->context, listen_id->ps, IB_QPT_UD); if (IS_ERR(id)) return NULL; id_priv = container_of(id, struct rdma_id_private, id); if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr, (struct sockaddr *)&id->route.addr.dst_addr, listen_id, ib_event, ss_family, ib_event->param.sidr_req_rcvd.service_id)) goto err; if (net_dev) { ret = rdma_copy_addr(&id->route.addr.dev_addr, net_dev, NULL); if (ret) goto err; } else { - if (!cma_any_addr(cma_src_addr(id_priv))) { + if (!cma_any_addr(vnet, cma_src_addr(id_priv))) { ret = cma_translate_addr(cma_src_addr(id_priv), &id->route.addr.dev_addr); if (ret) goto err; } } id_priv->state = RDMA_CM_CONNECT; return id_priv; err: rdma_destroy_id(id); return NULL; } static void cma_set_req_event_data(struct rdma_cm_event *event, struct ib_cm_req_event_param *req_data, void *private_data, int offset) { event->param.conn.private_data = (char *)private_data + offset; event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset; event->param.conn.responder_resources = req_data->responder_resources; event->param.conn.initiator_depth = req_data->initiator_depth; event->param.conn.flow_control = req_data->flow_control; event->param.conn.retry_count = req_data->retry_count; event->param.conn.rnr_retry_count = req_data->rnr_retry_count; event->param.conn.srq = req_data->srq; event->param.conn.qp_num = req_data->remote_qpn; } static int cma_check_req_qp_type(struct rdma_cm_id *id, struct ib_cm_event *ib_event) { return (((ib_event->event == IB_CM_REQ_RECEIVED) && (ib_event->param.req_rcvd.qp_type == id->qp_type)) || ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) && (id->qp_type == IB_QPT_UD)) || (!id->qp_type)); } static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) { struct rdma_id_private *listen_id, *conn_id = NULL; struct rdma_cm_event event; struct ifnet *net_dev; int offset, ret; listen_id = cma_id_from_event(cm_id, ib_event, &net_dev); if (IS_ERR(listen_id)) return PTR_ERR(listen_id); if (!cma_check_req_qp_type(&listen_id->id, ib_event)) { ret = -EINVAL; goto net_dev_put; } mutex_lock(&listen_id->handler_mutex); if (listen_id->state != RDMA_CM_LISTEN) { ret = -ECONNABORTED; goto err1; } memset(&event, 0, sizeof event); offset = cma_user_data_offset(listen_id); event.event = RDMA_CM_EVENT_CONNECT_REQUEST; if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) { conn_id = cma_new_udp_id(&listen_id->id, ib_event, net_dev); event.param.ud.private_data = (char *)ib_event->private_data + offset; event.param.ud.private_data_len = IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset; } else { conn_id = cma_new_conn_id(&listen_id->id, ib_event, net_dev); cma_set_req_event_data(&event, &ib_event->param.req_rcvd, ib_event->private_data, offset); } if (!conn_id) { ret = -ENOMEM; goto err1; } mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); ret = cma_acquire_dev(conn_id, listen_id); if (ret) goto err2; conn_id->cm_id.ib = cm_id; cm_id->context = conn_id; cm_id->cm_handler = cma_ib_handler; /* * Protect against the user destroying conn_id from another thread * until we're done accessing it. */ atomic_inc(&conn_id->refcount); ret = conn_id->id.event_handler(&conn_id->id, &event); if (ret) goto err3; /* * Acquire mutex to prevent user executing rdma_destroy_id() * while we're accessing the cm_id. */ mutex_lock(&lock); if (cma_comp(conn_id, RDMA_CM_CONNECT) && (conn_id->id.qp_type != IB_QPT_UD)) ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); mutex_unlock(&lock); mutex_unlock(&conn_id->handler_mutex); mutex_unlock(&listen_id->handler_mutex); cma_deref_id(conn_id); if (net_dev) dev_put(net_dev); return 0; err3: cma_deref_id(conn_id); /* Destroy the CM ID by returning a non-zero value. */ conn_id->cm_id.ib = NULL; err2: cma_exch(conn_id, RDMA_CM_DESTROYING); mutex_unlock(&conn_id->handler_mutex); err1: mutex_unlock(&listen_id->handler_mutex); if (conn_id) rdma_destroy_id(&conn_id->id); net_dev_put: if (net_dev) dev_put(net_dev); return ret; } __be64 rdma_get_service_id(struct rdma_cm_id *id, struct sockaddr *addr) { if (addr->sa_family == AF_IB) return ((struct sockaddr_ib *) addr)->sib_sid; return cpu_to_be64(((u64)id->ps << 16) + be16_to_cpu(cma_port(addr))); } EXPORT_SYMBOL(rdma_get_service_id); static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) { struct rdma_id_private *id_priv = iw_id->context; struct rdma_cm_event event; int ret = 0; struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr; struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr; mutex_lock(&id_priv->handler_mutex); if (id_priv->state != RDMA_CM_CONNECT) goto out; memset(&event, 0, sizeof event); switch (iw_event->event) { case IW_CM_EVENT_CLOSE: event.event = RDMA_CM_EVENT_DISCONNECTED; break; case IW_CM_EVENT_CONNECT_REPLY: memcpy(cma_src_addr(id_priv), laddr, rdma_addr_size(laddr)); memcpy(cma_dst_addr(id_priv), raddr, rdma_addr_size(raddr)); switch (iw_event->status) { case 0: event.event = RDMA_CM_EVENT_ESTABLISHED; event.param.conn.initiator_depth = iw_event->ird; event.param.conn.responder_resources = iw_event->ord; break; case -ECONNRESET: case -ECONNREFUSED: event.event = RDMA_CM_EVENT_REJECTED; break; case -ETIMEDOUT: event.event = RDMA_CM_EVENT_UNREACHABLE; break; default: event.event = RDMA_CM_EVENT_CONNECT_ERROR; break; } break; case IW_CM_EVENT_ESTABLISHED: event.event = RDMA_CM_EVENT_ESTABLISHED; event.param.conn.initiator_depth = iw_event->ird; event.param.conn.responder_resources = iw_event->ord; break; default: BUG_ON(1); } event.status = iw_event->status; event.param.conn.private_data = iw_event->private_data; event.param.conn.private_data_len = iw_event->private_data_len; ret = id_priv->id.event_handler(&id_priv->id, &event); if (ret) { /* Destroy the CM ID by returning a non-zero value. */ id_priv->cm_id.iw = NULL; cma_exch(id_priv, RDMA_CM_DESTROYING); mutex_unlock(&id_priv->handler_mutex); rdma_destroy_id(&id_priv->id); return ret; } out: mutex_unlock(&id_priv->handler_mutex); return ret; } static int iw_conn_req_handler(struct iw_cm_id *cm_id, struct iw_cm_event *iw_event) { struct rdma_cm_id *new_cm_id; struct rdma_id_private *listen_id, *conn_id; struct rdma_cm_event event; int ret = -ECONNABORTED; struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr; struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr; listen_id = cm_id->context; mutex_lock(&listen_id->handler_mutex); if (listen_id->state != RDMA_CM_LISTEN) goto out; /* Create a new RDMA id for the new IW CM ID */ new_cm_id = rdma_create_id(listen_id->id.route.addr.dev_addr.net, listen_id->id.event_handler, listen_id->id.context, RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(new_cm_id)) { ret = -ENOMEM; goto out; } conn_id = container_of(new_cm_id, struct rdma_id_private, id); mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); conn_id->state = RDMA_CM_CONNECT; ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr); if (ret) { mutex_unlock(&conn_id->handler_mutex); rdma_destroy_id(new_cm_id); goto out; } ret = cma_acquire_dev(conn_id, listen_id); if (ret) { mutex_unlock(&conn_id->handler_mutex); rdma_destroy_id(new_cm_id); goto out; } conn_id->cm_id.iw = cm_id; cm_id->context = conn_id; cm_id->cm_handler = cma_iw_handler; memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr)); memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr)); memset(&event, 0, sizeof event); event.event = RDMA_CM_EVENT_CONNECT_REQUEST; event.param.conn.private_data = iw_event->private_data; event.param.conn.private_data_len = iw_event->private_data_len; event.param.conn.initiator_depth = iw_event->ird; event.param.conn.responder_resources = iw_event->ord; /* * Protect against the user destroying conn_id from another thread * until we're done accessing it. */ atomic_inc(&conn_id->refcount); ret = conn_id->id.event_handler(&conn_id->id, &event); if (ret) { /* User wants to destroy the CM ID */ conn_id->cm_id.iw = NULL; cma_exch(conn_id, RDMA_CM_DESTROYING); mutex_unlock(&conn_id->handler_mutex); cma_deref_id(conn_id); rdma_destroy_id(&conn_id->id); goto out; } mutex_unlock(&conn_id->handler_mutex); cma_deref_id(conn_id); out: mutex_unlock(&listen_id->handler_mutex); return ret; } static int cma_ib_listen(struct rdma_id_private *id_priv) { struct sockaddr *addr; struct ib_cm_id *id; __be64 svc_id; addr = cma_src_addr(id_priv); svc_id = rdma_get_service_id(&id_priv->id, addr); id = ib_cm_insert_listen(id_priv->id.device, cma_req_handler, svc_id); if (IS_ERR(id)) return PTR_ERR(id); id_priv->cm_id.ib = id; return 0; } static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog) { int ret; struct iw_cm_id *id; id = iw_create_cm_id(id_priv->id.device, iw_conn_req_handler, id_priv); if (IS_ERR(id)) return PTR_ERR(id); id->tos = id_priv->tos; id_priv->cm_id.iw = id; memcpy(&id_priv->cm_id.iw->local_addr, cma_src_addr(id_priv), rdma_addr_size(cma_src_addr(id_priv))); ret = iw_cm_listen(id_priv->cm_id.iw, backlog); if (ret) { iw_destroy_cm_id(id_priv->cm_id.iw); id_priv->cm_id.iw = NULL; } return ret; } static int cma_listen_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) { struct rdma_id_private *id_priv = id->context; id->context = id_priv->id.context; id->event_handler = id_priv->id.event_handler; return id_priv->id.event_handler(id, event); } static void cma_listen_on_dev(struct rdma_id_private *id_priv, struct cma_device *cma_dev) { struct rdma_id_private *dev_id_priv; struct rdma_cm_id *id; struct vnet *net = id_priv->id.route.addr.dev_addr.net; int ret; if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1)) return; id = rdma_create_id(net, cma_listen_handler, id_priv, id_priv->id.ps, id_priv->id.qp_type); if (IS_ERR(id)) return; dev_id_priv = container_of(id, struct rdma_id_private, id); dev_id_priv->state = RDMA_CM_ADDR_BOUND; memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv), rdma_addr_size(cma_src_addr(id_priv))); _cma_attach_to_dev(dev_id_priv, cma_dev); list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list); atomic_inc(&id_priv->refcount); dev_id_priv->internal_id = 1; dev_id_priv->afonly = id_priv->afonly; ret = rdma_listen(id, id_priv->backlog); if (ret) pr_warn("RDMA CMA: cma_listen_on_dev, error %d, listening on device %s\n", ret, cma_dev->device->name); } static void cma_listen_on_all(struct rdma_id_private *id_priv) { struct cma_device *cma_dev; mutex_lock(&lock); list_add_tail(&id_priv->list, &listen_any_list); list_for_each_entry(cma_dev, &dev_list, list) cma_listen_on_dev(id_priv, cma_dev); mutex_unlock(&lock); } void rdma_set_service_type(struct rdma_cm_id *id, int tos) { struct rdma_id_private *id_priv; id_priv = container_of(id, struct rdma_id_private, id); id_priv->tos = (u8) tos; } EXPORT_SYMBOL(rdma_set_service_type); /** * rdma_set_ack_timeout() - Set the ack timeout of QP associated * with a connection identifier. * @id: Communication identifier to associated with service type. * @timeout: Ack timeout to set a QP, expressed as 4.096 * 2^(timeout) usec. * * This function should be called before rdma_connect() on active side, * and on passive side before rdma_accept(). It is applicable to primary * path only. The timeout will affect the local side of the QP, it is not * negotiated with remote side and zero disables the timer. * * Return: 0 for success */ int rdma_set_ack_timeout(struct rdma_cm_id *id, u8 timeout) { struct rdma_id_private *id_priv; if (id->qp_type != IB_QPT_RC) return -EINVAL; id_priv = container_of(id, struct rdma_id_private, id); id_priv->timeout = timeout; id_priv->timeout_set = true; return 0; } EXPORT_SYMBOL(rdma_set_ack_timeout); static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec, void *context) { struct cma_work *work = context; struct rdma_route *route; route = &work->id->id.route; if (!status) { route->num_paths = 1; *route->path_rec = *path_rec; } else { work->old_state = RDMA_CM_ROUTE_QUERY; work->new_state = RDMA_CM_ADDR_RESOLVED; work->event.event = RDMA_CM_EVENT_ROUTE_ERROR; work->event.status = status; } queue_work(cma_wq, &work->work); } static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms, struct cma_work *work) { struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; struct ib_sa_path_rec path_rec; ib_sa_comp_mask comp_mask; struct sockaddr_in6 *sin6; struct sockaddr_ib *sib; memset(&path_rec, 0, sizeof path_rec); rdma_addr_get_sgid(dev_addr, &path_rec.sgid); rdma_addr_get_dgid(dev_addr, &path_rec.dgid); path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); path_rec.numb_path = 1; path_rec.reversible = 1; path_rec.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv)); comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID | IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH | IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID; switch (cma_family(id_priv)) { case AF_INET: path_rec.qos_class = cpu_to_be16((u16) id_priv->tos); comp_mask |= IB_SA_PATH_REC_QOS_CLASS; break; case AF_INET6: sin6 = (struct sockaddr_in6 *) cma_src_addr(id_priv); path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20); comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS; break; case AF_IB: sib = (struct sockaddr_ib *) cma_src_addr(id_priv); path_rec.traffic_class = (u8) (be32_to_cpu(sib->sib_flowinfo) >> 20); comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS; break; } id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device, id_priv->id.port_num, &path_rec, comp_mask, timeout_ms, GFP_KERNEL, cma_query_handler, work, &id_priv->query); return (id_priv->query_id < 0) ? id_priv->query_id : 0; } static void cma_work_handler(struct work_struct *_work) { struct cma_work *work = container_of(_work, struct cma_work, work); struct rdma_id_private *id_priv = work->id; int destroy = 0; mutex_lock(&id_priv->handler_mutex); if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) goto out; if (id_priv->id.event_handler(&id_priv->id, &work->event)) { cma_exch(id_priv, RDMA_CM_DESTROYING); destroy = 1; } out: mutex_unlock(&id_priv->handler_mutex); cma_deref_id(id_priv); if (destroy) rdma_destroy_id(&id_priv->id); kfree(work); } static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms) { struct rdma_route *route = &id_priv->id.route; struct cma_work *work; int ret; work = kzalloc(sizeof *work, GFP_KERNEL); if (!work) return -ENOMEM; work->id = id_priv; INIT_WORK(&work->work, cma_work_handler); work->old_state = RDMA_CM_ROUTE_QUERY; work->new_state = RDMA_CM_ROUTE_RESOLVED; work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL); if (!route->path_rec) { ret = -ENOMEM; goto err1; } ret = cma_query_ib_route(id_priv, timeout_ms, work); if (ret) goto err2; return 0; err2: kfree(route->path_rec); route->path_rec = NULL; err1: kfree(work); return ret; } int rdma_set_ib_paths(struct rdma_cm_id *id, struct ib_sa_path_rec *path_rec, int num_paths) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_RESOLVED)) return -EINVAL; id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths, GFP_KERNEL); if (!id->route.path_rec) { ret = -ENOMEM; goto err; } id->route.num_paths = num_paths; return 0; err: cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED); return ret; } EXPORT_SYMBOL(rdma_set_ib_paths); static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms) { struct cma_work *work; work = kzalloc(sizeof *work, GFP_KERNEL); if (!work) return -ENOMEM; work->id = id_priv; INIT_WORK(&work->work, cma_work_handler); work->old_state = RDMA_CM_ROUTE_QUERY; work->new_state = RDMA_CM_ROUTE_RESOLVED; work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; queue_work(cma_wq, &work->work); return 0; } static int iboe_tos_to_sl(struct ifnet *ndev, int tos) { /* get service level, SL, from IPv4 type of service, TOS */ int sl = (tos >> 5) & 0x7; /* final mappings are done by the vendor specific drivers */ return sl; } static enum ib_gid_type cma_route_gid_type(enum rdma_network_type network_type, unsigned long supported_gids, enum ib_gid_type default_gid) { if ((network_type == RDMA_NETWORK_IPV4 || network_type == RDMA_NETWORK_IPV6) && test_bit(IB_GID_TYPE_ROCE_UDP_ENCAP, &supported_gids)) return IB_GID_TYPE_ROCE_UDP_ENCAP; return default_gid; } static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) { struct rdma_route *route = &id_priv->id.route; struct rdma_addr *addr = &route->addr; struct cma_work *work; int ret; struct ifnet *ndev = NULL; work = kzalloc(sizeof *work, GFP_KERNEL); if (!work) return -ENOMEM; work->id = id_priv; INIT_WORK(&work->work, cma_work_handler); route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL); if (!route->path_rec) { ret = -ENOMEM; goto err1; } route->num_paths = 1; if (addr->dev_addr.bound_dev_if) { unsigned long supported_gids; ndev = dev_get_by_index(addr->dev_addr.net, addr->dev_addr.bound_dev_if); if (!ndev) { ret = -ENODEV; goto err2; } route->path_rec->net = ndev->if_vnet; route->path_rec->ifindex = ndev->if_index; supported_gids = roce_gid_type_mask_support(id_priv->id.device, id_priv->id.port_num); route->path_rec->gid_type = cma_route_gid_type(addr->dev_addr.network, supported_gids, id_priv->gid_type); } if (!ndev) { ret = -ENODEV; goto err2; } memcpy(route->path_rec->dmac, addr->dev_addr.dst_dev_addr, ETH_ALEN); rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, &route->path_rec->sgid); rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr, &route->path_rec->dgid); /* Use the hint from IP Stack to select GID Type */ if (route->path_rec->gid_type < ib_network_to_gid_type(addr->dev_addr.network)) route->path_rec->gid_type = ib_network_to_gid_type(addr->dev_addr.network); if (((struct sockaddr *)&id_priv->id.route.addr.dst_addr)->sa_family != AF_IB) /* TODO: get the hoplimit from the inet/inet6 device */ route->path_rec->hop_limit = addr->dev_addr.hoplimit; else route->path_rec->hop_limit = 1; route->path_rec->reversible = 1; route->path_rec->pkey = cpu_to_be16(0xffff); route->path_rec->mtu_selector = IB_SA_EQ; route->path_rec->sl = iboe_tos_to_sl(ndev, id_priv->tos); route->path_rec->traffic_class = id_priv->tos; route->path_rec->mtu = iboe_get_mtu(ndev->if_mtu); route->path_rec->rate_selector = IB_SA_EQ; route->path_rec->rate = iboe_get_rate(ndev); dev_put(ndev); route->path_rec->packet_life_time_selector = IB_SA_EQ; route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME; if (!route->path_rec->mtu) { ret = -EINVAL; goto err2; } work->old_state = RDMA_CM_ROUTE_QUERY; work->new_state = RDMA_CM_ROUTE_RESOLVED; work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; work->event.status = 0; queue_work(cma_wq, &work->work); return 0; err2: kfree(route->path_rec); route->path_rec = NULL; err1: kfree(work); return ret; } int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY)) return -EINVAL; atomic_inc(&id_priv->refcount); if (rdma_cap_ib_sa(id->device, id->port_num)) ret = cma_resolve_ib_route(id_priv, timeout_ms); else if (rdma_protocol_roce(id->device, id->port_num)) ret = cma_resolve_iboe_route(id_priv); else if (rdma_protocol_iwarp(id->device, id->port_num)) ret = cma_resolve_iw_route(id_priv, timeout_ms); else ret = -ENOSYS; if (ret) goto err; return 0; err: cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED); cma_deref_id(id_priv); return ret; } EXPORT_SYMBOL(rdma_resolve_route); static void cma_set_loopback(struct sockaddr *addr) { switch (addr->sa_family) { case AF_INET: ((struct sockaddr_in *) addr)->sin_addr.s_addr = htonl(INADDR_LOOPBACK); break; case AF_INET6: ipv6_addr_set(&((struct sockaddr_in6 *) addr)->sin6_addr, 0, 0, 0, htonl(1)); break; default: ib_addr_set(&((struct sockaddr_ib *) addr)->sib_addr, 0, 0, 0, htonl(1)); break; } } static int cma_bind_loopback(struct rdma_id_private *id_priv) { struct cma_device *cma_dev, *cur_dev; struct ib_port_attr port_attr; union ib_gid gid; u16 pkey; int ret; u8 p; cma_dev = NULL; mutex_lock(&lock); list_for_each_entry(cur_dev, &dev_list, list) { if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cur_dev->device, 1)) continue; if (!cma_dev) cma_dev = cur_dev; for (p = 1; p <= cur_dev->device->phys_port_cnt; ++p) { if (!ib_query_port(cur_dev->device, p, &port_attr) && port_attr.state == IB_PORT_ACTIVE) { cma_dev = cur_dev; goto port_found; } } } if (!cma_dev) { ret = -ENODEV; goto out; } p = 1; port_found: ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid, NULL); if (ret) goto out; ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey); if (ret) goto out; id_priv->id.route.addr.dev_addr.dev_type = (rdma_protocol_ib(cma_dev->device, p)) ? ARPHRD_INFINIBAND : ARPHRD_ETHER; rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid); ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey); id_priv->id.port_num = p; cma_attach_to_dev(id_priv, cma_dev); cma_set_loopback(cma_src_addr(id_priv)); out: mutex_unlock(&lock); return ret; } static void addr_handler(int status, struct sockaddr *src_addr, struct rdma_dev_addr *dev_addr, void *context) { struct rdma_id_private *id_priv = context; struct rdma_cm_event event; memset(&event, 0, sizeof event); mutex_lock(&id_priv->handler_mutex); if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_RESOLVED)) goto out; memcpy(cma_src_addr(id_priv), src_addr, rdma_addr_size(src_addr)); if (!status && !id_priv->cma_dev) status = cma_acquire_dev(id_priv, NULL); if (status) { if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ADDR_BOUND)) goto out; event.event = RDMA_CM_EVENT_ADDR_ERROR; event.status = status; } else event.event = RDMA_CM_EVENT_ADDR_RESOLVED; if (id_priv->id.event_handler(&id_priv->id, &event)) { cma_exch(id_priv, RDMA_CM_DESTROYING); mutex_unlock(&id_priv->handler_mutex); cma_deref_id(id_priv); rdma_destroy_id(&id_priv->id); return; } out: mutex_unlock(&id_priv->handler_mutex); cma_deref_id(id_priv); } static int cma_resolve_loopback(struct rdma_id_private *id_priv) { struct cma_work *work; union ib_gid gid; int ret; work = kzalloc(sizeof *work, GFP_KERNEL); if (!work) return -ENOMEM; if (!id_priv->cma_dev) { ret = cma_bind_loopback(id_priv); if (ret) goto err; } rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid); rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid); work->id = id_priv; INIT_WORK(&work->work, cma_work_handler); work->old_state = RDMA_CM_ADDR_QUERY; work->new_state = RDMA_CM_ADDR_RESOLVED; work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; queue_work(cma_wq, &work->work); return 0; err: kfree(work); return ret; } static int cma_resolve_ib_addr(struct rdma_id_private *id_priv) { struct cma_work *work; int ret; work = kzalloc(sizeof *work, GFP_KERNEL); if (!work) return -ENOMEM; if (!id_priv->cma_dev) { ret = cma_resolve_ib_dev(id_priv); if (ret) goto err; } rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *) &(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr)); work->id = id_priv; INIT_WORK(&work->work, cma_work_handler); work->old_state = RDMA_CM_ADDR_QUERY; work->new_state = RDMA_CM_ADDR_RESOLVED; work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; queue_work(cma_wq, &work->work); return 0; err: kfree(work); return ret; } static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, struct sockaddr *dst_addr) { if (!src_addr || !src_addr->sa_family) { src_addr = (struct sockaddr *) &id->route.addr.src_addr; src_addr->sa_family = dst_addr->sa_family; if (dst_addr->sa_family == AF_INET6) { struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *) src_addr; struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *) dst_addr; src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id; if (IN6_IS_SCOPE_LINKLOCAL(&dst_addr6->sin6_addr) || IN6_IS_ADDR_MC_INTFACELOCAL(&dst_addr6->sin6_addr)) id->route.addr.dev_addr.bound_dev_if = dst_addr6->sin6_scope_id; } else if (dst_addr->sa_family == AF_IB) { ((struct sockaddr_ib *) src_addr)->sib_pkey = ((struct sockaddr_ib *) dst_addr)->sib_pkey; } } return rdma_bind_addr(id, src_addr); } int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, struct sockaddr *dst_addr, int timeout_ms) { struct rdma_id_private *id_priv; + struct vnet *vnet = id->route.addr.dev_addr.net; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (id_priv->state == RDMA_CM_IDLE) { ret = cma_bind_addr(id, src_addr, dst_addr); if (ret) return ret; } if (cma_family(id_priv) != dst_addr->sa_family) return -EINVAL; if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) return -EINVAL; atomic_inc(&id_priv->refcount); memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr)); - if (cma_any_addr(dst_addr)) { + if (cma_any_addr(vnet, dst_addr)) { ret = cma_resolve_loopback(id_priv); } else { if (dst_addr->sa_family == AF_IB) { ret = cma_resolve_ib_addr(id_priv); } else { ret = cma_check_linklocal(&id->route.addr.dev_addr, dst_addr); if (ret) goto err; ret = rdma_resolve_ip(&addr_client, cma_src_addr(id_priv), dst_addr, &id->route.addr.dev_addr, timeout_ms, addr_handler, id_priv); } } if (ret) goto err; return 0; err: cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND); cma_deref_id(id_priv); return ret; } EXPORT_SYMBOL(rdma_resolve_addr); int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse) { struct rdma_id_private *id_priv; unsigned long flags; int ret; id_priv = container_of(id, struct rdma_id_private, id); spin_lock_irqsave(&id_priv->lock, flags); if (reuse || id_priv->state == RDMA_CM_IDLE) { id_priv->reuseaddr = reuse; ret = 0; } else { ret = -EINVAL; } spin_unlock_irqrestore(&id_priv->lock, flags); return ret; } EXPORT_SYMBOL(rdma_set_reuseaddr); int rdma_set_afonly(struct rdma_cm_id *id, int afonly) { struct rdma_id_private *id_priv; unsigned long flags; int ret; id_priv = container_of(id, struct rdma_id_private, id); spin_lock_irqsave(&id_priv->lock, flags); if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) { id_priv->options |= (1 << CMA_OPTION_AFONLY); id_priv->afonly = afonly; ret = 0; } else { ret = -EINVAL; } spin_unlock_irqrestore(&id_priv->lock, flags); return ret; } EXPORT_SYMBOL(rdma_set_afonly); static void cma_bind_port(struct rdma_bind_list *bind_list, struct rdma_id_private *id_priv) { struct sockaddr *addr; struct sockaddr_ib *sib; u64 sid, mask; __be16 port; addr = cma_src_addr(id_priv); port = htons(bind_list->port); switch (addr->sa_family) { case AF_INET: ((struct sockaddr_in *) addr)->sin_port = port; break; case AF_INET6: ((struct sockaddr_in6 *) addr)->sin6_port = port; break; case AF_IB: sib = (struct sockaddr_ib *) addr; sid = be64_to_cpu(sib->sib_sid); mask = be64_to_cpu(sib->sib_sid_mask); sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port)); sib->sib_sid_mask = cpu_to_be64(~0ULL); break; } id_priv->bind_list = bind_list; hlist_add_head(&id_priv->node, &bind_list->owners); } static int cma_alloc_port(enum rdma_port_space ps, struct rdma_id_private *id_priv, unsigned short snum) { struct rdma_bind_list *bind_list; int ret; bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL); if (!bind_list) return -ENOMEM; ret = cma_ps_alloc(id_priv->id.route.addr.dev_addr.net, ps, bind_list, snum); if (ret < 0) goto err; bind_list->ps = ps; bind_list->port = (unsigned short)ret; cma_bind_port(bind_list, id_priv); return 0; err: kfree(bind_list); return ret == -ENOSPC ? -EADDRNOTAVAIL : ret; } static int cma_alloc_any_port(enum rdma_port_space ps, struct rdma_id_private *id_priv) { static unsigned int last_used_port; int low, high, remaining; unsigned int rover; struct vnet *net = id_priv->id.route.addr.dev_addr.net; u32 rand; inet_get_local_port_range(net, &low, &high); remaining = (high - low) + 1; get_random_bytes(&rand, sizeof(rand)); rover = rand % remaining + low; retry: if (last_used_port != rover && !cma_ps_find(net, ps, (unsigned short)rover)) { int ret = cma_alloc_port(ps, id_priv, rover); /* * Remember previously used port number in order to avoid * re-using same port immediately after it is closed. */ if (!ret) last_used_port = rover; if (ret != -EADDRNOTAVAIL) return ret; } if (--remaining) { rover++; if ((rover < low) || (rover > high)) rover = low; goto retry; } return -EADDRNOTAVAIL; } /* * Check that the requested port is available. This is called when trying to * bind to a specific port, or when trying to listen on a bound port. In * the latter case, the provided id_priv may already be on the bind_list, but * we still need to check that it's okay to start listening. */ static int cma_check_port(struct rdma_bind_list *bind_list, struct rdma_id_private *id_priv, uint8_t reuseaddr) { struct rdma_id_private *cur_id; struct sockaddr *addr, *cur_addr; + struct vnet *vnet; addr = cma_src_addr(id_priv); hlist_for_each_entry(cur_id, &bind_list->owners, node) { if (id_priv == cur_id) continue; if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr && cur_id->reuseaddr) continue; cur_addr = cma_src_addr(cur_id); if (id_priv->afonly && cur_id->afonly && (addr->sa_family != cur_addr->sa_family)) continue; - if (cma_any_addr(addr) || cma_any_addr(cur_addr)) + vnet = cur_id->id.route.addr.dev_addr.net; + if (cma_any_addr(vnet, addr) || cma_any_addr(vnet, cur_addr)) return -EADDRNOTAVAIL; if (!cma_addr_cmp(addr, cur_addr)) return -EADDRINUSE; } return 0; } static int cma_use_port(enum rdma_port_space ps, struct rdma_id_private *id_priv) { struct rdma_bind_list *bind_list; unsigned short snum; int ret; snum = ntohs(cma_port(cma_src_addr(id_priv))); if (snum < IPPORT_RESERVED && priv_check(curthread, PRIV_NETINET_BINDANY) != 0) return -EACCES; bind_list = cma_ps_find(id_priv->id.route.addr.dev_addr.net, ps, snum); if (!bind_list) { ret = cma_alloc_port(ps, id_priv, snum); } else { ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr); if (!ret) cma_bind_port(bind_list, id_priv); } return ret; } static int cma_bind_listen(struct rdma_id_private *id_priv) { struct rdma_bind_list *bind_list = id_priv->bind_list; int ret = 0; mutex_lock(&lock); if (bind_list->owners.first->next) ret = cma_check_port(bind_list, id_priv, 0); mutex_unlock(&lock); return ret; } static enum rdma_port_space cma_select_inet_ps( struct rdma_id_private *id_priv) { switch (id_priv->id.ps) { case RDMA_PS_TCP: case RDMA_PS_UDP: case RDMA_PS_IPOIB: case RDMA_PS_IB: case RDMA_PS_SDP: return id_priv->id.ps; default: return 0; } } static enum rdma_port_space cma_select_ib_ps(struct rdma_id_private *id_priv) { enum rdma_port_space ps = 0; struct sockaddr_ib *sib; u64 sid_ps, mask, sid; sib = (struct sockaddr_ib *) cma_src_addr(id_priv); mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK; sid = be64_to_cpu(sib->sib_sid) & mask; if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) { sid_ps = RDMA_IB_IP_PS_IB; ps = RDMA_PS_IB; } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) && (sid == (RDMA_IB_IP_PS_TCP & mask))) { sid_ps = RDMA_IB_IP_PS_TCP; ps = RDMA_PS_TCP; } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) && (sid == (RDMA_IB_IP_PS_UDP & mask))) { sid_ps = RDMA_IB_IP_PS_UDP; ps = RDMA_PS_UDP; } if (ps) { sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib))); sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK | be64_to_cpu(sib->sib_sid_mask)); } return ps; } static int cma_get_port(struct rdma_id_private *id_priv) { enum rdma_port_space ps; int ret; if (cma_family(id_priv) != AF_IB) ps = cma_select_inet_ps(id_priv); else ps = cma_select_ib_ps(id_priv); if (!ps) return -EPROTONOSUPPORT; mutex_lock(&lock); if (cma_any_port(cma_src_addr(id_priv))) ret = cma_alloc_any_port(ps, id_priv); else ret = cma_use_port(ps, id_priv); mutex_unlock(&lock); return ret; } static int cma_check_linklocal(struct rdma_dev_addr *dev_addr, struct sockaddr *addr) { #ifdef INET6 struct sockaddr_in6 sin6; if (addr->sa_family != AF_INET6) return 0; sin6 = *(struct sockaddr_in6 *)addr; if (IN6_IS_SCOPE_LINKLOCAL(&sin6.sin6_addr) || IN6_IS_ADDR_MC_INTFACELOCAL(&sin6.sin6_addr)) { bool failure; CURVNET_SET_QUIET(dev_addr->net); failure = sa6_recoverscope(&sin6) || sin6.sin6_scope_id == 0; CURVNET_RESTORE(); /* check if IPv6 scope ID is not set */ if (failure) return -EINVAL; dev_addr->bound_dev_if = sin6.sin6_scope_id; } #endif return 0; } int rdma_listen(struct rdma_cm_id *id, int backlog) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (id_priv->state == RDMA_CM_IDLE) { id->route.addr.src_addr.ss_family = AF_INET; ret = rdma_bind_addr(id, cma_src_addr(id_priv)); if (ret) return ret; } if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN)) return -EINVAL; if (id_priv->reuseaddr) { ret = cma_bind_listen(id_priv); if (ret) goto err; } id_priv->backlog = backlog; if (id->device) { if (rdma_cap_ib_cm(id->device, 1)) { ret = cma_ib_listen(id_priv); if (ret) goto err; } else if (rdma_cap_iw_cm(id->device, 1)) { ret = cma_iw_listen(id_priv, backlog); if (ret) goto err; } else { ret = -ENOSYS; goto err; } } else cma_listen_on_all(id_priv); return 0; err: id_priv->backlog = 0; cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND); return ret; } EXPORT_SYMBOL(rdma_listen); int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) { struct rdma_id_private *id_priv; + struct vnet *vnet = id->route.addr.dev_addr.net; int ret; if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 && addr->sa_family != AF_IB) return -EAFNOSUPPORT; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND)) return -EINVAL; ret = cma_check_linklocal(&id->route.addr.dev_addr, addr); if (ret) goto err1; memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr)); - if (!cma_any_addr(addr)) { + if (!cma_any_addr(vnet, addr)) { ret = cma_translate_addr(addr, &id->route.addr.dev_addr); if (ret) goto err1; ret = cma_acquire_dev(id_priv, NULL); if (ret) goto err1; } if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) { if (addr->sa_family == AF_INET) id_priv->afonly = 1; #ifdef INET6 else if (addr->sa_family == AF_INET6) { - CURVNET_SET_QUIET(id_priv->id.route.addr.dev_addr.net); + CURVNET_SET_QUIET(vnet); id_priv->afonly = V_ip6_v6only; CURVNET_RESTORE(); } #endif } ret = cma_get_port(id_priv); if (ret) goto err2; return 0; err2: if (id_priv->cma_dev) cma_release_dev(id_priv); err1: cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE); return ret; } EXPORT_SYMBOL(rdma_bind_addr); static int sdp_format_hdr(struct sdp_hh *sdp_hdr, struct rdma_id_private *id_priv) { /* * XXXCEM: CMA just sets the version itself rather than relying on * passed in packet to have the major version set. Should we? */ if (sdp_get_majv(sdp_hdr->majv_minv) != SDP_MAJ_VERSION) return -EINVAL; if (cma_family(id_priv) == AF_INET) { struct sockaddr_in *src4, *dst4; src4 = (struct sockaddr_in *) cma_src_addr(id_priv); dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv); sdp_set_ip_ver(sdp_hdr, 4); sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; sdp_hdr->port = src4->sin_port; } else if (cma_family(id_priv) == AF_INET6) { struct sockaddr_in6 *src6, *dst6; src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv); dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv); sdp_set_ip_ver(sdp_hdr, 6); sdp_hdr->src_addr.ip6 = src6->sin6_addr; sdp_hdr->dst_addr.ip6 = dst6->sin6_addr; sdp_hdr->port = src6->sin6_port; cma_ip6_clear_scope_id(&sdp_hdr->src_addr.ip6); cma_ip6_clear_scope_id(&sdp_hdr->dst_addr.ip6); } else return -EAFNOSUPPORT; return 0; } static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv) { struct cma_hdr *cma_hdr; if (id_priv->id.ps == RDMA_PS_SDP) return sdp_format_hdr(hdr, id_priv); cma_hdr = hdr; cma_hdr->cma_version = CMA_VERSION; if (cma_family(id_priv) == AF_INET) { struct sockaddr_in *src4, *dst4; src4 = (struct sockaddr_in *) cma_src_addr(id_priv); dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv); cma_set_ip_ver(cma_hdr, 4); cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; cma_hdr->port = src4->sin_port; } else if (cma_family(id_priv) == AF_INET6) { struct sockaddr_in6 *src6, *dst6; src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv); dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv); cma_set_ip_ver(cma_hdr, 6); cma_hdr->src_addr.ip6 = src6->sin6_addr; cma_hdr->dst_addr.ip6 = dst6->sin6_addr; cma_hdr->port = src6->sin6_port; cma_ip6_clear_scope_id(&cma_hdr->src_addr.ip6); cma_ip6_clear_scope_id(&cma_hdr->dst_addr.ip6); } return 0; } static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) { struct rdma_id_private *id_priv = cm_id->context; struct rdma_cm_event event; struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd; int ret = 0; mutex_lock(&id_priv->handler_mutex); if (id_priv->state != RDMA_CM_CONNECT) goto out; memset(&event, 0, sizeof event); switch (ib_event->event) { case IB_CM_SIDR_REQ_ERROR: event.event = RDMA_CM_EVENT_UNREACHABLE; event.status = -ETIMEDOUT; break; case IB_CM_SIDR_REP_RECEIVED: event.param.ud.private_data = ib_event->private_data; event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE; if (rep->status != IB_SIDR_SUCCESS) { event.event = RDMA_CM_EVENT_UNREACHABLE; event.status = ib_event->param.sidr_rep_rcvd.status; break; } ret = cma_set_qkey(id_priv, rep->qkey); if (ret) { event.event = RDMA_CM_EVENT_ADDR_ERROR; event.status = ret; break; } ret = ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num, id_priv->id.route.path_rec, &event.param.ud.ah_attr); if (ret) { event.event = RDMA_CM_EVENT_ADDR_ERROR; event.status = ret; break; } event.param.ud.qp_num = rep->qpn; event.param.ud.qkey = rep->qkey; event.event = RDMA_CM_EVENT_ESTABLISHED; event.status = 0; break; default: pr_err("RDMA CMA: unexpected IB CM event: %d\n", ib_event->event); goto out; } ret = id_priv->id.event_handler(&id_priv->id, &event); if (ret) { /* Destroy the CM ID by returning a non-zero value. */ id_priv->cm_id.ib = NULL; cma_exch(id_priv, RDMA_CM_DESTROYING); mutex_unlock(&id_priv->handler_mutex); rdma_destroy_id(&id_priv->id); return ret; } out: mutex_unlock(&id_priv->handler_mutex); return ret; } static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct ib_cm_sidr_req_param req; struct ib_cm_id *id; void *private_data; int offset, ret; memset(&req, 0, sizeof req); offset = cma_user_data_offset(id_priv); req.private_data_len = offset + conn_param->private_data_len; if (req.private_data_len < conn_param->private_data_len) return -EINVAL; if (req.private_data_len) { private_data = kzalloc(req.private_data_len, GFP_ATOMIC); if (!private_data) return -ENOMEM; } else { private_data = NULL; } if (conn_param->private_data && conn_param->private_data_len) memcpy((char *)private_data + offset, conn_param->private_data, conn_param->private_data_len); if (private_data) { ret = cma_format_hdr(private_data, id_priv); if (ret) goto out; req.private_data = private_data; } id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler, id_priv); if (IS_ERR(id)) { ret = PTR_ERR(id); goto out; } id_priv->cm_id.ib = id; req.path = id_priv->id.route.path_rec; req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv)); req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8); req.max_cm_retries = CMA_MAX_CM_RETRIES; ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req); if (ret) { ib_destroy_cm_id(id_priv->cm_id.ib); id_priv->cm_id.ib = NULL; } out: kfree(private_data); return ret; } static int cma_connect_ib(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct ib_cm_req_param req; struct rdma_route *route; void *private_data; struct ib_cm_id *id; int offset, ret; memset(&req, 0, sizeof req); offset = cma_user_data_offset(id_priv); req.private_data_len = offset + conn_param->private_data_len; if (req.private_data_len < conn_param->private_data_len) return -EINVAL; if (req.private_data_len) { private_data = kzalloc(req.private_data_len, GFP_ATOMIC); if (!private_data) return -ENOMEM; } else { private_data = NULL; } if (conn_param->private_data && conn_param->private_data_len) memcpy((char *)private_data + offset, conn_param->private_data, conn_param->private_data_len); id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv); if (IS_ERR(id)) { ret = PTR_ERR(id); goto out; } id_priv->cm_id.ib = id; route = &id_priv->id.route; if (private_data) { ret = cma_format_hdr(private_data, id_priv); if (ret) goto out; req.private_data = private_data; } req.primary_path = &route->path_rec[0]; if (route->num_paths == 2) req.alternate_path = &route->path_rec[1]; req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv)); req.qp_num = id_priv->qp_num; req.qp_type = id_priv->id.qp_type; req.starting_psn = id_priv->seq_num; req.responder_resources = conn_param->responder_resources; req.initiator_depth = conn_param->initiator_depth; req.flow_control = conn_param->flow_control; req.retry_count = min_t(u8, 7, conn_param->retry_count); req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count); req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; req.max_cm_retries = CMA_MAX_CM_RETRIES; req.srq = id_priv->srq ? 1 : 0; ret = ib_send_cm_req(id_priv->cm_id.ib, &req); out: if (ret && !IS_ERR(id)) { ib_destroy_cm_id(id); id_priv->cm_id.ib = NULL; } kfree(private_data); return ret; } static int cma_connect_iw(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct iw_cm_id *cm_id; int ret; struct iw_cm_conn_param iw_param; cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv); if (IS_ERR(cm_id)) return PTR_ERR(cm_id); cm_id->tos = id_priv->tos; id_priv->cm_id.iw = cm_id; memcpy(&cm_id->local_addr, cma_src_addr(id_priv), rdma_addr_size(cma_src_addr(id_priv))); memcpy(&cm_id->remote_addr, cma_dst_addr(id_priv), rdma_addr_size(cma_dst_addr(id_priv))); ret = cma_modify_qp_rtr(id_priv, conn_param); if (ret) goto out; if (conn_param) { iw_param.ord = conn_param->initiator_depth; iw_param.ird = conn_param->responder_resources; iw_param.private_data = conn_param->private_data; iw_param.private_data_len = conn_param->private_data_len; iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num; } else { memset(&iw_param, 0, sizeof iw_param); iw_param.qpn = id_priv->qp_num; } ret = iw_cm_connect(cm_id, &iw_param); out: if (ret) { iw_destroy_cm_id(cm_id); id_priv->cm_id.iw = NULL; } return ret; } int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT)) return -EINVAL; if (!id->qp) { id_priv->qp_num = conn_param->qp_num; id_priv->srq = conn_param->srq; } if (rdma_cap_ib_cm(id->device, id->port_num)) { if (id->qp_type == IB_QPT_UD) ret = cma_resolve_ib_udp(id_priv, conn_param); else ret = cma_connect_ib(id_priv, conn_param); } else if (rdma_cap_iw_cm(id->device, id->port_num)) ret = cma_connect_iw(id_priv, conn_param); else ret = -ENOSYS; if (ret) goto err; return 0; err: cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED); return ret; } EXPORT_SYMBOL(rdma_connect); static int cma_accept_ib(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct ib_cm_rep_param rep; int ret; ret = cma_modify_qp_rtr(id_priv, conn_param); if (ret) goto out; ret = cma_modify_qp_rts(id_priv, conn_param); if (ret) goto out; memset(&rep, 0, sizeof rep); rep.qp_num = id_priv->qp_num; rep.starting_psn = id_priv->seq_num; rep.private_data = conn_param->private_data; rep.private_data_len = conn_param->private_data_len; rep.responder_resources = conn_param->responder_resources; rep.initiator_depth = conn_param->initiator_depth; rep.failover_accepted = 0; rep.flow_control = conn_param->flow_control; rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count); rep.srq = id_priv->srq ? 1 : 0; ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep); out: return ret; } static int cma_accept_iw(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct iw_cm_conn_param iw_param; int ret; ret = cma_modify_qp_rtr(id_priv, conn_param); if (ret) return ret; iw_param.ord = conn_param->initiator_depth; iw_param.ird = conn_param->responder_resources; iw_param.private_data = conn_param->private_data; iw_param.private_data_len = conn_param->private_data_len; if (id_priv->id.qp) { iw_param.qpn = id_priv->qp_num; } else iw_param.qpn = conn_param->qp_num; return iw_cm_accept(id_priv->cm_id.iw, &iw_param); } static int cma_send_sidr_rep(struct rdma_id_private *id_priv, enum ib_cm_sidr_status status, u32 qkey, const void *private_data, int private_data_len) { struct ib_cm_sidr_rep_param rep; int ret; memset(&rep, 0, sizeof rep); rep.status = status; if (status == IB_SIDR_SUCCESS) { ret = cma_set_qkey(id_priv, qkey); if (ret) return ret; rep.qp_num = id_priv->qp_num; rep.qkey = id_priv->qkey; } rep.private_data = private_data; rep.private_data_len = private_data_len; return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep); } int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); id_priv->owner = task_pid_nr(current); if (!cma_comp(id_priv, RDMA_CM_CONNECT)) return -EINVAL; if (!id->qp && conn_param) { id_priv->qp_num = conn_param->qp_num; id_priv->srq = conn_param->srq; } if (rdma_cap_ib_cm(id->device, id->port_num)) { if (id->qp_type == IB_QPT_UD) { if (conn_param) ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS, conn_param->qkey, conn_param->private_data, conn_param->private_data_len); else ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS, 0, NULL, 0); } else { if (conn_param) ret = cma_accept_ib(id_priv, conn_param); else ret = cma_rep_recv(id_priv); } } else if (rdma_cap_iw_cm(id->device, id->port_num)) ret = cma_accept_iw(id_priv, conn_param); else ret = -ENOSYS; if (ret) goto reject; return 0; reject: cma_modify_qp_err(id_priv); rdma_reject(id, NULL, 0); return ret; } EXPORT_SYMBOL(rdma_accept); int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!id_priv->cm_id.ib) return -EINVAL; switch (id->device->node_type) { case RDMA_NODE_IB_CA: ret = ib_cm_notify(id_priv->cm_id.ib, event); break; default: ret = 0; break; } return ret; } EXPORT_SYMBOL(rdma_notify); int rdma_reject(struct rdma_cm_id *id, const void *private_data, u8 private_data_len) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!id_priv->cm_id.ib) return -EINVAL; if (rdma_cap_ib_cm(id->device, id->port_num)) { if (id->qp_type == IB_QPT_UD) ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0, private_data, private_data_len); else ret = ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0, private_data, private_data_len); } else if (rdma_cap_iw_cm(id->device, id->port_num)) { ret = iw_cm_reject(id_priv->cm_id.iw, private_data, private_data_len); } else ret = -ENOSYS; return ret; } EXPORT_SYMBOL(rdma_reject); int rdma_disconnect(struct rdma_cm_id *id) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!id_priv->cm_id.ib) return -EINVAL; if (rdma_cap_ib_cm(id->device, id->port_num)) { ret = cma_modify_qp_err(id_priv); if (ret) goto out; /* Initiate or respond to a disconnect. */ if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0); } else if (rdma_cap_iw_cm(id->device, id->port_num)) { ret = iw_cm_disconnect(id_priv->cm_id.iw, 0); } else ret = -EINVAL; out: return ret; } EXPORT_SYMBOL(rdma_disconnect); static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) { struct rdma_id_private *id_priv; struct cma_multicast *mc = multicast->context; struct rdma_cm_event event; int ret = 0; id_priv = mc->id_priv; mutex_lock(&id_priv->handler_mutex); if (id_priv->state != RDMA_CM_ADDR_BOUND && id_priv->state != RDMA_CM_ADDR_RESOLVED) goto out; if (!status) status = cma_set_qkey(id_priv, be32_to_cpu(multicast->rec.qkey)); mutex_lock(&id_priv->qp_mutex); if (!status && id_priv->id.qp) status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid, be16_to_cpu(multicast->rec.mlid)); mutex_unlock(&id_priv->qp_mutex); memset(&event, 0, sizeof event); event.status = status; event.param.ud.private_data = mc->context; if (!status) { struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; struct ifnet *ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); enum ib_gid_type gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num - rdma_start_port(id_priv->cma_dev->device)]; event.event = RDMA_CM_EVENT_MULTICAST_JOIN; ret = ib_init_ah_from_mcmember(id_priv->id.device, id_priv->id.port_num, &multicast->rec, ndev, gid_type, &event.param.ud.ah_attr); if (ret) event.event = RDMA_CM_EVENT_MULTICAST_ERROR; event.param.ud.qp_num = 0xFFFFFF; event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey); if (ndev) dev_put(ndev); } else event.event = RDMA_CM_EVENT_MULTICAST_ERROR; ret = id_priv->id.event_handler(&id_priv->id, &event); if (ret) { cma_exch(id_priv, RDMA_CM_DESTROYING); mutex_unlock(&id_priv->handler_mutex); rdma_destroy_id(&id_priv->id); return 0; } out: mutex_unlock(&id_priv->handler_mutex); return 0; } static void cma_set_mgid(struct rdma_id_private *id_priv, struct sockaddr *addr, union ib_gid *mgid) { unsigned char mc_map[MAX_ADDR_LEN]; struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; struct sockaddr_in *sin = (struct sockaddr_in *) addr; struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr; - if (cma_any_addr(addr)) { + if (cma_any_addr(dev_addr->net, addr)) { memset(mgid, 0, sizeof *mgid); } else if ((addr->sa_family == AF_INET6) && ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) == 0xFF10A01B)) { /* IPv6 address is an SA assigned MGID. */ memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); } else if (addr->sa_family == AF_IB) { memcpy(mgid, &((struct sockaddr_ib *) addr)->sib_addr, sizeof *mgid); } else if (addr->sa_family == AF_INET6) { ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map); if (id_priv->id.ps == RDMA_PS_UDP) mc_map[7] = 0x01; /* Use RDMA CM signature */ *mgid = *(union ib_gid *) (mc_map + 4); } else { ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map); if (id_priv->id.ps == RDMA_PS_UDP) mc_map[7] = 0x01; /* Use RDMA CM signature */ *mgid = *(union ib_gid *) (mc_map + 4); } } static void cma_query_sa_classport_info_cb(int status, struct ib_class_port_info *rec, void *context) { struct class_port_info_context *cb_ctx = context; WARN_ON(!context); if (status || !rec) { pr_debug("RDMA CM: %s port %u failed query ClassPortInfo status: %d\n", cb_ctx->device->name, cb_ctx->port_num, status); goto out; } memcpy(cb_ctx->class_port_info, rec, sizeof(struct ib_class_port_info)); out: complete(&cb_ctx->done); } static int cma_query_sa_classport_info(struct ib_device *device, u8 port_num, struct ib_class_port_info *class_port_info) { struct class_port_info_context *cb_ctx; int ret; cb_ctx = kmalloc(sizeof(*cb_ctx), GFP_KERNEL); if (!cb_ctx) return -ENOMEM; cb_ctx->device = device; cb_ctx->class_port_info = class_port_info; cb_ctx->port_num = port_num; init_completion(&cb_ctx->done); ret = ib_sa_classport_info_rec_query(&sa_client, device, port_num, CMA_QUERY_CLASSPORT_INFO_TIMEOUT, GFP_KERNEL, cma_query_sa_classport_info_cb, cb_ctx, &cb_ctx->sa_query); if (ret < 0) { pr_err("RDMA CM: %s port %u failed to send ClassPortInfo query, ret: %d\n", device->name, port_num, ret); goto out; } wait_for_completion(&cb_ctx->done); out: kfree(cb_ctx); return ret; } static int cma_join_ib_multicast(struct rdma_id_private *id_priv, struct cma_multicast *mc) { struct ib_sa_mcmember_rec rec; struct ib_class_port_info class_port_info; struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; ib_sa_comp_mask comp_mask; int ret; ib_addr_get_mgid(dev_addr, &rec.mgid); ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, &rec.mgid, &rec); if (ret) return ret; ret = cma_set_qkey(id_priv, 0); if (ret) return ret; cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid); rec.qkey = cpu_to_be32(id_priv->qkey); rdma_addr_get_sgid(dev_addr, &rec.port_gid); rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); rec.join_state = mc->join_state; if (rec.join_state == BIT(SENDONLY_FULLMEMBER_JOIN)) { ret = cma_query_sa_classport_info(id_priv->id.device, id_priv->id.port_num, &class_port_info); if (ret) return ret; if (!(ib_get_cpi_capmask2(&class_port_info) & IB_SA_CAP_MASK2_SENDONLY_FULL_MEM_SUPPORT)) { pr_warn("RDMA CM: %s port %u Unable to multicast join\n" "RDMA CM: SM doesn't support Send Only Full Member option\n", id_priv->id.device->name, id_priv->id.port_num); return -EOPNOTSUPP; } } comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID | IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE | IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL | IB_SA_MCMEMBER_REC_FLOW_LABEL | IB_SA_MCMEMBER_REC_TRAFFIC_CLASS; if (id_priv->id.ps == RDMA_PS_IPOIB) comp_mask |= IB_SA_MCMEMBER_REC_RATE | IB_SA_MCMEMBER_REC_RATE_SELECTOR | IB_SA_MCMEMBER_REC_MTU_SELECTOR | IB_SA_MCMEMBER_REC_MTU | IB_SA_MCMEMBER_REC_HOP_LIMIT; mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device, id_priv->id.port_num, &rec, comp_mask, GFP_KERNEL, cma_ib_mc_handler, mc); return PTR_ERR_OR_ZERO(mc->multicast.ib); } static void iboe_mcast_work_handler(struct work_struct *work) { struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work); struct cma_multicast *mc = mw->mc; struct ib_sa_multicast *m = mc->multicast.ib; mc->multicast.ib->context = mc; cma_ib_mc_handler(0, m); kref_put(&mc->mcref, release_mc); kfree(mw); } -static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid, - enum ib_gid_type gid_type) +static void cma_iboe_set_mgid(struct vnet *vnet, struct sockaddr *addr, + union ib_gid *mgid, enum ib_gid_type gid_type) { struct sockaddr_in *sin = (struct sockaddr_in *)addr; struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; - if (cma_any_addr(addr)) { + if (cma_any_addr(vnet, addr)) { memset(mgid, 0, sizeof *mgid); } else if (addr->sa_family == AF_INET6) { memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); } else { mgid->raw[0] = (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0xff; mgid->raw[1] = (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0x0e; mgid->raw[2] = 0; mgid->raw[3] = 0; mgid->raw[4] = 0; mgid->raw[5] = 0; mgid->raw[6] = 0; mgid->raw[7] = 0; mgid->raw[8] = 0; mgid->raw[9] = 0; mgid->raw[10] = 0xff; mgid->raw[11] = 0xff; *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr; } } static int cma_iboe_join_multicast(struct rdma_id_private *id_priv, struct cma_multicast *mc) { struct iboe_mcast_work *work; struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; int err = 0; struct sockaddr *addr = (struct sockaddr *)&mc->addr; struct ifnet *ndev = NULL; enum ib_gid_type gid_type; bool send_only; send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN); if (cma_zero_addr((struct sockaddr *)&mc->addr)) return -EINVAL; work = kzalloc(sizeof *work, GFP_KERNEL); if (!work) return -ENOMEM; mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL); if (!mc->multicast.ib) { err = -ENOMEM; goto out1; } gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num - rdma_start_port(id_priv->cma_dev->device)]; - cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid, gid_type); + cma_iboe_set_mgid(dev_addr->net, addr, &mc->multicast.ib->rec.mgid, gid_type); mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff); if (id_priv->id.ps == RDMA_PS_UDP) mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); if (dev_addr->bound_dev_if) ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); if (!ndev) { err = -ENODEV; goto out2; } mc->multicast.ib->rec.rate = iboe_get_rate(ndev); mc->multicast.ib->rec.hop_limit = 1; mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->if_mtu); if (addr->sa_family == AF_INET || addr->sa_family == AF_INET6) { if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { mc->multicast.ib->rec.hop_limit = IPV6_DEFAULT_HOPLIMIT; if (!send_only) { err = cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid, true); if (!err) mc->igmp_joined = true; } } } else { if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) err = -ENOTSUPP; } dev_put(ndev); if (err || !mc->multicast.ib->rec.mtu) { if (!err) err = -EINVAL; goto out2; } rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, &mc->multicast.ib->rec.port_gid); work->id = id_priv; work->mc = mc; INIT_WORK(&work->work, iboe_mcast_work_handler); kref_get(&mc->mcref); queue_work(cma_wq, &work->work); return 0; out2: kfree(mc->multicast.ib); out1: kfree(work); return err; } int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, u8 join_state, void *context) { struct rdma_id_private *id_priv; struct cma_multicast *mc; int ret; if (!id->device) return -EINVAL; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) && !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED)) return -EINVAL; mc = kmalloc(sizeof *mc, GFP_KERNEL); if (!mc) return -ENOMEM; memcpy(&mc->addr, addr, rdma_addr_size(addr)); mc->context = context; mc->id_priv = id_priv; mc->igmp_joined = false; mc->join_state = join_state; spin_lock(&id_priv->lock); list_add(&mc->list, &id_priv->mc_list); spin_unlock(&id_priv->lock); if (rdma_protocol_roce(id->device, id->port_num)) { kref_init(&mc->mcref); ret = cma_iboe_join_multicast(id_priv, mc); } else if (rdma_cap_ib_mcast(id->device, id->port_num)) ret = cma_join_ib_multicast(id_priv, mc); else ret = -ENOSYS; if (ret) { spin_lock_irq(&id_priv->lock); list_del(&mc->list); spin_unlock_irq(&id_priv->lock); kfree(mc); } return ret; } EXPORT_SYMBOL(rdma_join_multicast); void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) { struct rdma_id_private *id_priv; struct cma_multicast *mc; id_priv = container_of(id, struct rdma_id_private, id); spin_lock_irq(&id_priv->lock); list_for_each_entry(mc, &id_priv->mc_list, list) { if (!memcmp(&mc->addr, addr, rdma_addr_size(addr))) { list_del(&mc->list); spin_unlock_irq(&id_priv->lock); if (id->qp) ib_detach_mcast(id->qp, &mc->multicast.ib->rec.mgid, be16_to_cpu(mc->multicast.ib->rec.mlid)); BUG_ON(id_priv->cma_dev->device != id->device); if (rdma_cap_ib_mcast(id->device, id->port_num)) { ib_sa_free_multicast(mc->multicast.ib); kfree(mc); } else if (rdma_protocol_roce(id->device, id->port_num)) { if (mc->igmp_joined) { struct rdma_dev_addr *dev_addr = &id->route.addr.dev_addr; struct ifnet *ndev = NULL; if (dev_addr->bound_dev_if) ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); if (ndev) { cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid, false); dev_put(ndev); } mc->igmp_joined = false; } kref_put(&mc->mcref, release_mc); } return; } } spin_unlock_irq(&id_priv->lock); } EXPORT_SYMBOL(rdma_leave_multicast); static int sysctl_cma_default_roce_mode(SYSCTL_HANDLER_ARGS) { struct cma_device *cma_dev = arg1; const int port = arg2; char buf[64]; int error; strlcpy(buf, ib_cache_gid_type_str( cma_get_default_gid_type(cma_dev, port)), sizeof(buf)); error = sysctl_handle_string(oidp, buf, sizeof(buf), req); if (error != 0 || req->newptr == NULL) goto done; error = ib_cache_gid_parse_type_str(buf); if (error < 0) { error = EINVAL; goto done; } cma_set_default_gid_type(cma_dev, port, error); error = 0; done: return (error); } static void cma_add_one(struct ib_device *device) { struct cma_device *cma_dev; struct rdma_id_private *id_priv; unsigned int i; cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL); if (!cma_dev) return; sysctl_ctx_init(&cma_dev->sysctl_ctx); cma_dev->device = device; cma_dev->default_gid_type = kcalloc(device->phys_port_cnt, sizeof(*cma_dev->default_gid_type), GFP_KERNEL); if (!cma_dev->default_gid_type) { kfree(cma_dev); return; } for (i = rdma_start_port(device); i <= rdma_end_port(device); i++) { unsigned long supported_gids; unsigned int default_gid_type; supported_gids = roce_gid_type_mask_support(device, i); if (WARN_ON(!supported_gids)) { /* set something valid */ default_gid_type = 0; } else if (test_bit(IB_GID_TYPE_ROCE_UDP_ENCAP, &supported_gids)) { /* prefer RoCEv2, if supported */ default_gid_type = IB_GID_TYPE_ROCE_UDP_ENCAP; } else { default_gid_type = find_first_bit(&supported_gids, BITS_PER_LONG); } cma_dev->default_gid_type[i - rdma_start_port(device)] = default_gid_type; } init_completion(&cma_dev->comp); atomic_set(&cma_dev->refcount, 1); INIT_LIST_HEAD(&cma_dev->id_list); ib_set_client_data(device, &cma_client, cma_dev); mutex_lock(&lock); list_add_tail(&cma_dev->list, &dev_list); list_for_each_entry(id_priv, &listen_any_list, list) cma_listen_on_dev(id_priv, cma_dev); mutex_unlock(&lock); for (i = rdma_start_port(device); i <= rdma_end_port(device); i++) { char buf[64]; snprintf(buf, sizeof(buf), "default_roce_mode_port%d", i); (void) SYSCTL_ADD_PROC(&cma_dev->sysctl_ctx, SYSCTL_CHILDREN(device->ports_parent->parent->oidp), OID_AUTO, buf, CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, cma_dev, i, &sysctl_cma_default_roce_mode, "A", "Default RoCE mode. Valid values: IB/RoCE v1 and RoCE v2"); } } static int cma_remove_id_dev(struct rdma_id_private *id_priv) { struct rdma_cm_event event; enum rdma_cm_state state; int ret = 0; /* Record that we want to remove the device */ state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL); if (state == RDMA_CM_DESTROYING) return 0; cma_cancel_operation(id_priv, state); mutex_lock(&id_priv->handler_mutex); /* Check for destruction from another callback. */ if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL)) goto out; memset(&event, 0, sizeof event); event.event = RDMA_CM_EVENT_DEVICE_REMOVAL; ret = id_priv->id.event_handler(&id_priv->id, &event); out: mutex_unlock(&id_priv->handler_mutex); return ret; } static void cma_process_remove(struct cma_device *cma_dev) { struct rdma_id_private *id_priv; int ret; mutex_lock(&lock); while (!list_empty(&cma_dev->id_list)) { id_priv = list_entry(cma_dev->id_list.next, struct rdma_id_private, list); list_del(&id_priv->listen_list); list_del_init(&id_priv->list); atomic_inc(&id_priv->refcount); mutex_unlock(&lock); ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv); cma_deref_id(id_priv); if (ret) rdma_destroy_id(&id_priv->id); mutex_lock(&lock); } mutex_unlock(&lock); cma_deref_dev(cma_dev); wait_for_completion(&cma_dev->comp); } static void cma_remove_one(struct ib_device *device, void *client_data) { struct cma_device *cma_dev = client_data; if (!cma_dev) return; mutex_lock(&lock); list_del(&cma_dev->list); mutex_unlock(&lock); cma_process_remove(cma_dev); sysctl_ctx_free(&cma_dev->sysctl_ctx); kfree(cma_dev->default_gid_type); kfree(cma_dev); } static void cma_init_vnet(void *arg) { struct cma_pernet *pernet = &VNET(cma_pernet); idr_init(&pernet->tcp_ps); idr_init(&pernet->udp_ps); idr_init(&pernet->ipoib_ps); idr_init(&pernet->ib_ps); idr_init(&pernet->sdp_ps); } VNET_SYSINIT(cma_init_vnet, SI_SUB_OFED_MODINIT - 1, SI_ORDER_FIRST, cma_init_vnet, NULL); static void cma_destroy_vnet(void *arg) { struct cma_pernet *pernet = &VNET(cma_pernet); idr_destroy(&pernet->tcp_ps); idr_destroy(&pernet->udp_ps); idr_destroy(&pernet->ipoib_ps); idr_destroy(&pernet->ib_ps); idr_destroy(&pernet->sdp_ps); } VNET_SYSUNINIT(cma_destroy_vnet, SI_SUB_OFED_MODINIT - 1, SI_ORDER_SECOND, cma_destroy_vnet, NULL); static int __init cma_init(void) { int ret; cma_wq = alloc_ordered_workqueue("rdma_cm", WQ_MEM_RECLAIM); if (!cma_wq) return -ENOMEM; ib_sa_register_client(&sa_client); rdma_addr_register_client(&addr_client); ret = ib_register_client(&cma_client); if (ret) goto err; cma_configfs_init(); return 0; err: rdma_addr_unregister_client(&addr_client); ib_sa_unregister_client(&sa_client); destroy_workqueue(cma_wq); return ret; } static void __exit cma_cleanup(void) { cma_configfs_exit(); ib_unregister_client(&cma_client); rdma_addr_unregister_client(&addr_client); ib_sa_unregister_client(&sa_client); destroy_workqueue(cma_wq); } module_init_order(cma_init, SI_ORDER_FOURTH); module_exit_order(cma_cleanup, SI_ORDER_FOURTH);