diff --git a/contrib/ofed/libirdma/osdep.h b/contrib/ofed/libirdma/osdep.h index 2cb4f8d57cb5..23bdbe805b01 100644 --- a/contrib/ofed/libirdma/osdep.h +++ b/contrib/ofed/libirdma/osdep.h @@ -1,229 +1,227 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * * Copyright (c) 2021 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenFabrics.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /*$FreeBSD$*/ #ifndef _ICRDMA_OSDEP_H_ #define _ICRDMA_OSDEP_H_ #include #include #include #include #include #include #include #include #include #define ATOMIC atomic_t #define IOMEM #define IRDMA_NTOHL(a) ntohl(a) #define IRDMA_NTOHS(a) ntohs(a) #define MAKEMASK(m, s) ((m) << (s)) #define OS_TIMER timer_list #define OS_LIST_HEAD list_head #define OS_LIST_ENTRY list_head #define DECLARE_HASHTABLE(n, b) struct hlist_head (n)[1 << (b)] #define HASH_MIN(v, b) (sizeof(v) <= 4 ? hash_32(v, b) : hash_long(v, b)) #define HASH_FOR_EACH_RCU(n, b, o, m) for ((b) = 0, o = NULL; o == NULL && (b) < ARRAY_SIZE(n);\ (b)++)\ hlist_for_each_entry_rcu(o, &n[(b)], m) #define HASH_FOR_EACH_POSSIBLE_RCU(n, o, m, k) \ hlist_for_each_entry_rcu(o, &n[jhash(&k, sizeof(k), 0) >> (32 - ilog2(ARRAY_SIZE(n)))],\ m) #define HASH_FOR_EACH_POSSIBLE(n, o, m, k) \ hlist_for_each_entry(o, &n[jhash(&k, sizeof(k), 0) >> (32 - ilog2(ARRAY_SIZE(n)))],\ m) #define HASH_ADD_RCU(h, n, k) \ hlist_add_head_rcu(n, &h[jhash(&k, sizeof(k), 0) >> (32 - ilog2(ARRAY_SIZE(h)))]) #define HASH_DEL_RCU(tbl, node) hlist_del_rcu(node) #define HASH_ADD(h, n, k) \ hlist_add_head(n, &h[jhash(&k, sizeof(k), 0) >> (32 - ilog2(ARRAY_SIZE(h)))]) #define HASH_DEL(tbl, node) hlist_del(node) #define WQ_UNBOUND_MAX_ACTIVE max_t(int, 512, num_possible_cpus() * 4) #define if_addr_rlock(x) #define if_addr_runlock(x) /* constants */ #define STATS_TIMER_DELAY 60000 /* a couple of linux size defines */ #define SZ_128 128 #define SZ_2K SZ_128 * 16 #define SZ_1G (SZ_1K * SZ_1K * SZ_1K) #define SPEED_1000 1000 #define SPEED_10000 10000 #define SPEED_20000 20000 #define SPEED_25000 25000 #define SPEED_40000 40000 #define SPEED_100000 100000 #define BIT_ULL(a) (1ULL << (a)) #define min(a, b) ((a) > (b) ? (b) : (a)) #ifndef likely #define likely(x) __builtin_expect((x), 1) #endif #ifndef unlikely #define unlikely(x) __builtin_expect((x), 0) #endif #define __aligned_u64 uint64_t __aligned(8) #define VLAN_PRIO_SHIFT 13 -#if __FreeBSD_version < 1400000 #define IB_USER_VERBS_EX_CMD_MODIFY_QP IB_USER_VERBS_CMD_MODIFY_QP -#endif /* * debug definition section */ #define irdma_print(S, ...) printf("%s:%d "S, __FUNCTION__, __LINE__, ##__VA_ARGS__) #define irdma_debug_buf(dev, mask, desc, buf, size) \ do { \ u32 i; \ if (!((mask) & (dev)->debug_mask)) { \ break; \ } \ irdma_debug(dev, mask, "%s\n", desc); \ irdma_debug(dev, mask, "starting address virt=%p phy=%lxh\n", buf, irdma_get_virt_to_phy(buf)); \ for (i = 0; i < size ; i += 8) \ irdma_debug(dev, mask, "index %03d val: %016lx\n", i, ((unsigned long *)buf)[i / 8]); \ } while(0) #define irdma_debug(h, m, s, ...) \ do { \ if (!(h)) { \ if ((m) == IRDMA_DEBUG_INIT) \ printf("irdma INIT " s, ##__VA_ARGS__); \ } else if (((m) & (h)->debug_mask)) { \ printf("irdma " s, ##__VA_ARGS__); \ } \ } while (0) extern unsigned int irdma_dbg; #define libirdma_debug(fmt, args...) \ do { \ if (irdma_dbg) \ printf("libirdma-%s: " fmt, __func__, ##args); \ } while (0) #define irdma_dev_err(a, b, ...) printf(b, ##__VA_ARGS__) #define irdma_dev_warn(a, b, ...) printf(b, ##__VA_ARGS__) /*dev_warn(a, b)*/ #define irdma_dev_info(a, b, ...) printf(b, ##__VA_ARGS__) #define irdma_pr_warn printf #define ibdev_err(ibdev, fmt, ...) printf("%s:"fmt, (ibdev)->name, ##__VA_ARGS__) #define dump_struct(s, sz, name) \ do { \ unsigned char *a; \ printf("%s %u", (name), (unsigned int)(sz)); \ for (a = (unsigned char*)(s); a < (unsigned char *)(s) + (sz) ; a ++) { \ if ((u64)a % 8 == 0) \ printf("\n%p ", a); \ printf("%2x ", *a); \ } \ printf("\n"); \ }while(0) /* * debug definition end */ typedef __be16 BE16; typedef __be32 BE32; typedef uintptr_t irdma_uintptr; struct irdma_hw; struct irdma_pci_f; struct irdma_sc_dev; struct irdma_sc_qp; struct irdma_sc_vsi; #define irdma_pr_info(fmt, args ...) printf("%s: WARN "fmt, __func__, ## args) #define irdma_pr_err(fmt, args ...) printf("%s: ERR "fmt, __func__, ## args) #define irdma_memcpy(a, b, c) memcpy((a), (b), (c)) #define irdma_memset(a, b, c) memset((a), (b), (c)) #define irdma_usec_delay(x) DELAY(x) #define mdelay(x) DELAY((x) * 1000) #define rt_tos2priority(tos) (tos >> 5) #define ah_attr_to_dmac(attr) ((attr).dmac) #define kc_ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll) \ ib_modify_qp_is_ok(cur_state, next_state, type, mask) #define kc_typeq_ib_wr const #define kc_ifp_find ip_ifp_find #define kc_ifp6_find ip6_ifp_find #define kc_rdma_gid_attr_network_type(sgid_attr, gid_type, gid) \ ib_gid_to_network_type(gid_type, gid) #define irdma_del_timer_compat(tt) del_timer((tt)) #define IRDMA_TAILQ_FOREACH CK_STAILQ_FOREACH #define IRDMA_TAILQ_FOREACH_SAFE CK_STAILQ_FOREACH_SAFE #define between(a, b, c) (bool)(c-a >= b-a) static inline void db_wr32(__u32 val, __u32 *wqe_word) { *wqe_word = val; } void *hw_to_dev(struct irdma_hw *hw); struct irdma_dma_mem { void *va; u64 pa; bus_dma_tag_t tag; bus_dmamap_t map; bus_dma_segment_t seg; bus_size_t size; int nseg; int flags; }; struct irdma_virt_mem { void *va; u32 size; }; #ifndef verbs_mr enum ibv_mr_type { IBV_MR_TYPE_MR, IBV_MR_TYPE_NULL_MR, }; struct verbs_mr { struct ibv_mr ibv_mr; enum ibv_mr_type mr_type; int access; }; #define verbs_get_mr(mr) container_of((mr), struct verbs_mr, ibv_mr) #endif #endif /* _ICRDMA_OSDEP_H_ */ diff --git a/sys/dev/irdma/fbsd_kcompat.h b/sys/dev/irdma/fbsd_kcompat.h index 1e0544738d7e..7184818399c7 100644 --- a/sys/dev/irdma/fbsd_kcompat.h +++ b/sys/dev/irdma/fbsd_kcompat.h @@ -1,341 +1,268 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * * Copyright (c) 2021 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenFabrics.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /*$FreeBSD$*/ #ifndef FBSD_KCOMPAT_H #define FBSD_KCOMPAT_H #include "ice_rdma.h" #define TASKLET_DATA_TYPE unsigned long #define TASKLET_FUNC_TYPE void (*)(TASKLET_DATA_TYPE) #ifndef tasklet_setup #define tasklet_setup(tasklet, callback) \ tasklet_init((tasklet), (TASKLET_FUNC_TYPE)(callback), \ (TASKLET_DATA_TYPE)(tasklet)) #endif #ifndef from_tasklet #define from_tasklet(var, callback_tasklet, tasklet_fieldname) \ container_of(callback_tasklet, typeof(*var), tasklet_fieldname) #endif -#if __FreeBSD_version >= 1400000 -#define IRDMA_SET_RDMA_OBJ_SIZE(ib_struct, drv_struct, member) \ - (sizeof(struct drv_struct) + \ - BUILD_BUG_ON_ZERO(offsetof(struct drv_struct, member)) + \ - BUILD_BUG_ON_ZERO( \ - !__same_type(((struct drv_struct *)NULL)->member, \ - struct ib_struct))) -#endif /* __FreeBSD_version > 1400000 */ - #define set_ibdev_dma_device(ibdev, dev) \ ibdev.dma_device = (dev) #define set_max_sge(props, rf) \ ((props)->max_sge = (rf)->sc_dev.hw_attrs.uk_attrs.max_hw_wq_frags) #define kc_set_props_ip_gid_caps(props) \ ((props)->port_cap_flags |= IB_PORT_IP_BASED_GIDS) #define rdma_query_gid(ibdev, port, index, gid) \ ib_get_cached_gid(ibdev, port, index, gid, NULL) #define kmap(pg) page_address(pg) #define kmap_local_page(pg) page_address(pg) #define kunmap(pg) #define kunmap_local(pg) -#if __FreeBSD_version >= 1400026 -#define kc_free_lsmm_dereg_mr(iwdev, iwqp) \ - ((iwdev)->ibdev.dereg_mr((iwqp)->lsmm_mr, NULL)) -#else #define kc_free_lsmm_dereg_mr(iwdev, iwqp) \ ((iwdev)->ibdev.dereg_mr((iwqp)->lsmm_mr)) -#endif #define IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION IB_CQ_FLAGS_TIMESTAMP_COMPLETION -#if __FreeBSD_version < 1400026 #define kc_irdma_destroy_qp(ibqp, udata) irdma_destroy_qp(ibqp) -#else -#define kc_irdma_destroy_qp(ibqp, udata) irdma_destroy_qp(ibqp, udata) -#endif #ifndef IB_QP_ATTR_STANDARD_BITS #define IB_QP_ATTR_STANDARD_BITS GENMASK(20, 0) #endif #define IRDMA_QOS_MODE_VLAN 0x0 #define IRDMA_QOS_MODE_DSCP 0x1 #define IRDMA_VER_LEN 24 void kc_set_roce_uverbs_cmd_mask(struct irdma_device *iwdev); void kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev); struct irdma_tunable_info { struct sysctl_ctx_list irdma_sysctl_ctx; struct sysctl_oid *irdma_sysctl_tree; char drv_ver[IRDMA_VER_LEN]; u8 roce_ena; }; static inline int irdma_iw_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) { *pkey = 0; return 0; } static inline int cq_validate_flags(u32 flags, u8 hw_rev) { /* GEN1 does not support CQ create flags */ if (hw_rev == IRDMA_GEN_1) return flags ? -EOPNOTSUPP : 0; return flags & ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ? -EOPNOTSUPP : 0; } static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo, u32 *idx) { *idx += 1; if (!(*pinfo) || *idx != (*pinfo)->cnt) return ++pbl; *idx = 0; (*pinfo)++; return (*pinfo)->addr; } -#if __FreeBSD_version < 1400026 struct ib_cq *irdma_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, struct ib_ucontext *context, struct ib_udata *udata); -#else -int irdma_create_cq(struct ib_cq *ibcq, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); -#endif struct ib_qp *irdma_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata); -#if __FreeBSD_version >= 1400026 -int irdma_create_ah(struct ib_ah *ib_ah, - struct ib_ah_attr *attr, u32 flags, - struct ib_udata *udata); -int irdma_create_ah_stub(struct ib_ah *ib_ah, - struct ib_ah_attr *attr, u32 flags, - struct ib_udata *udata); -#else struct ib_ah *irdma_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr, struct ib_udata *udata); struct ib_ah *irdma_create_ah_stub(struct ib_pd *ibpd, struct ib_ah_attr *attr, struct ib_udata *udata); -#endif void irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr); -#if __FreeBSD_version >= 1400026 -void irdma_destroy_ah(struct ib_ah *ibah, u32 flags); -void irdma_destroy_ah_stub(struct ib_ah *ibah, u32 flags); -#else int irdma_destroy_ah(struct ib_ah *ibah); int irdma_destroy_ah_stub(struct ib_ah *ibah); -#endif -#if __FreeBSD_version < 1400026 int irdma_destroy_qp(struct ib_qp *ibqp); -#else -int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); -#endif -#if __FreeBSD_version < 1400026 int irdma_dereg_mr(struct ib_mr *ib_mr); -#else -int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata); -#endif void irdma_get_eth_speed_and_width(u32 link_speed, u8 *active_speed, u8 *active_width); enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, u8 port_num); int irdma_roce_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable); int irdma_iw_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable); int irdma_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid); int irdma_query_gid_roce(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid); int irdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey); int irdma_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props); struct rdma_hw_stats *irdma_alloc_hw_stats(struct ib_device *ibdev, u8 port_num); int irdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, u8 port_num, int index); int irdma_register_qset(struct irdma_sc_vsi *vsi, struct irdma_ws_node *tc_node); void irdma_unregister_qset(struct irdma_sc_vsi *vsi, struct irdma_ws_node *tc_node); void ib_unregister_device(struct ib_device *ibdev); -#if __FreeBSD_version < 1400026 int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma, unsigned long pfn, unsigned long size, pgprot_t prot); -#endif void irdma_disassociate_ucontext(struct ib_ucontext *context); int kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp, struct ib_qp_attr *attr, u16 *vlan_id); struct irdma_device *kc_irdma_get_device(struct ifnet *netdev); void kc_irdma_put_device(struct irdma_device *iwdev); void kc_set_loc_seq_num_mss(struct irdma_cm_node *cm_node); u16 kc_rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn); void irdma_get_dev_fw_str(struct ib_device *dev, char *str, size_t str_len); int irdma_modify_port(struct ib_device *ibdev, u8 port, int mask, struct ib_port_modify *props); int irdma_get_dst_mac(struct irdma_cm_node *cm_node, struct sockaddr *dst_sin, u8 *dst_mac); int irdma_resolve_neigh_lpb_chk(struct irdma_device *iwdev, struct irdma_cm_node *cm_node, struct irdma_cm_info *cm_info); int irdma_addr_resolve_neigh(struct irdma_cm_node *cm_node, u32 dst_ip, int arpindex); int irdma_addr_resolve_neigh_ipv6(struct irdma_cm_node *cm_node, u32 *dest, int arpindex); void irdma_dcqcn_tunables_init(struct irdma_pci_f *rf); u32 irdma_create_stag(struct irdma_device *iwdev); void irdma_free_stag(struct irdma_device *iwdev, u32 stag); int irdma_hwdereg_mr(struct ib_mr *ib_mr); int irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 len, u64 virt, int new_access, struct ib_pd *new_pd, struct ib_udata *udata); struct irdma_mr; struct irdma_cq; struct irdma_cq_buf; -#if __FreeBSD_version < 1400026 struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg); -#else -struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, - u32 max_num_sg, struct ib_udata *udata); -#endif int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, u16 access); struct ib_mr *irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, u64 virt, struct ib_udata *udata); int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr); struct ib_mw *irdma_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, struct ib_udata *udata); int irdma_hw_alloc_stag(struct irdma_device *iwdev, struct irdma_mr *iwmr); void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq); int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr, struct irdma_device *iwdev); void irdma_setup_virt_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, struct irdma_qp_init_info *init_info); int irdma_setup_kmode_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, struct irdma_qp_init_info *info, struct ib_qp_init_attr *init_attr); int irdma_setup_umode_qp(struct ib_udata *udata, struct irdma_device *iwdev, struct irdma_qp *iwqp, struct irdma_qp_init_info *info, struct ib_qp_init_attr *init_attr); void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp, struct irdma_qp_host_ctx_info *ctx_info); void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp, struct irdma_qp_host_ctx_info *ctx_info); int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp); void irdma_dealloc_push_page(struct irdma_pci_f *rf, struct irdma_sc_qp *qp); int irdma_process_resize_list(struct irdma_cq *iwcq, struct irdma_device *iwdev, struct irdma_cq_buf *lcqe_buf); -#if __FreeBSD_version < 1400026 int irdma_destroy_cq(struct ib_cq *ib_cq); -#else -void irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); -#endif -#if __FreeBSD_version < 1400026 struct ib_ucontext *irdma_alloc_ucontext(struct ib_device *, struct ib_udata *); -#else -int irdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata); -#endif -#if __FreeBSD_version < 1400026 int irdma_dealloc_ucontext(struct ib_ucontext *); -#else -void irdma_dealloc_ucontext(struct ib_ucontext *context); -#endif -#if __FreeBSD_version < 1400026 struct ib_pd *irdma_alloc_pd(struct ib_device *, struct ib_ucontext *, struct ib_udata *); -#else -int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata); -#endif -#if __FreeBSD_version < 1400026 int irdma_dealloc_pd(struct ib_pd *); -#else -void irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); -#endif int irdma_add_gid(struct ib_device *, u8, unsigned int, const union ib_gid *, const struct ib_gid_attr *, void **); int irdma_del_gid(struct ib_device *, u8, unsigned int, void **); struct ib_device *ib_device_get_by_netdev(struct ifnet *ndev, int driver_id); void ib_device_put(struct ib_device *device); void ib_unregister_device_put(struct ib_device *device); enum ib_mtu ib_mtu_int_to_enum(int mtu); struct irdma_pbl *irdma_get_pbl(unsigned long va, struct list_head *pbl_list); void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq); void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp); struct irdma_ucontext; void irdma_del_memlist(struct irdma_mr *iwmr, struct irdma_ucontext *ucontext); void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, enum irdma_pble_level level); void irdma_reg_ipaddr_event_cb(struct irdma_pci_f *rf); void irdma_dereg_ipaddr_event_cb(struct irdma_pci_f *rf); /* Introduced in this series https://lore.kernel.org/linux-rdma/0-v2-270386b7e60b+28f4-umem_1_jgg@nvidia.com/ * An irdma version helper doing same for older functions with difference that iova is passed in * as opposed to derived from umem->iova. */ static inline size_t irdma_ib_umem_num_dma_blocks(struct ib_umem *umem, unsigned long pgsz, u64 iova) { /* some older OFED distros do not have ALIGN_DOWN */ #ifndef ALIGN_DOWN #define ALIGN_DOWN(x, a) ALIGN((x) - ((a) - 1), (a)) #endif return (size_t)((ALIGN(iova + umem->length, pgsz) - ALIGN_DOWN(iova, pgsz))) / pgsz; } #endif /* FBSD_KCOMPAT_H */ diff --git a/sys/dev/irdma/irdma_kcompat.c b/sys/dev/irdma/irdma_kcompat.c index d49293aedd78..3b219eff1901 100644 --- a/sys/dev/irdma/irdma_kcompat.c +++ b/sys/dev/irdma/irdma_kcompat.c @@ -1,2306 +1,1838 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * * Copyright (c) 2018 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenFabrics.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /*$FreeBSD$*/ #include "irdma_main.h" #define IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000) static u16 kc_rdma_flow_label_to_udp_sport(u32 fl) { u32 fl_low = fl & 0x03FFF; u32 fl_high = fl & 0xFC000; fl_low ^= fl_high >> 14; return (u16)(fl_low | IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN); } #define IRDMA_GRH_FLOWLABEL_MASK (0x000FFFFF) static u32 kc_rdma_calc_flow_label(u32 lqpn, u32 rqpn) { u64 fl = (u64)lqpn * rqpn; fl ^= fl >> 20; fl ^= fl >> 40; return (u32)(fl & IRDMA_GRH_FLOWLABEL_MASK); } u16 kc_rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn) { if (!fl) fl = kc_rdma_calc_flow_label(lqpn, rqpn); return kc_rdma_flow_label_to_udp_sport(fl); } void irdma_get_dev_fw_str(struct ib_device *dev, char *str, size_t str_len) { struct irdma_device *iwdev = to_iwdev(dev); snprintf(str, str_len, "%u.%u", irdma_fw_major_ver(&iwdev->rf->sc_dev), irdma_fw_minor_ver(&iwdev->rf->sc_dev)); } int irdma_add_gid(struct ib_device *device, u8 port_num, unsigned int index, const union ib_gid *gid, const struct ib_gid_attr *attr, void **context) { return 0; } int irdma_del_gid(struct ib_device *device, u8 port_num, unsigned int index, void **context) { return 0; } -#if __FreeBSD_version >= 1400026 -/** - * irdma_alloc_mr - register stag for fast memory registration - * @pd: ibpd pointer - * @mr_type: memory for stag registrion - * @max_num_sg: man number of pages - * @udata: user data - */ -struct ib_mr * -irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, - u32 max_num_sg, struct ib_udata *udata) -{ -#else /** * irdma_alloc_mr - register stag for fast memory registration * @pd: ibpd pointer * @mr_type: memory for stag registrion * @max_num_sg: man number of pages */ struct ib_mr * irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg) { -#endif struct irdma_device *iwdev = to_iwdev(pd->device); struct irdma_pble_alloc *palloc; struct irdma_pbl *iwpbl; struct irdma_mr *iwmr; int status; u32 stag; int err_code = -ENOMEM; iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); if (!iwmr) return ERR_PTR(-ENOMEM); stag = irdma_create_stag(iwdev); if (!stag) { err_code = -ENOMEM; goto err; } iwmr->stag = stag; iwmr->ibmr.rkey = stag; iwmr->ibmr.lkey = stag; iwmr->ibmr.pd = pd; iwmr->ibmr.device = pd->device; iwpbl = &iwmr->iwpbl; iwpbl->iwmr = iwmr; iwmr->type = IRDMA_MEMREG_TYPE_MEM; palloc = &iwpbl->pble_alloc; iwmr->page_cnt = max_num_sg; /* Assume system PAGE_SIZE as the sg page sizes are unknown. */ iwmr->len = max_num_sg * PAGE_SIZE; status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt, false); if (status) goto err_get_pble; err_code = irdma_hw_alloc_stag(iwdev, iwmr); if (err_code) goto err_alloc_stag; iwpbl->pbl_allocated = true; return &iwmr->ibmr; err_alloc_stag: irdma_free_pble(iwdev->rf->pble_rsrc, palloc); err_get_pble: irdma_free_stag(iwdev, stag); err: kfree(iwmr); return ERR_PTR(err_code); } #define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8) #define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd) -#if __FreeBSD_version >= 1400026 -/** - * irdma_alloc_ucontext - Allocate the user context data structure - * @uctx: context - * @udata: user data - * - * This keeps track of all objects associated with a particular - * user-mode client. - */ -int -irdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) -{ - struct ib_device *ibdev = uctx->device; - struct irdma_device *iwdev = to_iwdev(ibdev); - struct irdma_alloc_ucontext_req req = {0}; - struct irdma_alloc_ucontext_resp uresp = {0}; - struct irdma_ucontext *ucontext = to_ucontext(uctx); - struct irdma_uk_attrs *uk_attrs; - - if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN || - udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN) - return -EINVAL; - - if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) - return -EINVAL; - - if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) - goto ver_error; - - ucontext->iwdev = iwdev; - ucontext->abi_ver = req.userspace_ver; - - uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; - /* GEN_1 support for libi40iw */ - if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) { - if (uk_attrs->hw_rev != IRDMA_GEN_1) - return -EOPNOTSUPP; - - ucontext->legacy_mode = true; - uresp.max_qps = iwdev->rf->max_qp; - uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; - uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; - uresp.kernel_ver = req.userspace_ver; - if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) - return -EFAULT; - } else { - u64 bar_off; - - uresp.kernel_ver = IRDMA_ABI_VER; - uresp.feature_flags = uk_attrs->feature_flags; - uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; - uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; - uresp.max_hw_inline = uk_attrs->max_hw_inline; - uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; - uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; - uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; - uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; - uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; - uresp.hw_rev = uk_attrs->hw_rev; - - bar_off = - (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; - ucontext->db_mmap_entry = - irdma_user_mmap_entry_insert(ucontext, bar_off, - IRDMA_MMAP_IO_NC, - &uresp.db_mmap_key); - if (!ucontext->db_mmap_entry) { - return -ENOMEM; - } - - if (ib_copy_to_udata(udata, &uresp, - min(sizeof(uresp), udata->outlen))) { - rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); - return -EFAULT; - } - } - - INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); - spin_lock_init(&ucontext->cq_reg_mem_list_lock); - INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); - spin_lock_init(&ucontext->qp_reg_mem_list_lock); - INIT_LIST_HEAD(&ucontext->vma_list); - mutex_init(&ucontext->vma_list_mutex); - - return 0; - -ver_error: - irdma_dev_err(&iwdev->rf->sc_dev, - "Invalid userspace driver version detected. Detected version %d, should be %d\n", - req.userspace_ver, IRDMA_ABI_VER); - return -EINVAL; -} -#endif - -#if __FreeBSD_version < 1400026 /** * irdma_alloc_ucontext - Allocate the user context data structure * @ibdev: ib device pointer * @udata: user data * * This keeps track of all objects associated with a particular * user-mode client. */ struct ib_ucontext * irdma_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata) { struct irdma_device *iwdev = to_iwdev(ibdev); struct irdma_alloc_ucontext_req req = {0}; struct irdma_alloc_ucontext_resp uresp = {0}; struct irdma_ucontext *ucontext; struct irdma_uk_attrs *uk_attrs; if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN || udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN) return ERR_PTR(-EINVAL); if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) return ERR_PTR(-EINVAL); if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) goto ver_error; ucontext = kzalloc(sizeof(*ucontext), GFP_KERNEL); if (!ucontext) return ERR_PTR(-ENOMEM); ucontext->iwdev = iwdev; ucontext->abi_ver = req.userspace_ver; uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; /* GEN_1 legacy support with libi40iw */ if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) { if (uk_attrs->hw_rev != IRDMA_GEN_1) { kfree(ucontext); return ERR_PTR(-EOPNOTSUPP); } ucontext->legacy_mode = true; uresp.max_qps = iwdev->rf->max_qp; uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; uresp.kernel_ver = req.userspace_ver; if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) { kfree(ucontext); return ERR_PTR(-EFAULT); } } else { u64 bar_off; uresp.kernel_ver = IRDMA_ABI_VER; uresp.feature_flags = uk_attrs->feature_flags; uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; uresp.max_hw_inline = uk_attrs->max_hw_inline; uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; uresp.hw_rev = uk_attrs->hw_rev; bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; spin_lock_init(&ucontext->mmap_tbl_lock); ucontext->db_mmap_entry = irdma_user_mmap_entry_add_hash(ucontext, bar_off, IRDMA_MMAP_IO_NC, &uresp.db_mmap_key); if (!ucontext->db_mmap_entry) { spin_lock_destroy(&ucontext->mmap_tbl_lock); kfree(ucontext); return ERR_PTR(-ENOMEM); } if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) { irdma_user_mmap_entry_del_hash(ucontext->db_mmap_entry); spin_lock_destroy(&ucontext->mmap_tbl_lock); kfree(ucontext); return ERR_PTR(-EFAULT); } } INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); spin_lock_init(&ucontext->cq_reg_mem_list_lock); INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); spin_lock_init(&ucontext->qp_reg_mem_list_lock); INIT_LIST_HEAD(&ucontext->vma_list); mutex_init(&ucontext->vma_list_mutex); return &ucontext->ibucontext; ver_error: ibdev_err(&iwdev->ibdev, "Invalid userspace driver version detected. Detected version %d, should be %d\n", req.userspace_ver, IRDMA_ABI_VER); return ERR_PTR(-EINVAL); } -#endif -#if __FreeBSD_version >= 1400026 -/** - * irdma_dealloc_ucontext - deallocate the user context data structure - * @context: user context created during alloc - */ -void -irdma_dealloc_ucontext(struct ib_ucontext *context) -{ - struct irdma_ucontext *ucontext = to_ucontext(context); - - rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); - - return; -} -#endif - -#if __FreeBSD_version < 1400026 /** * irdma_dealloc_ucontext - deallocate the user context data structure * @context: user context created during alloc */ int irdma_dealloc_ucontext(struct ib_ucontext *context) { struct irdma_ucontext *ucontext = to_ucontext(context); irdma_user_mmap_entry_del_hash(ucontext->db_mmap_entry); spin_lock_destroy(&ucontext->mmap_tbl_lock); kfree(ucontext); return 0; } -#endif #define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd) -#if __FreeBSD_version >= 1400026 -/** - * irdma_alloc_pd - allocate protection domain - * @pd: protection domain - * @udata: user data - */ -int -irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) -{ - struct irdma_pd *iwpd = to_iwpd(pd); - struct irdma_device *iwdev = to_iwdev(pd->device); - struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; - struct irdma_pci_f *rf = iwdev->rf; - struct irdma_alloc_pd_resp uresp = {0}; - struct irdma_sc_pd *sc_pd; - u32 pd_id = 0; - int err; - - if (udata && udata->outlen < IRDMA_ALLOC_PD_MIN_RESP_LEN) - return -EINVAL; - - err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, - &rf->next_pd); - if (err) - return err; - - sc_pd = &iwpd->sc_pd; - if (udata) { - struct irdma_ucontext *ucontext = - rdma_udata_to_drv_context(udata, struct irdma_ucontext, - ibucontext); - - irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); - uresp.pd_id = pd_id; - if (ib_copy_to_udata(udata, &uresp, - min(sizeof(uresp), udata->outlen))) { - err = -EFAULT; - goto error; - } - } else { - irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); - } - - return 0; - -error: - - irdma_free_rsrc(rf, rf->allocated_pds, pd_id); - - return err; -} -#endif - -#if __FreeBSD_version < 1400026 /** * irdma_alloc_pd - allocate protection domain * @ibdev: IB device * @context: user context * @udata: user data */ struct ib_pd * irdma_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context, struct ib_udata *udata) { struct irdma_pd *iwpd; struct irdma_device *iwdev = to_iwdev(ibdev); struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; struct irdma_pci_f *rf = iwdev->rf; struct irdma_alloc_pd_resp uresp = {0}; struct irdma_sc_pd *sc_pd; u32 pd_id = 0; int err; err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, &rf->next_pd); if (err) return ERR_PTR(err); iwpd = kzalloc(sizeof(*iwpd), GFP_KERNEL); if (!iwpd) { err = -ENOMEM; goto free_res; } sc_pd = &iwpd->sc_pd; if (udata) { struct irdma_ucontext *ucontext = to_ucontext(context); irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); uresp.pd_id = pd_id; if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) { err = -EFAULT; goto error; } } else { irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); } return &iwpd->ibpd; error: kfree(iwpd); free_res: irdma_free_rsrc(rf, rf->allocated_pds, pd_id); return ERR_PTR(err); } -#endif - -#if __FreeBSD_version >= 1400026 -void -irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) -{ - struct irdma_pd *iwpd = to_iwpd(ibpd); - struct irdma_device *iwdev = to_iwdev(ibpd->device); - - irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); -} - -#endif - -#if __FreeBSD_version < 1400026 int irdma_dealloc_pd(struct ib_pd *ibpd) { struct irdma_pd *iwpd = to_iwpd(ibpd); struct irdma_device *iwdev = to_iwdev(ibpd->device); irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); kfree(iwpd); return 0; } -#endif static void irdma_fill_ah_info(struct vnet *vnet, struct irdma_ah_info *ah_info, const struct ib_gid_attr *sgid_attr, struct sockaddr *sgid_addr, struct sockaddr *dgid_addr, u8 *dmac, u8 net_type) { if (net_type == RDMA_NETWORK_IPV4) { ah_info->ipv4_valid = true; ah_info->dest_ip_addr[0] = ntohl(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr); ah_info->src_ip_addr[0] = ntohl(((struct sockaddr_in *)sgid_addr)->sin_addr.s_addr); #ifdef VIMAGE CURVNET_SET_QUIET(vnet); ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], ah_info->dest_ip_addr[0]); CURVNET_RESTORE(); #endif if (ipv4_is_multicast(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr)) { irdma_mcast_mac_v4(ah_info->dest_ip_addr, dmac); } } else { irdma_copy_ip_ntohl(ah_info->dest_ip_addr, ((struct sockaddr_in6 *)dgid_addr)->sin6_addr.__u6_addr.__u6_addr32); irdma_copy_ip_ntohl(ah_info->src_ip_addr, ((struct sockaddr_in6 *)sgid_addr)->sin6_addr.__u6_addr.__u6_addr32); ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, ah_info->dest_ip_addr); if (rdma_is_multicast_addr(&((struct sockaddr_in6 *)dgid_addr)->sin6_addr)) { irdma_mcast_mac_v6(ah_info->dest_ip_addr, dmac); } } } static int irdma_create_ah_vlan_tag(struct irdma_device *iwdev, struct irdma_ah_info *ah_info, const struct ib_gid_attr *sgid_attr, u8 *dmac) { if (sgid_attr->ndev && is_vlan_dev(sgid_attr->ndev)) ah_info->vlan_tag = vlan_dev_vlan_id(sgid_attr->ndev); else ah_info->vlan_tag = VLAN_N_VID; ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, dmac); if (ah_info->dst_arpindex == -1) return -EINVAL; if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode) ah_info->vlan_tag = 0; if (ah_info->vlan_tag < VLAN_N_VID) { ah_info->insert_vlan_tag = true; ah_info->vlan_tag |= (u16)rt_tos2priority(ah_info->tc_tos) << VLAN_PRIO_SHIFT; } if (iwdev->roce_dcqcn_en) { ah_info->tc_tos &= ~ECN_CODE_PT_MASK; ah_info->tc_tos |= ECN_CODE_PT_VAL; } return 0; } static int irdma_create_ah_wait(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, bool sleep) { if (!sleep) { int cnt = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms * CQP_TIMEOUT_THRESHOLD; do { irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); mdelay(1); } while (!sc_ah->ah_info.ah_valid && --cnt); if (!cnt) return -ETIMEDOUT; } return 0; } #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd) -#if __FreeBSD_version >= 1400026 -/** - * irdma_create_ah - create address handle - * @ib_ah: ptr to AH - * @attr: address handle attributes - * @flags: AH flags to wait - * @udata: user data - * - * returns 0 on success, error otherwise - */ -int -irdma_create_ah(struct ib_ah *ib_ah, - struct ib_ah_attr *attr, u32 flags, - struct ib_udata *udata) -{ - struct irdma_pd *pd = to_iwpd(ib_ah->pd); - struct irdma_ah *ah = container_of(ib_ah, struct irdma_ah, ibah); - struct irdma_device *iwdev = to_iwdev(ib_ah->pd->device); - union ib_gid sgid; - struct ib_gid_attr sgid_attr; - struct irdma_pci_f *rf = iwdev->rf; - struct irdma_sc_ah *sc_ah; - u32 ah_id = 0; - struct irdma_ah_info *ah_info; - struct irdma_create_ah_resp uresp; - union { - struct sockaddr saddr; - struct sockaddr_in saddr_in; - struct sockaddr_in6 saddr_in6; - } sgid_addr, dgid_addr; - int err; - u8 dmac[ETH_ALEN]; - bool sleep = (flags & RDMA_CREATE_AH_SLEEPABLE) != 0; - - if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN) - return -EINVAL; - - err = irdma_alloc_rsrc(rf, rf->allocated_ahs, - rf->max_ah, &ah_id, &rf->next_ah); - - if (err) - return err; - - ah->pd = pd; - sc_ah = &ah->sc_ah; - sc_ah->ah_info.ah_idx = ah_id; - sc_ah->ah_info.vsi = &iwdev->vsi; - irdma_sc_init_ah(&rf->sc_dev, sc_ah); - ah->sgid_index = attr->grh.sgid_index; - memcpy(&ah->dgid, &attr->grh.dgid, sizeof(ah->dgid)); - rcu_read_lock(); - err = ib_get_cached_gid(&iwdev->ibdev, attr->port_num, - attr->grh.sgid_index, &sgid, &sgid_attr); - rcu_read_unlock(); - if (err) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "GID lookup at idx=%d with port=%d failed\n", - attr->grh.sgid_index, attr->port_num); - err = -EINVAL; - goto err_gid_l2; - } - rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid); - rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid); - ah->av.attrs = *attr; - ah->av.net_type = kc_rdma_gid_attr_network_type(sgid_attr, - sgid_attr.gid_type, - &sgid); - - if (sgid_attr.ndev) - dev_put(sgid_attr.ndev); - - ah->av.sgid_addr.saddr = sgid_addr.saddr; - ah->av.dgid_addr.saddr = dgid_addr.saddr; - ah_info = &sc_ah->ah_info; - ah_info->ah_idx = ah_id; - ah_info->pd_idx = pd->sc_pd.pd_id; - ether_addr_copy(ah_info->mac_addr, IF_LLADDR(iwdev->netdev)); - - if (attr->ah_flags & IB_AH_GRH) { - ah_info->flow_label = attr->grh.flow_label; - ah_info->hop_ttl = attr->grh.hop_limit; - ah_info->tc_tos = attr->grh.traffic_class; - } - - ether_addr_copy(dmac, attr->dmac); - - irdma_fill_ah_info(iwdev->netdev->if_vnet, ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr, - dmac, ah->av.net_type); - - err = irdma_create_ah_vlan_tag(iwdev, ah_info, &sgid_attr, dmac); - if (err) - goto err_gid_l2; - - err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE, - sleep, irdma_gsi_ud_qp_ah_cb, sc_ah); - if (err) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "CQP-OP Create AH fail"); - goto err_gid_l2; - } - - err = irdma_create_ah_wait(rf, sc_ah, sleep); - if (err) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "CQP create AH timed out"); - goto err_gid_l2; - } - - if (udata) { - uresp.ah_id = ah->sc_ah.ah_info.ah_idx; - err = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); - if (err) { - irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, - IRDMA_OP_AH_DESTROY, false, NULL, ah); - goto err_gid_l2; - } - } - - return 0; -err_gid_l2: - irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id); - - return err; -} -#endif - void irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr) { ether_addr_copy(dmac, attr->dmac); } -#if __FreeBSD_version < 1400026 struct ib_ah * irdma_create_ah_stub(struct ib_pd *ibpd, struct ib_ah_attr *attr, struct ib_udata *udata) -#else -int -irdma_create_ah_stub(struct ib_ah *ib_ah, - struct ib_ah_attr *attr, u32 flags, - struct ib_udata *udata) -#endif { -#if __FreeBSD_version >= 1400026 - return -ENOSYS; -#else return ERR_PTR(-ENOSYS); -#endif } -#if __FreeBSD_version >= 1400026 -void -irdma_destroy_ah_stub(struct ib_ah *ibah, u32 flags) -{ - return; -} -#else int irdma_destroy_ah_stub(struct ib_ah *ibah) { return -ENOSYS; } -#endif -#if __FreeBSD_version < 1400026 /** * irdma_create_ah - create address handle * @ibpd: ptr to pd * @attr: address handle attributes * @udata: user data * * returns a pointer to an address handle */ struct ib_ah * irdma_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr, struct ib_udata *udata) { struct irdma_pd *pd = to_iwpd(ibpd); struct irdma_device *iwdev = to_iwdev(ibpd->device); struct irdma_ah *ah; union ib_gid sgid; struct ib_gid_attr sgid_attr; struct irdma_pci_f *rf = iwdev->rf; struct irdma_sc_ah *sc_ah; u32 ah_id = 0; struct irdma_ah_info *ah_info; struct irdma_create_ah_resp uresp; union { struct sockaddr saddr; struct sockaddr_in saddr_in; struct sockaddr_in6 saddr_in6; } sgid_addr, dgid_addr; int err; u8 dmac[ETH_ALEN]; bool sleep = udata ? true : false; if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN) return ERR_PTR(-EINVAL); err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, &ah_id, &rf->next_ah); if (err) return ERR_PTR(err); ah = kzalloc(sizeof(*ah), GFP_ATOMIC); if (!ah) { irdma_free_rsrc(rf, rf->allocated_ahs, ah_id); return ERR_PTR(-ENOMEM); } ah->pd = pd; sc_ah = &ah->sc_ah; sc_ah->ah_info.ah_idx = ah_id; sc_ah->ah_info.vsi = &iwdev->vsi; irdma_sc_init_ah(&rf->sc_dev, sc_ah); ah->sgid_index = attr->grh.sgid_index; memcpy(&ah->dgid, &attr->grh.dgid, sizeof(ah->dgid)); rcu_read_lock(); err = ib_get_cached_gid(&iwdev->ibdev, attr->port_num, attr->grh.sgid_index, &sgid, &sgid_attr); rcu_read_unlock(); if (err) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "GID lookup at idx=%d with port=%d failed\n", attr->grh.sgid_index, attr->port_num); err = -EINVAL; goto err_gid_l2; } rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid); rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid); ah->av.attrs = *attr; ah->av.net_type = kc_rdma_gid_attr_network_type(sgid_attr, sgid_attr.gid_type, &sgid); if (sgid_attr.ndev) dev_put(sgid_attr.ndev); ah->av.sgid_addr.saddr = sgid_addr.saddr; ah->av.dgid_addr.saddr = dgid_addr.saddr; ah_info = &sc_ah->ah_info; ah_info->ah_idx = ah_id; ah_info->pd_idx = pd->sc_pd.pd_id; ether_addr_copy(ah_info->mac_addr, IF_LLADDR(iwdev->netdev)); if (attr->ah_flags & IB_AH_GRH) { ah_info->flow_label = attr->grh.flow_label; ah_info->hop_ttl = attr->grh.hop_limit; ah_info->tc_tos = attr->grh.traffic_class; } if (udata) ib_resolve_eth_dmac(ibpd->device, attr); irdma_ether_copy(dmac, attr); irdma_fill_ah_info(iwdev->netdev->if_vnet, ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr, dmac, ah->av.net_type); err = irdma_create_ah_vlan_tag(iwdev, ah_info, &sgid_attr, dmac); if (err) goto err_gid_l2; err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE, sleep, irdma_gsi_ud_qp_ah_cb, sc_ah); if (err) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "CQP-OP Create AH fail"); goto err_gid_l2; } err = irdma_create_ah_wait(rf, sc_ah, sleep); if (err) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "CQP create AH timed out"); goto err_gid_l2; } if (udata) { uresp.ah_id = ah->sc_ah.ah_info.ah_idx; err = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); if (err) { irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, false, NULL, ah); goto err_gid_l2; } } return &ah->ibah; err_gid_l2: kfree(ah); irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id); return ERR_PTR(err); } -#endif /** * irdma_free_qp_rsrc - free up memory resources for qp * @iwqp: qp ptr (user or kernel) */ void irdma_free_qp_rsrc(struct irdma_qp *iwqp) { struct irdma_device *iwdev = iwqp->iwdev; struct irdma_pci_f *rf = iwdev->rf; u32 qp_num = iwqp->ibqp.qp_num; irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); irdma_dealloc_push_page(rf, &iwqp->sc_qp); if (iwqp->sc_qp.vsi) { irdma_qp_rem_qos(&iwqp->sc_qp); iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, iwqp->sc_qp.user_pri); } if (qp_num > 2) irdma_free_rsrc(rf, rf->allocated_qps, qp_num); irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->q2_ctx_mem); irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->kqp.dma_mem); kfree(iwqp->kqp.sig_trk_mem); iwqp->kqp.sig_trk_mem = NULL; kfree(iwqp->kqp.sq_wrid_mem); kfree(iwqp->kqp.rq_wrid_mem); kfree(iwqp->sg_list); kfree(iwqp); } /** * irdma_create_qp - create qp * @ibpd: ptr of pd * @init_attr: attributes for qp * @udata: user data for create qp */ struct ib_qp * irdma_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { #define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx) #define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd) struct irdma_pd *iwpd = to_iwpd(ibpd); struct irdma_device *iwdev = to_iwdev(ibpd->device); struct irdma_pci_f *rf = iwdev->rf; struct irdma_qp *iwqp; struct irdma_create_qp_resp uresp = {0}; u32 qp_num = 0; int ret; int err_code; struct irdma_sc_qp *qp; struct irdma_sc_dev *dev = &rf->sc_dev; struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; struct irdma_qp_init_info init_info = {{0}}; struct irdma_qp_host_ctx_info *ctx_info; err_code = irdma_validate_qp_attrs(init_attr, iwdev); if (err_code) return ERR_PTR(err_code); if (udata && (udata->inlen < IRDMA_CREATE_QP_MIN_REQ_LEN || udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN)) return ERR_PTR(-EINVAL); init_info.vsi = &iwdev->vsi; init_info.qp_uk_init_info.uk_attrs = uk_attrs; init_info.qp_uk_init_info.sq_size = init_attr->cap.max_send_wr; init_info.qp_uk_init_info.rq_size = init_attr->cap.max_recv_wr; init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL); if (!iwqp) return ERR_PTR(-ENOMEM); iwqp->sg_list = kcalloc(uk_attrs->max_hw_wq_frags, sizeof(*iwqp->sg_list), GFP_KERNEL); if (!iwqp->sg_list) { kfree(iwqp); return ERR_PTR(-ENOMEM); } qp = &iwqp->sc_qp; qp->qp_uk.back_qp = iwqp; qp->qp_uk.lock = &iwqp->lock; qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; iwqp->iwdev = iwdev; iwqp->q2_ctx_mem.size = IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE; iwqp->q2_ctx_mem.va = irdma_allocate_dma_mem(dev->hw, &iwqp->q2_ctx_mem, iwqp->q2_ctx_mem.size, 256); if (!iwqp->q2_ctx_mem.va) { kfree(iwqp->sg_list); kfree(iwqp); return ERR_PTR(-ENOMEM); } init_info.q2 = iwqp->q2_ctx_mem.va; init_info.q2_pa = iwqp->q2_ctx_mem.pa; init_info.host_ctx = (__le64 *) (init_info.q2 + IRDMA_Q2_BUF_SIZE); init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE; if (init_attr->qp_type == IB_QPT_GSI) qp_num = 1; else err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, &qp_num, &rf->next_qp); if (err_code) goto error; iwqp->iwpd = iwpd; iwqp->ibqp.qp_num = qp_num; qp = &iwqp->sc_qp; iwqp->iwscq = to_iwcq(init_attr->send_cq); iwqp->iwrcq = to_iwcq(init_attr->recv_cq); iwqp->host_ctx.va = init_info.host_ctx; iwqp->host_ctx.pa = init_info.host_ctx_pa; iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; init_info.pd = &iwpd->sc_pd; init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num; if (!rdma_protocol_roce(&iwdev->ibdev, 1)) init_info.qp_uk_init_info.first_sq_wq = 1; iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; init_waitqueue_head(&iwqp->waitq); init_waitqueue_head(&iwqp->mod_qp_waitq); if (udata) { init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info, init_attr); } else { INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker); init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); } if (err_code) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "setup qp failed\n"); goto error; } if (rdma_protocol_roce(&iwdev->ibdev, 1)) { if (init_attr->qp_type == IB_QPT_RC) { init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC; init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | IRDMA_WRITE_WITH_IMM | IRDMA_ROCE; } else { init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD; init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | IRDMA_ROCE; } } else { init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP; init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM; } ret = irdma_sc_qp_init(qp, &init_info); if (ret) { err_code = -EPROTO; irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "qp_init fail\n"); goto error; } ctx_info = &iwqp->ctx_info; ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; if (rdma_protocol_roce(&iwdev->ibdev, 1)) irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); else irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); err_code = irdma_cqp_create_qp_cmd(iwqp); if (err_code) goto error; atomic_set(&iwqp->refcnt, 1); spin_lock_init(&iwqp->lock); spin_lock_init(&iwqp->sc_qp.pfpdu.lock); iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; rf->qp_table[qp_num] = iwqp; if (rdma_protocol_roce(&iwdev->ibdev, 1)) { if (dev->ws_add(&iwdev->vsi, 0)) { irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); err_code = -EINVAL; goto error; } irdma_qp_add_qos(&iwqp->sc_qp); } if (udata) { /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ if (udata->outlen < sizeof(uresp)) { uresp.lsmm = 1; uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; } else { if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) uresp.lsmm = 1; } uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size; uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size; uresp.qp_id = qp_num; uresp.qp_caps = qp->qp_uk.qp_caps; err_code = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)); if (err_code) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "copy_to_udata failed\n"); kc_irdma_destroy_qp(&iwqp->ibqp, udata); return ERR_PTR(err_code); } } init_completion(&iwqp->free_qp); return &iwqp->ibqp; error: irdma_free_qp_rsrc(iwqp); return ERR_PTR(err_code); } /** * irdma_destroy_qp - destroy qp * @ibqp: qp's ib pointer also to get to device's qp address * @udata: user data */ -#if __FreeBSD_version >= 1400026 -int -irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) -#else int irdma_destroy_qp(struct ib_qp *ibqp) -#endif { struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_device *iwdev = iwqp->iwdev; if (iwqp->sc_qp.qp_uk.destroy_pending) goto free_rsrc; iwqp->sc_qp.qp_uk.destroy_pending = true; if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) irdma_modify_qp_to_err(&iwqp->sc_qp); irdma_qp_rem_ref(&iwqp->ibqp); wait_for_completion(&iwqp->free_qp); irdma_free_lsmm_rsrc(iwqp); if (!iwdev->rf->reset && irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp)) return -ENOTRECOVERABLE; free_rsrc: if (!iwqp->user_mode) { if (iwqp->iwscq) { irdma_clean_cqes(iwqp, iwqp->iwscq); if (iwqp->iwrcq != iwqp->iwscq) irdma_clean_cqes(iwqp, iwqp->iwrcq); } } irdma_remove_push_mmap_entries(iwqp); irdma_free_qp_rsrc(iwqp); return 0; } /** * irdma_create_cq - create cq * @ibcq: CQ allocated * @attr: attributes for cq * @udata: user data */ -#if __FreeBSD_version >= 1400026 -int -irdma_create_cq(struct ib_cq *ibcq, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) -#else struct ib_cq * irdma_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, struct ib_ucontext *context, struct ib_udata *udata) -#endif { #define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf) #define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size) -#if __FreeBSD_version >= 1400026 - struct ib_device *ibdev = ibcq->device; -#endif struct irdma_device *iwdev = to_iwdev(ibdev); struct irdma_pci_f *rf = iwdev->rf; -#if __FreeBSD_version >= 1400026 - struct irdma_cq *iwcq = to_iwcq(ibcq); -#else struct irdma_cq *iwcq; -#endif u32 cq_num = 0; struct irdma_sc_cq *cq; struct irdma_sc_dev *dev = &rf->sc_dev; struct irdma_cq_init_info info = {0}; int status; struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; unsigned long flags; int err_code; int entries = attr->cqe; bool cqe_64byte_ena; -#if __FreeBSD_version >= 1400026 - err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); - if (err_code) - return err_code; - - if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN || - udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN)) - return -EINVAL; -#else err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); if (err_code) return ERR_PTR(err_code); if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN || udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN)) return ERR_PTR(-EINVAL); iwcq = kzalloc(sizeof(*iwcq), GFP_KERNEL); if (!iwcq) return ERR_PTR(-ENOMEM); -#endif err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, &rf->next_cq); if (err_code) -#if __FreeBSD_version >= 1400026 - return err_code; -#else goto error; -#endif cq = &iwcq->sc_cq; cq->back_cq = iwcq; atomic_set(&iwcq->refcnt, 1); spin_lock_init(&iwcq->lock); INIT_LIST_HEAD(&iwcq->resize_list); INIT_LIST_HEAD(&iwcq->cmpl_generated); info.dev = dev; ukinfo->cq_size = max(entries, 4); ukinfo->cq_id = cq_num; cqe_64byte_ena = (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_64_BYTE_CQE) ? true : false; ukinfo->avoid_mem_cflct = cqe_64byte_ena; iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; if (attr->comp_vector < rf->ceqs_count) info.ceq_id = attr->comp_vector; info.ceq_id_valid = true; info.ceqe_mask = 1; info.type = IRDMA_CQ_TYPE_IWARP; info.vsi = &iwdev->vsi; if (udata) { struct irdma_ucontext *ucontext; struct irdma_create_cq_req req = {0}; struct irdma_cq_mr *cqmr; struct irdma_pbl *iwpbl; struct irdma_pbl *iwpbl_shadow; struct irdma_cq_mr *cqmr_shadow; iwcq->user_mode = true; -#if __FreeBSD_version >= 1400026 - ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); -#else ucontext = to_ucontext(context); -#endif if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) { err_code = -EFAULT; goto cq_free_rsrc; } spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, &ucontext->cq_reg_mem_list); spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); if (!iwpbl) { err_code = -EPROTO; goto cq_free_rsrc; } iwcq->iwpbl = iwpbl; iwcq->cq_mem_size = 0; cqmr = &iwpbl->cq_mr; if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); iwpbl_shadow = irdma_get_pbl((unsigned long)req.user_shadow_area, &ucontext->cq_reg_mem_list); spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); if (!iwpbl_shadow) { err_code = -EPROTO; goto cq_free_rsrc; } iwcq->iwpbl_shadow = iwpbl_shadow; cqmr_shadow = &iwpbl_shadow->cq_mr; info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; cqmr->split = true; } else { info.shadow_area_pa = cqmr->shadow; } if (iwpbl->pbl_allocated) { info.virtual_map = true; info.pbl_chunk_size = 1; info.first_pm_pbl_idx = cqmr->cq_pbl.idx; } else { info.cq_base_pa = cqmr->cq_pbl.addr; } } else { /* Kmode allocations */ int rsize; if (entries < 1 || entries > rf->max_cqe) { err_code = -EINVAL; goto cq_free_rsrc; } entries++; if (!cqe_64byte_ena && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) entries *= 2; ukinfo->cq_size = entries; if (cqe_64byte_ena) rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_extended_cqe); else rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe); iwcq->kmem.size = round_up(rsize, IRDMA_HW_PAGE_SIZE); iwcq->kmem.va = irdma_allocate_dma_mem(dev->hw, &iwcq->kmem, iwcq->kmem.size, IRDMA_HW_PAGE_SIZE); if (!iwcq->kmem.va) { err_code = -ENOMEM; goto cq_free_rsrc; } iwcq->kmem_shadow.size = IRDMA_SHADOW_AREA_SIZE << 3; iwcq->kmem_shadow.va = irdma_allocate_dma_mem(dev->hw, &iwcq->kmem_shadow, iwcq->kmem_shadow.size, 64); if (!iwcq->kmem_shadow.va) { err_code = -ENOMEM; goto cq_free_rsrc; } info.shadow_area_pa = iwcq->kmem_shadow.pa; ukinfo->shadow_area = iwcq->kmem_shadow.va; ukinfo->cq_base = iwcq->kmem.va; info.cq_base_pa = iwcq->kmem.pa; } if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, (u32)IRDMA_MAX_CQ_READ_THRESH); if (irdma_sc_cq_init(cq, &info)) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "init cq fail\n"); err_code = -EPROTO; goto cq_free_rsrc; } cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); if (!cqp_request) { err_code = -ENOMEM; goto cq_free_rsrc; } cqp_info = &cqp_request->info; cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; cqp_info->post_sq = 1; cqp_info->in.u.cq_create.cq = cq; cqp_info->in.u.cq_create.check_overflow = true; cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; status = irdma_handle_cqp_op(rf, cqp_request); irdma_put_cqp_request(&rf->cqp, cqp_request); if (status) { err_code = -ENOMEM; goto cq_free_rsrc; } if (udata) { struct irdma_create_cq_resp resp = {0}; resp.cq_id = info.cq_uk_init_info.cq_id; resp.cq_size = info.cq_uk_init_info.cq_size; if (ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen))) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "copy to user data\n"); err_code = -EPROTO; goto cq_destroy; } } rf->cq_table[cq_num] = iwcq; init_completion(&iwcq->free_cq); -#if __FreeBSD_version >= 1400026 - return 0; -#else return &iwcq->ibcq; -#endif cq_destroy: irdma_cq_wq_destroy(rf, cq); cq_free_rsrc: irdma_cq_free_rsrc(rf, iwcq); -#if __FreeBSD_version >= 1400026 - return err_code; -#else error: kfree(iwcq); return ERR_PTR(err_code); -#endif } /** * irdma_copy_user_pgaddrs - copy user page address to pble's os locally * @iwmr: iwmr for IB's user page addresses * @pbl: ple pointer to save 1 level or 0 level pble * @level: indicated level 0, 1 or 2 */ void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, enum irdma_pble_level level) { struct ib_umem *region = iwmr->region; struct irdma_pbl *iwpbl = &iwmr->iwpbl; int chunk_pages, entry, i; struct scatterlist *sg; u64 pg_addr = 0; struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; struct irdma_pble_info *pinfo; u32 idx = 0; u32 pbl_cnt = 0; pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) { chunk_pages = DIV_ROUND_UP(sg_dma_len(sg), iwmr->page_size); if (iwmr->type == IRDMA_MEMREG_TYPE_QP && !iwpbl->qp_mr.sq_page) iwpbl->qp_mr.sq_page = sg_page(sg); for (i = 0; i < chunk_pages; i++) { pg_addr = sg_dma_address(sg) + (i * iwmr->page_size); if ((entry + i) == 0) *pbl = pg_addr & iwmr->page_msk; else if (!(pg_addr & ~iwmr->page_msk)) *pbl = pg_addr; else continue; if (++pbl_cnt == palloc->total_cnt) break; pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); } } } /** * irdma_destroy_ah - Destroy address handle * @ibah: pointer to address handle * @ah_flags: destroy flags */ - -#if __FreeBSD_version >= 1400026 -void -irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags) -{ - struct irdma_device *iwdev = to_iwdev(ibah->device); - struct irdma_ah *ah = to_iwah(ibah); - - irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, - false, NULL, ah); - - irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, - ah->sc_ah.ah_info.ah_idx); -} -#endif - -#if __FreeBSD_version < 1400026 int irdma_destroy_ah(struct ib_ah *ibah) { struct irdma_device *iwdev = to_iwdev(ibah->device); struct irdma_ah *ah = to_iwah(ibah); irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, false, NULL, ah); irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah->sc_ah.ah_info.ah_idx); kfree(ah); return 0; } -#endif -#if __FreeBSD_version >= 1400026 -int -irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) -#else int irdma_dereg_mr(struct ib_mr *ib_mr) -#endif { struct irdma_mr *iwmr = to_iwmr(ib_mr); struct irdma_device *iwdev = to_iwdev(ib_mr->device); struct irdma_pbl *iwpbl = &iwmr->iwpbl; int ret; if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { if (iwmr->region) { struct irdma_ucontext *ucontext; -#if __FreeBSD_version >= 1400026 - - ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); - -#else struct ib_pd *ibpd = ib_mr->pd; ucontext = to_ucontext(ibpd->uobject->context); -#endif irdma_del_memlist(iwmr, ucontext); } goto done; } ret = irdma_hwdereg_mr(ib_mr); if (ret) return ret; irdma_free_stag(iwdev, iwmr->stag); done: if (iwpbl->pbl_allocated) irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); if (iwmr->region) ib_umem_release(iwmr->region); kfree(iwmr); return 0; } /* * irdma_rereg_user_mr - Re-Register a user memory region @ibmr: ib mem to access iwarp mr pointer @flags: bit mask to * indicate which of the attr's of MR modified @start: virtual start address @len: length of mr @virt: virtual address * @new access flags: bit mask of access flags @new_pd: ptr of pd @udata: user data */ int irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 len, u64 virt, int new_access, struct ib_pd *new_pd, struct ib_udata *udata) { struct irdma_device *iwdev = to_iwdev(ib_mr->device); struct irdma_mr *iwmr = to_iwmr(ib_mr); struct irdma_pbl *iwpbl = &iwmr->iwpbl; int ret; if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) return -EINVAL; if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) return -EOPNOTSUPP; ret = irdma_hwdereg_mr(ib_mr); if (ret) return ret; if (flags & IB_MR_REREG_ACCESS) iwmr->access = new_access; if (flags & IB_MR_REREG_PD) { iwmr->ibmr.pd = new_pd; iwmr->ibmr.device = new_pd->device; } if (flags & IB_MR_REREG_TRANS) { if (iwpbl->pbl_allocated) { irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); iwpbl->pbl_allocated = false; } if (iwmr->region) { ib_umem_release(iwmr->region); iwmr->region = NULL; } ib_mr = irdma_rereg_mr_trans(iwmr, start, len, virt, udata); if (IS_ERR(ib_mr)) return PTR_ERR(ib_mr); } else { ret = irdma_hwreg_mr(iwdev, iwmr, iwmr->access); if (ret) return ret; } return 0; } int kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp, struct ib_qp_attr *attr, u16 *vlan_id) { int ret; union ib_gid sgid; struct ib_gid_attr sgid_attr; struct irdma_av *av = &iwqp->roce_ah.av; ret = ib_get_cached_gid(iwqp->ibqp.device, attr->ah_attr.port_num, attr->ah_attr.grh.sgid_index, &sgid, &sgid_attr); if (ret) return ret; if (sgid_attr.ndev) { *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr.ndev); ether_addr_copy(iwqp->ctx_info.roce_info->mac_addr, IF_LLADDR(sgid_attr.ndev)); } rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid); dev_put(sgid_attr.ndev); iwqp->sc_qp.user_pri = iwqp->ctx_info.user_pri; return 0; } -#if __FreeBSD_version >= 1400026 -/** - * irdma_destroy_cq - destroy cq - * @ib_cq: cq pointer - * @udata: user data - */ -void -irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) -{ - struct irdma_device *iwdev = to_iwdev(ib_cq->device); - struct irdma_cq *iwcq = to_iwcq(ib_cq); - struct irdma_sc_cq *cq = &iwcq->sc_cq; - struct irdma_sc_dev *dev = cq->dev; - struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; - struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); - unsigned long flags; - - spin_lock_irqsave(&iwcq->lock, flags); - if (!list_empty(&iwcq->cmpl_generated)) - irdma_remove_cmpls_list(iwcq); - if (!list_empty(&iwcq->resize_list)) - irdma_process_resize_list(iwcq, iwdev, NULL); - spin_unlock_irqrestore(&iwcq->lock, flags); - - irdma_cq_rem_ref(ib_cq); - wait_for_completion(&iwcq->free_cq); - - irdma_cq_wq_destroy(iwdev->rf, cq); - - spin_lock_irqsave(&iwceq->ce_lock, flags); - irdma_sc_cleanup_ceqes(cq, ceq); - spin_unlock_irqrestore(&iwceq->ce_lock, flags); - irdma_cq_free_rsrc(iwdev->rf, iwcq); -} - -#endif -#if __FreeBSD_version < 1400026 /** * irdma_destroy_cq - destroy cq * @ib_cq: cq pointer */ int irdma_destroy_cq(struct ib_cq *ib_cq) { struct irdma_device *iwdev = to_iwdev(ib_cq->device); struct irdma_cq *iwcq = to_iwcq(ib_cq); struct irdma_sc_cq *cq = &iwcq->sc_cq; struct irdma_sc_dev *dev = cq->dev; struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); unsigned long flags; spin_lock_irqsave(&iwcq->lock, flags); if (!list_empty(&iwcq->cmpl_generated)) irdma_remove_cmpls_list(iwcq); if (!list_empty(&iwcq->resize_list)) irdma_process_resize_list(iwcq, iwdev, NULL); spin_unlock_irqrestore(&iwcq->lock, flags); irdma_cq_rem_ref(ib_cq); wait_for_completion(&iwcq->free_cq); irdma_cq_wq_destroy(iwdev->rf, cq); spin_lock_irqsave(&iwceq->ce_lock, flags); irdma_sc_cleanup_ceqes(cq, ceq); spin_unlock_irqrestore(&iwceq->ce_lock, flags); irdma_cq_free_rsrc(iwdev->rf, iwcq); kfree(iwcq); return 0; } -#endif /** * irdma_alloc_mw - Allocate memory window * @pd: Protection domain * @type: Window type * @udata: user data pointer */ struct ib_mw * irdma_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, struct ib_udata *udata) { struct irdma_device *iwdev = to_iwdev(pd->device); struct irdma_mr *iwmr; int err_code; u32 stag; iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); if (!iwmr) return ERR_PTR(-ENOMEM); stag = irdma_create_stag(iwdev); if (!stag) { kfree(iwmr); return ERR_PTR(-ENOMEM); } iwmr->stag = stag; iwmr->ibmw.rkey = stag; iwmr->ibmw.pd = pd; iwmr->ibmw.type = type; iwmr->ibmw.device = pd->device; err_code = irdma_hw_alloc_mw(iwdev, iwmr); if (err_code) { irdma_free_stag(iwdev, stag); kfree(iwmr); return ERR_PTR(err_code); } return &iwmr->ibmw; } /** * kc_set_loc_seq_num_mss - Set local seq number and mss * @cm_node: cm node info */ void kc_set_loc_seq_num_mss(struct irdma_cm_node *cm_node) { struct timespec ts; getnanotime(&ts); cm_node->tcp_cntxt.loc_seq_num = ts.tv_nsec; if (cm_node->iwdev->vsi.mtu > 1500 && 2 * cm_node->iwdev->vsi.mtu > cm_node->iwdev->rcv_wnd) cm_node->tcp_cntxt.mss = (cm_node->ipv4) ? (1500 - IRDMA_MTU_TO_MSS_IPV4) : (1500 - IRDMA_MTU_TO_MSS_IPV6); else cm_node->tcp_cntxt.mss = (cm_node->ipv4) ? (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV4) : (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6); } -#if __FreeBSD_version < 1400026 struct irdma_vma_data { struct list_head list; struct vm_area_struct *vma; struct mutex *vma_list_mutex; /* protect the vma_list */ }; /** * irdma_vma_open - * @vma: User VMA */ static void irdma_vma_open(struct vm_area_struct *vma) { vma->vm_ops = NULL; } /** * irdma_vma_close - Remove vma data from vma list * @vma: User VMA */ static void irdma_vma_close(struct vm_area_struct *vma) { struct irdma_vma_data *vma_data; vma_data = vma->vm_private_data; vma->vm_private_data = NULL; vma_data->vma = NULL; mutex_lock(vma_data->vma_list_mutex); list_del(&vma_data->list); mutex_unlock(vma_data->vma_list_mutex); kfree(vma_data); } static const struct vm_operations_struct irdma_vm_ops = { .open = irdma_vma_open, .close = irdma_vma_close }; /** * irdma_set_vma_data - Save vma data in context list * @vma: User VMA * @context: ib user context */ static int irdma_set_vma_data(struct vm_area_struct *vma, struct irdma_ucontext *context) { struct list_head *vma_head = &context->vma_list; struct irdma_vma_data *vma_entry; vma_entry = kzalloc(sizeof(*vma_entry), GFP_KERNEL); if (!vma_entry) return -ENOMEM; vma->vm_private_data = vma_entry; vma->vm_ops = &irdma_vm_ops; vma_entry->vma = vma; vma_entry->vma_list_mutex = &context->vma_list_mutex; mutex_lock(&context->vma_list_mutex); list_add(&vma_entry->list, vma_head); mutex_unlock(&context->vma_list_mutex); return 0; } /** * irdma_disassociate_ucontext - Disassociate user context * @context: ib user context */ void irdma_disassociate_ucontext(struct ib_ucontext *context) { struct irdma_ucontext *ucontext = to_ucontext(context); struct irdma_vma_data *vma_data, *n; struct vm_area_struct *vma; mutex_lock(&ucontext->vma_list_mutex); list_for_each_entry_safe(vma_data, n, &ucontext->vma_list, list) { vma = vma_data->vma; zap_vma_ptes(vma, vma->vm_start, PAGE_SIZE); vma->vm_ops = NULL; list_del(&vma_data->list); kfree(vma_data); } mutex_unlock(&ucontext->vma_list_mutex); } int rdma_user_mmap_io(struct ib_ucontext *context, struct vm_area_struct *vma, unsigned long pfn, unsigned long size, pgprot_t prot) { if (io_remap_pfn_range(vma, vma->vm_start, pfn, size, prot)) return -EAGAIN; return irdma_set_vma_data(vma, to_ucontext(context)); } -#else -/** - * irdma_disassociate_ucontext - Disassociate user context - * @context: ib user context - */ -void -irdma_disassociate_ucontext(struct ib_ucontext *context) -{ -} -#endif struct ib_device * ib_device_get_by_netdev(struct ifnet *netdev, int driver_id) { struct irdma_device *iwdev; struct irdma_handler *hdl; unsigned long flags; spin_lock_irqsave(&irdma_handler_lock, flags); list_for_each_entry(hdl, &irdma_handlers, list) { iwdev = hdl->iwdev; if (netdev == iwdev->netdev) { spin_unlock_irqrestore(&irdma_handler_lock, flags); return &iwdev->ibdev; } } spin_unlock_irqrestore(&irdma_handler_lock, flags); return NULL; } void ib_unregister_device_put(struct ib_device *device) { ib_unregister_device(device); } /** * irdma_query_gid_roce - Query port GID for Roce * @ibdev: device pointer from stack * @port: port number * @index: Entry index * @gid: Global ID */ int irdma_query_gid_roce(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid) { int ret; ret = rdma_query_gid(ibdev, port, index, gid); if (ret == -EAGAIN) { memcpy(gid, &zgid, sizeof(*gid)); return 0; } return ret; } /** * irdma_modify_port - modify port attributes * @ibdev: device pointer from stack * @port: port number for query * @mask: Property mask * @props: returning device attributes */ int irdma_modify_port(struct ib_device *ibdev, u8 port, int mask, struct ib_port_modify *props) { if (port > 1) return -EINVAL; return 0; } /** * irdma_query_pkey - Query partition key * @ibdev: device pointer from stack * @port: port number * @index: index of pkey * @pkey: pointer to store the pkey */ int irdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) { if (index >= IRDMA_PKEY_TBL_SZ) return -EINVAL; *pkey = IRDMA_DEFAULT_PKEY; return 0; } int irdma_roce_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; int err; immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; err = ib_query_port(ibdev, port_num, &attr); if (err) return err; immutable->max_mad_size = IB_MGMT_MAD_SIZE; immutable->pkey_tbl_len = attr.pkey_tbl_len; immutable->gid_tbl_len = attr.gid_tbl_len; return 0; } int irdma_iw_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable) { struct ib_port_attr attr; int err; immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; err = ib_query_port(ibdev, port_num, &attr); if (err) return err; immutable->gid_tbl_len = 1; return 0; } /** * irdma_get_eth_speed_and_width - Get IB port speed and width from netdev speed * @link_speed: netdev phy link speed * @active_speed: IB port speed * @active_width: IB port width */ void irdma_get_eth_speed_and_width(u32 link_speed, u8 *active_speed, u8 *active_width) { if (link_speed <= SPEED_1000) { *active_width = IB_WIDTH_1X; *active_speed = IB_SPEED_SDR; } else if (link_speed <= SPEED_10000) { *active_width = IB_WIDTH_1X; *active_speed = IB_SPEED_FDR10; } else if (link_speed <= SPEED_20000) { *active_width = IB_WIDTH_4X; *active_speed = IB_SPEED_DDR; } else if (link_speed <= SPEED_25000) { *active_width = IB_WIDTH_1X; *active_speed = IB_SPEED_EDR; } else if (link_speed <= SPEED_40000) { *active_width = IB_WIDTH_4X; *active_speed = IB_SPEED_FDR10; } else { *active_width = IB_WIDTH_4X; *active_speed = IB_SPEED_EDR; } } /** * irdma_query_port - get port attributes * @ibdev: device pointer from stack * @port: port number for query * @props: returning device attributes */ int irdma_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) { struct irdma_device *iwdev = to_iwdev(ibdev); struct ifnet *netdev = iwdev->netdev; /* no need to zero out pros here. done by caller */ props->max_mtu = IB_MTU_4096; props->active_mtu = ib_mtu_int_to_enum(netdev->if_mtu); props->lid = 1; props->lmc = 0; props->sm_lid = 0; props->sm_sl = 0; if ((netdev->if_link_state == LINK_STATE_UP) && (netdev->if_drv_flags & IFF_DRV_RUNNING)) { props->state = IB_PORT_ACTIVE; props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; } else { props->state = IB_PORT_DOWN; props->phys_state = IB_PORT_PHYS_STATE_DISABLED; } irdma_get_eth_speed_and_width(SPEED_100000, &props->active_speed, &props->active_width); if (rdma_protocol_roce(ibdev, 1)) { props->gid_tbl_len = 32; kc_set_props_ip_gid_caps(props); props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; } else { props->gid_tbl_len = 1; } props->qkey_viol_cntr = 0; props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP; props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size; return 0; } static const char *const irdma_hw_stat_names[] = { /* gen1 - 32-bit */ [IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards", [IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts", [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes", [IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards", [IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts", [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes", [IRDMA_HW_STAT_INDEX_RXVLANERR] = "rxVlanErrors", /* gen1 - 64-bit */ [IRDMA_HW_STAT_INDEX_IP4RXOCTS] = "ip4InOctets", [IRDMA_HW_STAT_INDEX_IP4RXPKTS] = "ip4InPkts", [IRDMA_HW_STAT_INDEX_IP4RXFRAGS] = "ip4InReasmRqd", [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] = "ip4InMcastPkts", [IRDMA_HW_STAT_INDEX_IP4TXOCTS] = "ip4OutOctets", [IRDMA_HW_STAT_INDEX_IP4TXPKTS] = "ip4OutPkts", [IRDMA_HW_STAT_INDEX_IP4TXFRAGS] = "ip4OutSegRqd", [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] = "ip4OutMcastPkts", [IRDMA_HW_STAT_INDEX_IP6RXOCTS] = "ip6InOctets", [IRDMA_HW_STAT_INDEX_IP6RXPKTS] = "ip6InPkts", [IRDMA_HW_STAT_INDEX_IP6RXFRAGS] = "ip6InReasmRqd", [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] = "ip6InMcastPkts", [IRDMA_HW_STAT_INDEX_IP6TXOCTS] = "ip6OutOctets", [IRDMA_HW_STAT_INDEX_IP6TXPKTS] = "ip6OutPkts", [IRDMA_HW_STAT_INDEX_IP6TXFRAGS] = "ip6OutSegRqd", [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] = "ip6OutMcastPkts", [IRDMA_HW_STAT_INDEX_RDMARXRDS] = "InRdmaReads", [IRDMA_HW_STAT_INDEX_RDMARXSNDS] = "InRdmaSends", [IRDMA_HW_STAT_INDEX_RDMARXWRS] = "InRdmaWrites", [IRDMA_HW_STAT_INDEX_RDMATXRDS] = "OutRdmaReads", [IRDMA_HW_STAT_INDEX_RDMATXSNDS] = "OutRdmaSends", [IRDMA_HW_STAT_INDEX_RDMATXWRS] = "OutRdmaWrites", [IRDMA_HW_STAT_INDEX_RDMAVBND] = "RdmaBnd", [IRDMA_HW_STAT_INDEX_RDMAVINV] = "RdmaInv", /* gen2 - 32-bit */ [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = "cnpHandled", [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = "cnpIgnored", [IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = "cnpSent", /* gen2 - 64-bit */ [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] = "ip4InMcastOctets", [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] = "ip4OutMcastOctets", [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] = "ip6InMcastOctets", [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] = "ip6OutMcastOctets", [IRDMA_HW_STAT_INDEX_UDPRXPKTS] = "RxUDP", [IRDMA_HW_STAT_INDEX_UDPTXPKTS] = "TxUDP", [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] = "RxECNMrkd", [IRDMA_HW_STAT_INDEX_TCPRTXSEG] = "RetransSegs", [IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = "InOptErrors", [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = "InProtoErrors", [IRDMA_HW_STAT_INDEX_TCPRXSEGS] = "InSegs", [IRDMA_HW_STAT_INDEX_TCPTXSEG] = "OutSegs", }; /** * irdma_alloc_hw_stats - Allocate a hw stats structure * @ibdev: device pointer from stack * @port_num: port number */ struct rdma_hw_stats * irdma_alloc_hw_stats(struct ib_device *ibdev, u8 port_num) { struct irdma_device *iwdev = to_iwdev(ibdev); struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; int num_counters = dev->hw_attrs.max_stat_idx; unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; return rdma_alloc_hw_stats_struct(irdma_hw_stat_names, num_counters, lifespan); } /** * irdma_get_hw_stats - Populates the rdma_hw_stats structure * @ibdev: device pointer from stack * @stats: stats pointer from stack * @port_num: port number * @index: which hw counter the stack is requesting we update */ int irdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, u8 port_num, int index) { struct irdma_device *iwdev = to_iwdev(ibdev); struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats; if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true); memcpy(&stats->value[0], hw_stats, sizeof(u64)* stats->num_counters); return stats->num_counters; } /** * irdma_query_gid - Query port GID * @ibdev: device pointer from stack * @port: port number * @index: Entry index * @gid: Global ID */ int irdma_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid) { struct irdma_device *iwdev = to_iwdev(ibdev); memset(gid->raw, 0, sizeof(gid->raw)); ether_addr_copy(gid->raw, IF_LLADDR(iwdev->netdev)); return 0; } enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, u8 port_num) { return IB_LINK_LAYER_ETHERNET; } inline enum ib_mtu ib_mtu_int_to_enum(int mtu) { if (mtu >= 4096) return IB_MTU_4096; else if (mtu >= 2048) return IB_MTU_2048; else if (mtu >= 1024) return IB_MTU_1024; else if (mtu >= 512) return IB_MTU_512; else return IB_MTU_256; } inline void kc_set_roce_uverbs_cmd_mask(struct irdma_device *iwdev) { iwdev->ibdev.uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST) | BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH) | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH) | BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST); } inline void kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev) { iwdev->ibdev.uverbs_cmd_mask = BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) | BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE) | BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT) | BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) | BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) | BIT_ULL(IB_USER_VERBS_CMD_REG_MR) | BIT_ULL(IB_USER_VERBS_CMD_REREG_MR) | BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) | BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) | BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ) | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ) | BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP) | BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP) | BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) | BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ) | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) | BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) | BIT_ULL(IB_USER_VERBS_CMD_BIND_MW) | BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) | BIT_ULL(IB_USER_VERBS_CMD_POST_RECV) | BIT_ULL(IB_USER_VERBS_CMD_POST_SEND); iwdev->ibdev.uverbs_ex_cmd_mask = BIT_ULL(IB_USER_VERBS_EX_CMD_MODIFY_QP) | BIT_ULL(IB_USER_VERBS_EX_CMD_QUERY_DEVICE); if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) iwdev->ibdev.uverbs_ex_cmd_mask |= BIT_ULL(IB_USER_VERBS_EX_CMD_CREATE_CQ); } diff --git a/sys/dev/irdma/irdma_main.h b/sys/dev/irdma/irdma_main.h index 180bff09d457..8ce4aaab4552 100644 --- a/sys/dev/irdma/irdma_main.h +++ b/sys/dev/irdma/irdma_main.h @@ -1,602 +1,590 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * * Copyright (c) 2015 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenFabrics.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /*$FreeBSD$*/ #ifndef IRDMA_MAIN_H #define IRDMA_MAIN_H #include #include #include #include #include #include #include #include -#if __FreeBSD_version >= 1400000 -#include -#endif #include #include #include #include #include #include #include #include #include "osdep.h" #include "irdma_defs.h" #include "irdma_hmc.h" #include "irdma_type.h" #include "irdma_ws.h" #include "irdma_protos.h" #include "irdma_pble.h" #include "irdma_cm.h" #include "fbsd_kcompat.h" #include "irdma-abi.h" #include "irdma_verbs.h" #include "irdma_user.h" #include "irdma_puda.h" extern struct list_head irdma_handlers; extern spinlock_t irdma_handler_lock; extern bool irdma_upload_context; #define IRDMA_FW_VER_DEFAULT 2 #define IRDMA_HW_VER 2 #define IRDMA_ARP_ADD 1 #define IRDMA_ARP_DELETE 2 #define IRDMA_ARP_RESOLVE 3 #define IRDMA_MACIP_ADD 1 #define IRDMA_MACIP_DELETE 2 #define IW_CCQ_SIZE (IRDMA_CQP_SW_SQSIZE_2048 + 1) #define IW_CEQ_SIZE 2048 #define IW_AEQ_SIZE 2048 #define RX_BUF_SIZE (1536 + 8) #define IW_REG0_SIZE (4 * 1024) #define IW_TX_TIMEOUT (6 * HZ) #define IW_FIRST_QPN 1 #define IW_SW_CONTEXT_ALIGN 1024 #define MAX_DPC_ITERATIONS 128 #define IRDMA_EVENT_TIMEOUT_MS 5000 #define IRDMA_VCHNL_EVENT_TIMEOUT_MS 10000 #define IRDMA_RST_TIMEOUT_HZ 4 #define IRDMA_NO_QSET 0xffff #define IW_CFG_FPM_QP_COUNT 32768 #define IRDMA_MAX_PAGES_PER_FMR 262144 #define IRDMA_MIN_PAGES_PER_FMR 1 #define IRDMA_CQP_COMPL_RQ_WQE_FLUSHED 2 #define IRDMA_CQP_COMPL_SQ_WQE_FLUSHED 3 #define IRDMA_Q_TYPE_PE_AEQ 0x80 #define IRDMA_Q_INVALID_IDX 0xffff #define IRDMA_REM_ENDPOINT_TRK_QPID 3 #define IRDMA_DRV_OPT_ENA_MPA_VER_0 0x00000001 #define IRDMA_DRV_OPT_DISABLE_MPA_CRC 0x00000002 #define IRDMA_DRV_OPT_DISABLE_FIRST_WRITE 0x00000004 #define IRDMA_DRV_OPT_DISABLE_INTF 0x00000008 #define IRDMA_DRV_OPT_ENA_MSI 0x00000010 #define IRDMA_DRV_OPT_DUAL_LOGICAL_PORT 0x00000020 #define IRDMA_DRV_OPT_NO_INLINE_DATA 0x00000080 #define IRDMA_DRV_OPT_DISABLE_INT_MOD 0x00000100 #define IRDMA_DRV_OPT_DISABLE_VIRT_WQ 0x00000200 #define IRDMA_DRV_OPT_ENA_PAU 0x00000400 #define IRDMA_DRV_OPT_MCAST_LOGPORT_MAP 0x00000800 #define IW_HMC_OBJ_TYPE_NUM ARRAY_SIZE(iw_hmc_obj_types) #define IRDMA_ROCE_CWND_DEFAULT 0x400 #define IRDMA_ROCE_ACKCREDS_DEFAULT 0x1E #define IRDMA_FLUSH_SQ BIT(0) #define IRDMA_FLUSH_RQ BIT(1) #define IRDMA_REFLUSH BIT(2) #define IRDMA_FLUSH_WAIT BIT(3) enum init_completion_state { INVALID_STATE = 0, INITIAL_STATE, CQP_CREATED, HMC_OBJS_CREATED, HW_RSRC_INITIALIZED, CCQ_CREATED, CEQ0_CREATED, /* Last state of probe */ ILQ_CREATED, IEQ_CREATED, REM_ENDPOINT_TRK_CREATED, CEQS_CREATED, PBLE_CHUNK_MEM, AEQ_CREATED, IP_ADDR_REGISTERED, /* Last state of open */ }; struct irdma_rsrc_limits { u32 qplimit; u32 mrlimit; u32 cqlimit; }; struct irdma_cqp_err_info { u16 maj; u16 min; const char *desc; }; struct irdma_cqp_compl_info { u32 op_ret_val; u16 maj_err_code; u16 min_err_code; bool error; u8 op_code; }; struct irdma_cqp_request { struct cqp_cmds_info info; wait_queue_head_t waitq; struct list_head list; atomic_t refcnt; void (*callback_fcn)(struct irdma_cqp_request *cqp_request); void *param; struct irdma_cqp_compl_info compl_info; bool waiting:1; bool request_done:1; bool dynamic:1; }; struct irdma_cqp { struct irdma_sc_cqp sc_cqp; spinlock_t req_lock; /* protect CQP request list */ spinlock_t compl_lock; /* protect CQP completion processing */ wait_queue_head_t waitq; wait_queue_head_t remove_wq; struct irdma_dma_mem sq; struct irdma_dma_mem host_ctx; u64 *scratch_array; struct irdma_cqp_request *cqp_requests; struct list_head cqp_avail_reqs; struct list_head cqp_pending_reqs; }; struct irdma_ccq { struct irdma_sc_cq sc_cq; struct irdma_dma_mem mem_cq; struct irdma_dma_mem shadow_area; }; struct irdma_ceq { struct irdma_sc_ceq sc_ceq; struct irdma_dma_mem mem; u32 irq; u32 msix_idx; struct irdma_pci_f *rf; struct tasklet_struct dpc_tasklet; spinlock_t ce_lock; /* sync cq destroy with cq completion event notification */ }; struct irdma_aeq { struct irdma_sc_aeq sc_aeq; struct irdma_dma_mem mem; struct irdma_pble_alloc palloc; bool virtual_map; }; struct irdma_arp_entry { u32 ip_addr[4]; u8 mac_addr[ETH_ALEN]; }; struct irdma_msix_vector { u32 idx; u32 irq; u32 cpu_affinity; u32 ceq_id; struct resource *res; void *tag; }; struct irdma_mc_table_info { u32 mgn; u32 dest_ip[4]; bool lan_fwd:1; bool ipv4_valid:1; }; struct mc_table_list { struct list_head list; struct irdma_mc_table_info mc_info; struct irdma_mcast_grp_info mc_grp_ctx; }; struct irdma_qv_info { u32 v_idx; /* msix_vector */ u16 ceq_idx; u16 aeq_idx; u8 itr_idx; }; struct irdma_qvlist_info { u32 num_vectors; struct irdma_qv_info qv_info[1]; }; struct irdma_gen_ops { void (*request_reset)(struct irdma_pci_f *rf); int (*register_qset)(struct irdma_sc_vsi *vsi, struct irdma_ws_node *tc_node); void (*unregister_qset)(struct irdma_sc_vsi *vsi, struct irdma_ws_node *tc_node); }; struct irdma_pci_f { bool reset:1; bool rsrc_created:1; bool msix_shared:1; bool ftype:1; u8 rsrc_profile; u8 *hmc_info_mem; u8 *mem_rsrc; u8 rdma_ver; u8 rst_to; /* Not used in SRIOV VF mode */ u8 pf_id; enum irdma_protocol_used protocol_used; bool en_rem_endpoint_trk:1; bool dcqcn_ena:1; u32 sd_type; u32 msix_count; u32 max_mr; u32 max_qp; u32 max_cq; u32 max_ah; u32 next_ah; u32 max_mcg; u32 next_mcg; u32 max_pd; u32 next_qp; u32 next_cq; u32 next_pd; u32 max_mr_size; u32 max_cqe; u32 mr_stagmask; u32 used_pds; u32 used_cqs; u32 used_mrs; u32 used_qps; u32 arp_table_size; u32 next_arp_index; u32 ceqs_count; u32 next_ws_node_id; u32 max_ws_node_id; u32 limits_sel; unsigned long *allocated_ws_nodes; unsigned long *allocated_qps; unsigned long *allocated_cqs; unsigned long *allocated_mrs; unsigned long *allocated_pds; unsigned long *allocated_mcgs; unsigned long *allocated_ahs; unsigned long *allocated_arps; enum init_completion_state init_state; struct irdma_sc_dev sc_dev; struct irdma_dev_ctx dev_ctx; struct irdma_tunable_info tun_info; eventhandler_tag irdma_ifaddr_event; struct irdma_handler *hdl; struct pci_dev *pcidev; struct ice_rdma_peer *peer_info; struct irdma_hw hw; struct irdma_cqp cqp; struct irdma_ccq ccq; struct irdma_aeq aeq; struct irdma_ceq *ceqlist; struct irdma_hmc_pble_rsrc *pble_rsrc; struct irdma_arp_entry *arp_table; spinlock_t arp_lock; /*protect ARP table access*/ spinlock_t rsrc_lock; /* protect HW resource array access */ spinlock_t qptable_lock; /*protect QP table access*/ spinlock_t cqtable_lock; /*protect CQ table access*/ struct irdma_qp **qp_table; struct irdma_cq **cq_table; spinlock_t qh_list_lock; /* protect mc_qht_list */ struct mc_table_list mc_qht_list; struct irdma_msix_vector *iw_msixtbl; struct irdma_qvlist_info *iw_qvlist; struct tasklet_struct dpc_tasklet; struct msix_entry msix_info; struct irdma_dma_mem obj_mem; struct irdma_dma_mem obj_next; atomic_t vchnl_msgs; wait_queue_head_t vchnl_waitq; struct workqueue_struct *cqp_cmpl_wq; struct work_struct cqp_cmpl_work; struct irdma_sc_vsi default_vsi; void *back_fcn; struct irdma_gen_ops gen_ops; void (*check_fc)(struct irdma_sc_vsi *vsi, struct irdma_sc_qp *sc_qp); struct irdma_dcqcn_cc_params dcqcn_params; struct irdma_device *iwdev; }; struct irdma_device { struct ib_device ibdev; struct irdma_pci_f *rf; struct ifnet *netdev; struct notifier_block nb_netdevice_event; struct irdma_handler *hdl; struct workqueue_struct *cleanup_wq; struct irdma_sc_vsi vsi; struct irdma_cm_core cm_core; u32 roce_cwnd; u32 roce_ackcreds; u32 vendor_id; u32 vendor_part_id; u32 push_mode; u32 rcv_wnd; u16 mac_ip_table_idx; u16 vsi_num; u8 rcv_wscale; u8 iw_status; u8 rd_fence_rate; bool override_rcv_wnd:1; bool override_cwnd:1; bool override_ackcreds:1; bool override_ooo:1; bool override_rd_fence_rate:1; bool override_rtomin:1; bool roce_mode:1; bool roce_dcqcn_en:1; bool dcb_vlan_mode:1; bool iw_ooo:1; enum init_completion_state init_state; wait_queue_head_t suspend_wq; }; struct irdma_handler { struct list_head list; struct irdma_device *iwdev; struct task deferred_task; struct taskqueue *deferred_tq; bool shared_res_created; }; static inline struct irdma_device *to_iwdev(struct ib_device *ibdev) { return container_of(ibdev, struct irdma_device, ibdev); } static inline struct irdma_ucontext *to_ucontext(struct ib_ucontext *ibucontext) { return container_of(ibucontext, struct irdma_ucontext, ibucontext); } -#if __FreeBSD_version >= 1400026 -static inline struct irdma_user_mmap_entry * -to_irdma_mmap_entry(struct rdma_user_mmap_entry *rdma_entry) -{ - return container_of(rdma_entry, struct irdma_user_mmap_entry, - rdma_entry); -} - -#endif static inline struct irdma_pd *to_iwpd(struct ib_pd *ibpd) { return container_of(ibpd, struct irdma_pd, ibpd); } static inline struct irdma_ah *to_iwah(struct ib_ah *ibah) { return container_of(ibah, struct irdma_ah, ibah); } static inline struct irdma_mr *to_iwmr(struct ib_mr *ibmr) { return container_of(ibmr, struct irdma_mr, ibmr); } static inline struct irdma_mr *to_iwmw(struct ib_mw *ibmw) { return container_of(ibmw, struct irdma_mr, ibmw); } static inline struct irdma_cq *to_iwcq(struct ib_cq *ibcq) { return container_of(ibcq, struct irdma_cq, ibcq); } static inline struct irdma_qp *to_iwqp(struct ib_qp *ibqp) { return container_of(ibqp, struct irdma_qp, ibqp); } static inline struct irdma_pci_f *dev_to_rf(struct irdma_sc_dev *dev) { return container_of(dev, struct irdma_pci_f, sc_dev); } /** * irdma_alloc_resource - allocate a resource * @iwdev: device pointer * @resource_array: resource bit array: * @max_resources: maximum resource number * @req_resources_num: Allocated resource number * @next: next free id **/ static inline int irdma_alloc_rsrc(struct irdma_pci_f *rf, unsigned long *rsrc_array, u32 max_rsrc, u32 *req_rsrc_num, u32 *next) { u32 rsrc_num; unsigned long flags; spin_lock_irqsave(&rf->rsrc_lock, flags); rsrc_num = find_next_zero_bit(rsrc_array, max_rsrc, *next); if (rsrc_num >= max_rsrc) { rsrc_num = find_first_zero_bit(rsrc_array, max_rsrc); if (rsrc_num >= max_rsrc) { spin_unlock_irqrestore(&rf->rsrc_lock, flags); irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, "resource [%d] allocation failed\n", rsrc_num); return -EOVERFLOW; } } __set_bit(rsrc_num, rsrc_array); *next = rsrc_num + 1; if (*next == max_rsrc) *next = 0; *req_rsrc_num = rsrc_num; spin_unlock_irqrestore(&rf->rsrc_lock, flags); return 0; } /** * irdma_free_resource - free a resource * @iwdev: device pointer * @resource_array: resource array for the resource_num * @resource_num: resource number to free **/ static inline void irdma_free_rsrc(struct irdma_pci_f *rf, unsigned long *rsrc_array, u32 rsrc_num) { unsigned long flags; spin_lock_irqsave(&rf->rsrc_lock, flags); __clear_bit(rsrc_num, rsrc_array); spin_unlock_irqrestore(&rf->rsrc_lock, flags); } int irdma_ctrl_init_hw(struct irdma_pci_f *rf); void irdma_ctrl_deinit_hw(struct irdma_pci_f *rf); int irdma_rt_init_hw(struct irdma_device *iwdev, struct irdma_l2params *l2params); void irdma_rt_deinit_hw(struct irdma_device *iwdev); void irdma_qp_add_ref(struct ib_qp *ibqp); void irdma_qp_rem_ref(struct ib_qp *ibqp); void irdma_free_lsmm_rsrc(struct irdma_qp *iwqp); struct ib_qp *irdma_get_qp(struct ib_device *ibdev, int qpn); void irdma_flush_wqes(struct irdma_qp *iwqp, u32 flush_mask); void irdma_manage_arp_cache(struct irdma_pci_f *rf, const unsigned char *mac_addr, u32 *ip_addr, u32 action); struct irdma_apbvt_entry *irdma_add_apbvt(struct irdma_device *iwdev, u16 port); void irdma_del_apbvt(struct irdma_device *iwdev, struct irdma_apbvt_entry *entry); struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, bool wait); void irdma_free_cqp_request(struct irdma_cqp *cqp, struct irdma_cqp_request *cqp_request); void irdma_put_cqp_request(struct irdma_cqp *cqp, struct irdma_cqp_request *cqp_request); int irdma_alloc_local_mac_entry(struct irdma_pci_f *rf, u16 *mac_tbl_idx); int irdma_add_local_mac_entry(struct irdma_pci_f *rf, u8 *mac_addr, u16 idx); void irdma_del_local_mac_entry(struct irdma_pci_f *rf, u16 idx); u32 irdma_initialize_hw_rsrc(struct irdma_pci_f *rf); void irdma_port_ibevent(struct irdma_device *iwdev); void irdma_cm_disconn(struct irdma_qp *qp); bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd, u16 maj_err_code, u16 min_err_code); int irdma_handle_cqp_op(struct irdma_pci_f *rf, struct irdma_cqp_request *cqp_request); int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); void irdma_cq_add_ref(struct ib_cq *ibcq); void irdma_cq_rem_ref(struct ib_cq *ibcq); void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq); void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf); int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, struct irdma_modify_qp_info *info, bool wait); int irdma_qp_suspend_resume(struct irdma_sc_qp *qp, bool suspend); int irdma_manage_qhash(struct irdma_device *iwdev, struct irdma_cm_info *cminfo, enum irdma_quad_entry_type etype, enum irdma_quad_hash_manage_type mtype, void *cmnode, bool wait); void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf); void irdma_free_sqbuf(struct irdma_sc_vsi *vsi, void *bufp); void irdma_free_qp_rsrc(struct irdma_qp *iwqp); int irdma_setup_cm_core(struct irdma_device *iwdev, u8 ver); void irdma_cleanup_cm_core(struct irdma_cm_core *cm_core); void irdma_next_iw_state(struct irdma_qp *iwqp, u8 state, u8 del_hash, u8 term, u8 term_len); int irdma_send_syn(struct irdma_cm_node *cm_node, u32 sendack); int irdma_send_reset(struct irdma_cm_node *cm_node); struct irdma_cm_node *irdma_find_node(struct irdma_cm_core *cm_core, u16 rem_port, u32 *rem_addr, u16 loc_port, u32 *loc_addr, u16 vlan_id); int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp, struct irdma_qp_flush_info *info, bool wait); void irdma_gen_ae(struct irdma_pci_f *rf, struct irdma_sc_qp *qp, struct irdma_gen_ae_info *info, bool wait); void irdma_copy_ip_ntohl(u32 *dst, __be32 *src); void irdma_copy_ip_htonl(__be32 *dst, u32 *src); u16 irdma_get_vlan_ipv4(u32 *addr); struct ifnet *irdma_netdev_vlan_ipv6(u32 *addr, u16 *vlan_id, u8 *mac); struct ib_mr *irdma_reg_phys_mr(struct ib_pd *ib_pd, u64 addr, u64 size, int acc, u64 *iova_start); int irdma_upload_qp_context(struct irdma_qp *iwqp, bool freeze, bool raw); void irdma_del_hmc_objects(struct irdma_sc_dev *dev, struct irdma_hmc_info *hmc_info, bool privileged, bool reset, enum irdma_vers vers); void irdma_cqp_ce_handler(struct irdma_pci_f *rf, struct irdma_sc_cq *cq); int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, bool wait, void (*callback_fcn)(struct irdma_cqp_request *cqp_request), void *cb_param); void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request); bool irdma_cq_empty(struct irdma_cq *iwcq); int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event, void *ptr); void irdma_unregister_notifiers(struct irdma_device *iwdev); int irdma_register_notifiers(struct irdma_device *iwdev); void irdma_set_rf_user_cfg_params(struct irdma_pci_f *rf); void irdma_add_ip(struct irdma_device *iwdev); void irdma_add_handler(struct irdma_handler *hdl); void irdma_del_handler(struct irdma_handler *hdl); void cqp_compl_worker(struct work_struct *work); #endif /* IRDMA_MAIN_H */ diff --git a/sys/dev/irdma/irdma_verbs.c b/sys/dev/irdma/irdma_verbs.c index dbd20ea72d00..9daadf0706e5 100644 --- a/sys/dev/irdma/irdma_verbs.c +++ b/sys/dev/irdma/irdma_verbs.c @@ -1,3766 +1,3636 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * * Copyright (c) 2015 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenFabrics.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /*$FreeBSD$*/ #include "irdma_main.h" /** * irdma_query_device - get device attributes * @ibdev: device pointer from stack * @props: returning device attributes * @udata: user data */ static int irdma_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct ib_udata *udata) { struct irdma_device *iwdev = to_iwdev(ibdev); struct irdma_pci_f *rf = iwdev->rf; struct pci_dev *pcidev = iwdev->rf->pcidev; struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs; if (udata->inlen || udata->outlen) return -EINVAL; memset(props, 0, sizeof(*props)); ether_addr_copy((u8 *)&props->sys_image_guid, IF_LLADDR(iwdev->netdev)); props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 | irdma_fw_minor_ver(&rf->sc_dev); props->device_cap_flags = IB_DEVICE_MEM_WINDOW | IB_DEVICE_MEM_MGT_EXTENSIONS; props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY; props->vendor_id = pcidev->vendor; props->vendor_part_id = pcidev->device; props->hw_ver = pcidev->revision; props->page_size_cap = hw_attrs->page_size_cap; props->max_mr_size = hw_attrs->max_mr_size; props->max_qp = rf->max_qp - rf->used_qps; props->max_qp_wr = hw_attrs->max_qp_wr; set_max_sge(props, rf); props->max_cq = rf->max_cq - rf->used_cqs; props->max_cqe = rf->max_cqe - 1; props->max_mr = rf->max_mr - rf->used_mrs; props->max_mw = props->max_mr; props->max_pd = rf->max_pd - rf->used_pds; props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges; props->max_qp_rd_atom = hw_attrs->max_hw_ird; props->max_qp_init_rd_atom = hw_attrs->max_hw_ord; if (rdma_protocol_roce(ibdev, 1)) { props->device_cap_flags |= IB_DEVICE_RC_RNR_NAK_GEN; props->max_pkeys = IRDMA_PKEY_TBL_SZ; props->max_ah = rf->max_ah; if (hw_attrs->uk_attrs.hw_rev == IRDMA_GEN_2) { props->max_mcast_grp = rf->max_mcg; props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX; props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX; } } props->max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR; if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_2) props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B; return 0; } static int irdma_mmap_legacy(struct irdma_ucontext *ucontext, struct vm_area_struct *vma) { u64 pfn; if (vma->vm_pgoff || vma->vm_end - vma->vm_start != PAGE_SIZE) return -EINVAL; vma->vm_private_data = ucontext; pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] + pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; -#if __FreeBSD_version >= 1400026 - return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE, - pgprot_noncached(vma->vm_page_prot), NULL); -#else return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE, pgprot_noncached(vma->vm_page_prot)); -#endif } -#if __FreeBSD_version >= 1400026 -static void -irdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) -{ - struct irdma_user_mmap_entry *entry = to_irdma_mmap_entry(rdma_entry); - - kfree(entry); -} - -struct rdma_user_mmap_entry * -irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset, - enum irdma_mmap_flag mmap_flag, u64 *mmap_offset) -{ - struct irdma_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL); - int ret; - - if (!entry) - return NULL; - - entry->bar_offset = bar_offset; - entry->mmap_flag = mmap_flag; - - ret = rdma_user_mmap_entry_insert(&ucontext->ibucontext, - &entry->rdma_entry, PAGE_SIZE); - if (ret) { - kfree(entry); - return NULL; - } - *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); - - return &entry->rdma_entry; -} - -#else static inline bool find_key_in_mmap_tbl(struct irdma_ucontext *ucontext, u64 key) { struct irdma_user_mmap_entry *entry; HASH_FOR_EACH_POSSIBLE(ucontext->mmap_hash_tbl, entry, hlist, key) { if (entry->pgoff_key == key) return true; } return false; } struct irdma_user_mmap_entry * irdma_user_mmap_entry_add_hash(struct irdma_ucontext *ucontext, u64 bar_offset, enum irdma_mmap_flag mmap_flag, u64 *mmap_offset) { struct irdma_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL); unsigned long flags; int retry_cnt = 0; if (!entry) return NULL; entry->bar_offset = bar_offset; entry->mmap_flag = mmap_flag; entry->ucontext = ucontext; do { get_random_bytes(&entry->pgoff_key, sizeof(entry->pgoff_key)); /* The key is a page offset */ entry->pgoff_key >>= PAGE_SHIFT; /* In the event of a collision in the hash table, retry a new key */ spin_lock_irqsave(&ucontext->mmap_tbl_lock, flags); if (!find_key_in_mmap_tbl(ucontext, entry->pgoff_key)) { HASH_ADD(ucontext->mmap_hash_tbl, &entry->hlist, entry->pgoff_key); spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags); goto hash_add_done; } spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags); } while (retry_cnt++ < 10); irdma_debug(iwdev_to_idev(ucontext->iwdev), IRDMA_DEBUG_VERBS, "mmap table add failed: Cannot find a unique key\n"); kfree(entry); return NULL; hash_add_done: /* libc mmap uses a byte offset */ *mmap_offset = entry->pgoff_key << PAGE_SHIFT; return entry; } static struct irdma_user_mmap_entry * irdma_find_user_mmap_entry(struct irdma_ucontext *ucontext, struct vm_area_struct *vma) { struct irdma_user_mmap_entry *entry; unsigned long flags; if (vma->vm_end - vma->vm_start != PAGE_SIZE) return NULL; spin_lock_irqsave(&ucontext->mmap_tbl_lock, flags); HASH_FOR_EACH_POSSIBLE(ucontext->mmap_hash_tbl, entry, hlist, vma->vm_pgoff) { if (entry->pgoff_key == vma->vm_pgoff) { spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags); return entry; } } spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags); return NULL; } void irdma_user_mmap_entry_del_hash(struct irdma_user_mmap_entry *entry) { struct irdma_ucontext *ucontext; unsigned long flags; if (!entry) return; ucontext = entry->ucontext; spin_lock_irqsave(&ucontext->mmap_tbl_lock, flags); HASH_DEL(ucontext->mmap_hash_tbl, &entry->hlist); spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags); kfree(entry); } -#endif /** * irdma_mmap - user memory map * @context: context created during alloc * @vma: kernel info for user memory map */ static int irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) { -#if __FreeBSD_version >= 1400026 - struct rdma_user_mmap_entry *rdma_entry; -#endif struct irdma_user_mmap_entry *entry; struct irdma_ucontext *ucontext; u64 pfn; int ret; ucontext = to_ucontext(context); /* Legacy support for libi40iw with hard-coded mmap key */ if (ucontext->legacy_mode) return irdma_mmap_legacy(ucontext, vma); -#if __FreeBSD_version >= 1400026 - rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma); - if (!rdma_entry) { - irdma_debug(iwdev_to_idev(ucontext->iwdev), IRDMA_DEBUG_VERBS, - "pgoff[0x%lx] does not have valid entry\n", - vma->vm_pgoff); - return -EINVAL; - } - - entry = to_irdma_mmap_entry(rdma_entry); -#else entry = irdma_find_user_mmap_entry(ucontext, vma); if (!entry) { irdma_debug(iwdev_to_idev(ucontext->iwdev), IRDMA_DEBUG_VERBS, "pgoff[0x%lx] does not have valid entry\n", vma->vm_pgoff); return -EINVAL; } -#endif irdma_debug(iwdev_to_idev(ucontext->iwdev), IRDMA_DEBUG_VERBS, "bar_offset [0x%lx] mmap_flag [%d]\n", entry->bar_offset, entry->mmap_flag); pfn = (entry->bar_offset + pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; switch (entry->mmap_flag) { case IRDMA_MMAP_IO_NC: -#if __FreeBSD_version >= 1400026 - ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, - pgprot_noncached(vma->vm_page_prot), - rdma_entry); -#else ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, pgprot_noncached(vma->vm_page_prot)); -#endif break; case IRDMA_MMAP_IO_WC: -#if __FreeBSD_version >= 1400026 - ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, - pgprot_writecombine(vma->vm_page_prot), - rdma_entry); -#else ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, pgprot_writecombine(vma->vm_page_prot)); -#endif break; default: ret = -EINVAL; } if (ret) irdma_debug(iwdev_to_idev(ucontext->iwdev), IRDMA_DEBUG_VERBS, "bar_offset [0x%lx] mmap_flag[%d] err[%d]\n", entry->bar_offset, entry->mmap_flag, ret); -#if __FreeBSD_version >= 1400026 - rdma_user_mmap_entry_put(rdma_entry); -#endif return ret; } /** * irdma_alloc_push_page - allocate a push page for qp * @iwqp: qp pointer */ static void irdma_alloc_push_page(struct irdma_qp *iwqp) { struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; struct irdma_device *iwdev = iwqp->iwdev; struct irdma_sc_qp *qp = &iwqp->sc_qp; int status; cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); if (!cqp_request) return; cqp_info = &cqp_request->info; cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; cqp_info->post_sq = 1; cqp_info->in.u.manage_push_page.info.push_idx = 0; cqp_info->in.u.manage_push_page.info.qs_handle = qp->vsi->qos[qp->user_pri].qs_handle; cqp_info->in.u.manage_push_page.info.free_page = 0; cqp_info->in.u.manage_push_page.info.push_page_type = 0; cqp_info->in.u.manage_push_page.cqp = &iwdev->rf->cqp.sc_cqp; cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; status = irdma_handle_cqp_op(iwdev->rf, cqp_request); if (!status && cqp_request->compl_info.op_ret_val < iwdev->rf->sc_dev.hw_attrs.max_hw_device_pages) { qp->push_idx = cqp_request->compl_info.op_ret_val; qp->push_offset = 0; } irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); } /** * irdma_get_pbl - Retrieve pbl from a list given a virtual * address * @va: user virtual address * @pbl_list: pbl list to search in (QP's or CQ's) */ struct irdma_pbl * irdma_get_pbl(unsigned long va, struct list_head *pbl_list) { struct irdma_pbl *iwpbl; list_for_each_entry(iwpbl, pbl_list, list) { if (iwpbl->user_base == va) { list_del(&iwpbl->list); iwpbl->on_list = false; return iwpbl; } } return NULL; } /** * irdma_clean_cqes - clean cq entries for qp * @iwqp: qp ptr (user or kernel) * @iwcq: cq ptr */ void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq) { struct irdma_cq_uk *ukcq = &iwcq->sc_cq.cq_uk; unsigned long flags; spin_lock_irqsave(&iwcq->lock, flags); irdma_uk_clean_cq(&iwqp->sc_qp.qp_uk, ukcq); spin_unlock_irqrestore(&iwcq->lock, flags); } static u64 irdma_compute_push_wqe_offset(struct irdma_device *iwdev, u32 page_idx){ u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) { /* skip over db page */ bar_off += IRDMA_HW_PAGE_SIZE; /* skip over reserved space */ bar_off += IRDMA_PF_BAR_RSVD; } /* push wqe page */ bar_off += (u64)page_idx * IRDMA_HW_PAGE_SIZE; return bar_off; } void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp) { if (iwqp->push_db_mmap_entry) { -#if __FreeBSD_version >= 1400026 - rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry); -#else irdma_user_mmap_entry_del_hash(iwqp->push_db_mmap_entry); -#endif iwqp->push_db_mmap_entry = NULL; } if (iwqp->push_wqe_mmap_entry) { -#if __FreeBSD_version >= 1400026 - rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); -#else irdma_user_mmap_entry_del_hash(iwqp->push_wqe_mmap_entry); -#endif iwqp->push_wqe_mmap_entry = NULL; } } static int irdma_setup_push_mmap_entries(struct irdma_ucontext *ucontext, struct irdma_qp *iwqp, u64 *push_wqe_mmap_key, u64 *push_db_mmap_key) { struct irdma_device *iwdev = ucontext->iwdev; u64 bar_off; WARN_ON_ONCE(iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev < IRDMA_GEN_2); bar_off = irdma_compute_push_wqe_offset(iwdev, iwqp->sc_qp.push_idx); -#if __FreeBSD_version >= 1400026 - iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext, - bar_off, IRDMA_MMAP_IO_WC, - push_wqe_mmap_key); -#else iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_add_hash(ucontext, bar_off, IRDMA_MMAP_IO_WC, push_wqe_mmap_key); -#endif if (!iwqp->push_wqe_mmap_entry) return -ENOMEM; /* push doorbell page */ bar_off += IRDMA_HW_PAGE_SIZE; -#if __FreeBSD_version >= 1400026 - iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext, - bar_off, IRDMA_MMAP_IO_NC, - push_db_mmap_key); -#else iwqp->push_db_mmap_entry = irdma_user_mmap_entry_add_hash(ucontext, bar_off, IRDMA_MMAP_IO_NC, push_db_mmap_key); -#endif if (!iwqp->push_db_mmap_entry) { -#if __FreeBSD_version >= 1400026 - rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); -#else irdma_user_mmap_entry_del_hash(iwqp->push_wqe_mmap_entry); -#endif return -ENOMEM; } return 0; } /** * irdma_setup_virt_qp - setup for allocation of virtual qp * @iwdev: irdma device * @iwqp: qp ptr * @init_info: initialize info to return */ void irdma_setup_virt_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, struct irdma_qp_init_info *init_info) { struct irdma_pbl *iwpbl = iwqp->iwpbl; struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; iwqp->page = qpmr->sq_page; init_info->shadow_area_pa = qpmr->shadow; if (iwpbl->pbl_allocated) { init_info->virtual_map = true; init_info->sq_pa = qpmr->sq_pbl.idx; init_info->rq_pa = qpmr->rq_pbl.idx; } else { init_info->sq_pa = qpmr->sq_pbl.addr; init_info->rq_pa = qpmr->rq_pbl.addr; } } /** * irdma_setup_umode_qp - setup sq and rq size in user mode qp * @udata: user data * @iwdev: iwarp device * @iwqp: qp ptr (user or kernel) * @info: initialize info to return * @init_attr: Initial QP create attributes */ int irdma_setup_umode_qp(struct ib_udata *udata, struct irdma_device *iwdev, struct irdma_qp *iwqp, struct irdma_qp_init_info *info, struct ib_qp_init_attr *init_attr) { struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; struct irdma_create_qp_req req = {0}; unsigned long flags; int ret; ret = ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen)); if (ret) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "ib_copy_from_data fail\n"); return ret; } iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; iwqp->user_mode = 1; if (req.user_wqe_bufs) { -#if __FreeBSD_version >= 1400026 - struct irdma_ucontext *ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); -#else struct irdma_ucontext *ucontext = to_ucontext(iwqp->iwpd->ibpd.uobject->context); -#endif + info->qp_uk_init_info.legacy_mode = ucontext->legacy_mode; spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, &ucontext->qp_reg_mem_list); spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); if (!iwqp->iwpbl) { ret = -ENODATA; irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "no pbl info\n"); return ret; } } if (ukinfo->abi_ver <= 5) { /** * For ABI version less than 6 passes raw sq and rq * quanta in cap.max_send_wr and cap.max_recv_wr. */ iwqp->max_send_wr = init_attr->cap.max_send_wr; iwqp->max_recv_wr = init_attr->cap.max_recv_wr; ukinfo->sq_size = init_attr->cap.max_send_wr; ukinfo->rq_size = init_attr->cap.max_recv_wr; irdma_uk_calc_shift_wq(ukinfo, &ukinfo->sq_shift, &ukinfo->rq_shift); } else { ret = irdma_uk_calc_depth_shift_sq(ukinfo, &ukinfo->sq_depth, &ukinfo->sq_shift); if (ret) return ret; ret = irdma_uk_calc_depth_shift_rq(ukinfo, &ukinfo->rq_depth, &ukinfo->rq_shift); if (ret) return ret; iwqp->max_send_wr = (ukinfo->sq_depth - IRDMA_SQ_RSVD) >> ukinfo->sq_shift; iwqp->max_recv_wr = (ukinfo->rq_depth - IRDMA_RQ_RSVD) >> ukinfo->rq_shift; ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift; ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift; } irdma_setup_virt_qp(iwdev, iwqp, info); return 0; } /** * irdma_setup_kmode_qp - setup initialization for kernel mode qp * @iwdev: iwarp device * @iwqp: qp ptr (user or kernel) * @info: initialize info to return * @init_attr: Initial QP create attributes */ int irdma_setup_kmode_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, struct irdma_qp_init_info *info, struct ib_qp_init_attr *init_attr) { struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem; u32 size; int status; struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; status = irdma_uk_calc_depth_shift_sq(ukinfo, &ukinfo->sq_depth, &ukinfo->sq_shift); if (status) return status; status = irdma_uk_calc_depth_shift_rq(ukinfo, &ukinfo->rq_depth, &ukinfo->rq_shift); if (status) return status; iwqp->kqp.sq_wrid_mem = kcalloc(ukinfo->sq_depth, sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL); if (!iwqp->kqp.sq_wrid_mem) return -ENOMEM; iwqp->kqp.rq_wrid_mem = kcalloc(ukinfo->rq_depth, sizeof(*iwqp->kqp.rq_wrid_mem), GFP_KERNEL); if (!iwqp->kqp.rq_wrid_mem) { kfree(iwqp->kqp.sq_wrid_mem); iwqp->kqp.sq_wrid_mem = NULL; return -ENOMEM; } iwqp->kqp.sig_trk_mem = kcalloc(ukinfo->sq_depth, sizeof(u32), GFP_KERNEL); memset(iwqp->kqp.sig_trk_mem, 0, ukinfo->sq_depth * sizeof(u32)); if (!iwqp->kqp.sig_trk_mem) { kfree(iwqp->kqp.sq_wrid_mem); iwqp->kqp.sq_wrid_mem = NULL; kfree(iwqp->kqp.rq_wrid_mem); iwqp->kqp.rq_wrid_mem = NULL; return -ENOMEM; } ukinfo->sq_sigwrtrk_array = (void *)iwqp->kqp.sig_trk_mem; ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem; ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem; size = (ukinfo->sq_depth + ukinfo->rq_depth) * IRDMA_QP_WQE_MIN_SIZE; size += (IRDMA_SHADOW_AREA_SIZE << 3); mem->size = size; mem->va = irdma_allocate_dma_mem(&iwdev->rf->hw, mem, mem->size, 256); if (!mem->va) { kfree(iwqp->kqp.sq_wrid_mem); iwqp->kqp.sq_wrid_mem = NULL; kfree(iwqp->kqp.rq_wrid_mem); iwqp->kqp.rq_wrid_mem = NULL; return -ENOMEM; } ukinfo->sq = mem->va; info->sq_pa = mem->pa; ukinfo->rq = &ukinfo->sq[ukinfo->sq_depth]; info->rq_pa = info->sq_pa + (ukinfo->sq_depth * IRDMA_QP_WQE_MIN_SIZE); ukinfo->shadow_area = ukinfo->rq[ukinfo->rq_depth].elem; info->shadow_area_pa = info->rq_pa + (ukinfo->rq_depth * IRDMA_QP_WQE_MIN_SIZE); ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift; ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift; ukinfo->qp_id = iwqp->ibqp.qp_num; iwqp->max_send_wr = (ukinfo->sq_depth - IRDMA_SQ_RSVD) >> ukinfo->sq_shift; iwqp->max_recv_wr = (ukinfo->rq_depth - IRDMA_RQ_RSVD) >> ukinfo->rq_shift; init_attr->cap.max_send_wr = iwqp->max_send_wr; init_attr->cap.max_recv_wr = iwqp->max_recv_wr; return 0; } int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp) { struct irdma_pci_f *rf = iwqp->iwdev->rf; struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; struct irdma_create_qp_info *qp_info; int status; cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); if (!cqp_request) return -ENOMEM; cqp_info = &cqp_request->info; qp_info = &cqp_request->info.in.u.qp_create.info; memset(qp_info, 0, sizeof(*qp_info)); qp_info->mac_valid = true; qp_info->cq_num_valid = true; qp_info->next_iwarp_state = IRDMA_QP_STATE_IDLE; cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; cqp_info->post_sq = 1; cqp_info->in.u.qp_create.qp = &iwqp->sc_qp; cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; status = irdma_handle_cqp_op(rf, cqp_request); irdma_put_cqp_request(&rf->cqp, cqp_request); return status; } void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp, struct irdma_qp_host_ctx_info *ctx_info) { struct irdma_device *iwdev = iwqp->iwdev; struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; struct irdma_roce_offload_info *roce_info; struct irdma_udp_offload_info *udp_info; udp_info = &iwqp->udp_info; udp_info->snd_mss = ib_mtu_enum_to_int(ib_mtu_int_to_enum(iwdev->vsi.mtu)); udp_info->cwnd = iwdev->roce_cwnd; udp_info->rexmit_thresh = 2; udp_info->rnr_nak_thresh = 2; udp_info->src_port = 0xc000; udp_info->dst_port = ROCE_V2_UDP_DPORT; roce_info = &iwqp->roce_info; ether_addr_copy(roce_info->mac_addr, IF_LLADDR(iwdev->netdev)); roce_info->rd_en = true; roce_info->wr_rdresp_en = true; roce_info->bind_en = true; roce_info->dcqcn_en = false; roce_info->rtomin = 5; roce_info->ack_credits = iwdev->roce_ackcreds; roce_info->ird_size = dev->hw_attrs.max_hw_ird; roce_info->ord_size = dev->hw_attrs.max_hw_ord; if (!iwqp->user_mode) { roce_info->priv_mode_en = true; roce_info->fast_reg_en = true; roce_info->udprivcq_en = true; } roce_info->roce_tver = 0; ctx_info->roce_info = &iwqp->roce_info; ctx_info->udp_info = &iwqp->udp_info; irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); } void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp, struct irdma_qp_host_ctx_info *ctx_info) { struct irdma_device *iwdev = iwqp->iwdev; struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; struct irdma_iwarp_offload_info *iwarp_info; iwarp_info = &iwqp->iwarp_info; ether_addr_copy(iwarp_info->mac_addr, IF_LLADDR(iwdev->netdev)); iwarp_info->rd_en = true; iwarp_info->wr_rdresp_en = true; iwarp_info->bind_en = true; iwarp_info->ecn_en = true; iwarp_info->rtomin = 5; if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) iwarp_info->ib_rd_en = true; if (!iwqp->user_mode) { iwarp_info->priv_mode_en = true; iwarp_info->fast_reg_en = true; } iwarp_info->ddp_ver = 1; iwarp_info->rdmap_ver = 1; ctx_info->iwarp_info = &iwqp->iwarp_info; ctx_info->iwarp_info_valid = true; irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); ctx_info->iwarp_info_valid = false; } int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr, struct irdma_device *iwdev) { struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; if (init_attr->create_flags) return -EOPNOTSUPP; if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline || init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags || init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags) return -EINVAL; if (rdma_protocol_roce(&iwdev->ibdev, 1)) { if (init_attr->qp_type != IB_QPT_RC && init_attr->qp_type != IB_QPT_UD && init_attr->qp_type != IB_QPT_GSI) return -EOPNOTSUPP; } else { if (init_attr->qp_type != IB_QPT_RC) return -EOPNOTSUPP; } return 0; } void irdma_sched_qp_flush_work(struct irdma_qp *iwqp) { irdma_qp_add_ref(&iwqp->ibqp); if (mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS))) irdma_qp_rem_ref(&iwqp->ibqp); } void irdma_flush_worker(struct work_struct *work) { struct delayed_work *dwork = to_delayed_work(work); struct irdma_qp *iwqp = container_of(dwork, struct irdma_qp, dwork_flush); irdma_generate_flush_completions(iwqp); /* For the add in irdma_sched_qp_flush_work */ irdma_qp_rem_ref(&iwqp->ibqp); } static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp) { int acc_flags = 0; if (rdma_protocol_roce(iwqp->ibqp.device, 1)) { if (iwqp->roce_info.wr_rdresp_en) { acc_flags |= IB_ACCESS_LOCAL_WRITE; acc_flags |= IB_ACCESS_REMOTE_WRITE; } if (iwqp->roce_info.rd_en) acc_flags |= IB_ACCESS_REMOTE_READ; if (iwqp->roce_info.bind_en) acc_flags |= IB_ACCESS_MW_BIND; } else { if (iwqp->iwarp_info.wr_rdresp_en) { acc_flags |= IB_ACCESS_LOCAL_WRITE; acc_flags |= IB_ACCESS_REMOTE_WRITE; } if (iwqp->iwarp_info.rd_en) acc_flags |= IB_ACCESS_REMOTE_READ; if (iwqp->iwarp_info.bind_en) acc_flags |= IB_ACCESS_MW_BIND; } return acc_flags; } /** * irdma_query_qp - query qp attributes * @ibqp: qp pointer * @attr: attributes pointer * @attr_mask: Not used * @init_attr: qp attributes to return */ static int irdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_qp_init_attr *init_attr) { struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_sc_qp *qp = &iwqp->sc_qp; memset(attr, 0, sizeof(*attr)); memset(init_attr, 0, sizeof(*init_attr)); attr->qp_state = iwqp->ibqp_state; attr->cur_qp_state = iwqp->ibqp_state; attr->cap.max_send_wr = iwqp->max_send_wr; attr->cap.max_recv_wr = iwqp->max_recv_wr; attr->cap.max_inline_data = qp->qp_uk.max_inline_data; attr->cap.max_send_sge = qp->qp_uk.max_sq_frag_cnt; attr->cap.max_recv_sge = qp->qp_uk.max_rq_frag_cnt; attr->qp_access_flags = irdma_get_ib_acc_flags(iwqp); attr->port_num = 1; if (rdma_protocol_roce(ibqp->device, 1)) { attr->path_mtu = ib_mtu_int_to_enum(iwqp->udp_info.snd_mss); attr->qkey = iwqp->roce_info.qkey; attr->rq_psn = iwqp->udp_info.epsn; attr->sq_psn = iwqp->udp_info.psn_nxt; attr->dest_qp_num = iwqp->roce_info.dest_qp; attr->pkey_index = iwqp->roce_info.p_key; attr->retry_cnt = iwqp->udp_info.rexmit_thresh; attr->rnr_retry = iwqp->udp_info.rnr_nak_thresh; attr->max_rd_atomic = iwqp->roce_info.ord_size; attr->max_dest_rd_atomic = iwqp->roce_info.ird_size; } init_attr->event_handler = iwqp->ibqp.event_handler; init_attr->qp_context = iwqp->ibqp.qp_context; init_attr->send_cq = iwqp->ibqp.send_cq; init_attr->recv_cq = iwqp->ibqp.recv_cq; init_attr->cap = attr->cap; return 0; } /** * irdma_modify_qp_roce - modify qp request * @ibqp: qp's pointer for modify * @attr: access attributes * @attr_mask: state mask * @udata: user data */ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { #define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush) #define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) struct irdma_pd *iwpd = to_iwpd(ibqp->pd); struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_device *iwdev = iwqp->iwdev; struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; struct irdma_qp_host_ctx_info *ctx_info; struct irdma_roce_offload_info *roce_info; struct irdma_udp_offload_info *udp_info; struct irdma_modify_qp_info info = {0}; struct irdma_modify_qp_resp uresp = {}; struct irdma_modify_qp_req ureq; unsigned long flags; u8 issue_modify_qp = 0; int ret = 0; ctx_info = &iwqp->ctx_info; roce_info = &iwqp->roce_info; udp_info = &iwqp->udp_info; if (udata) { if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) || (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN)) return -EINVAL; } if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) return -EOPNOTSUPP; if (attr_mask & IB_QP_DEST_QPN) roce_info->dest_qp = attr->dest_qp_num; if (attr_mask & IB_QP_PKEY_INDEX) { ret = irdma_query_pkey(ibqp->device, 0, attr->pkey_index, &roce_info->p_key); if (ret) return ret; } if (attr_mask & IB_QP_QKEY) roce_info->qkey = attr->qkey; if (attr_mask & IB_QP_PATH_MTU) udp_info->snd_mss = ib_mtu_enum_to_int(attr->path_mtu); if (attr_mask & IB_QP_SQ_PSN) { udp_info->psn_nxt = attr->sq_psn; udp_info->lsn = 0xffff; udp_info->psn_una = attr->sq_psn; udp_info->psn_max = attr->sq_psn; } if (attr_mask & IB_QP_RQ_PSN) udp_info->epsn = attr->rq_psn; if (attr_mask & IB_QP_RNR_RETRY) udp_info->rnr_nak_thresh = attr->rnr_retry; if (attr_mask & IB_QP_RETRY_CNT) udp_info->rexmit_thresh = attr->retry_cnt; ctx_info->roce_info->pd_id = iwpd->sc_pd.pd_id; if (attr_mask & IB_QP_AV) { struct irdma_av *av = &iwqp->roce_ah.av; u16 vlan_id = VLAN_N_VID; u32 local_ip[4] = {}; memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah)); if (attr->ah_attr.ah_flags & IB_AH_GRH) { udp_info->ttl = attr->ah_attr.grh.hop_limit; udp_info->flow_label = attr->ah_attr.grh.flow_label; udp_info->tos = attr->ah_attr.grh.traffic_class; udp_info->src_port = kc_rdma_get_udp_sport(udp_info->flow_label, ibqp->qp_num, roce_info->dest_qp); irdma_qp_rem_qos(&iwqp->sc_qp); dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri); if (iwqp->sc_qp.vsi->dscp_mode) ctx_info->user_pri = iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(udp_info->tos)]; else ctx_info->user_pri = rt_tos2priority(udp_info->tos); } ret = kc_irdma_set_roce_cm_info(iwqp, attr, &vlan_id); if (ret) return ret; if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri)) return -ENOMEM; iwqp->sc_qp.user_pri = ctx_info->user_pri; irdma_qp_add_qos(&iwqp->sc_qp); if (vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode) vlan_id = 0; if (vlan_id < VLAN_N_VID) { udp_info->insert_vlan_tag = true; udp_info->vlan_tag = vlan_id | ctx_info->user_pri << VLAN_PRIO_SHIFT; } else { udp_info->insert_vlan_tag = false; } av->attrs = attr->ah_attr; rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid); if (av->sgid_addr.saddr.sa_family == AF_INET6) { __be32 *daddr = av->dgid_addr.saddr_in6.sin6_addr.__u6_addr.__u6_addr32; __be32 *saddr = av->sgid_addr.saddr_in6.sin6_addr.__u6_addr.__u6_addr32; irdma_copy_ip_ntohl(&udp_info->dest_ip_addr[0], daddr); irdma_copy_ip_ntohl(&udp_info->local_ipaddr[0], saddr); udp_info->ipv4 = false; irdma_copy_ip_ntohl(local_ip, daddr); udp_info->arp_idx = irdma_arp_table(iwdev->rf, local_ip, NULL, IRDMA_ARP_RESOLVE); } else { __be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr; __be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr; local_ip[0] = ntohl(daddr); udp_info->ipv4 = true; udp_info->dest_ip_addr[0] = 0; udp_info->dest_ip_addr[1] = 0; udp_info->dest_ip_addr[2] = 0; udp_info->dest_ip_addr[3] = local_ip[0]; udp_info->local_ipaddr[0] = 0; udp_info->local_ipaddr[1] = 0; udp_info->local_ipaddr[2] = 0; udp_info->local_ipaddr[3] = ntohl(saddr); } udp_info->arp_idx = irdma_add_arp(iwdev->rf, local_ip, ah_attr_to_dmac(attr->ah_attr)); } if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { if (attr->max_rd_atomic > dev->hw_attrs.max_hw_ord) { ibdev_err(&iwdev->ibdev, "rd_atomic = %d, above max_hw_ord=%d\n", attr->max_rd_atomic, dev->hw_attrs.max_hw_ord); return -EINVAL; } if (attr->max_rd_atomic) roce_info->ord_size = attr->max_rd_atomic; info.ord_valid = true; } if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { if (attr->max_dest_rd_atomic > dev->hw_attrs.max_hw_ird) { ibdev_err(&iwdev->ibdev, "rd_atomic = %d, above max_hw_ird=%d\n", attr->max_rd_atomic, dev->hw_attrs.max_hw_ird); return -EINVAL; } if (attr->max_dest_rd_atomic) roce_info->ird_size = attr->max_dest_rd_atomic; } if (attr_mask & IB_QP_ACCESS_FLAGS) { if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) roce_info->wr_rdresp_en = true; if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) roce_info->wr_rdresp_en = true; if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) roce_info->rd_en = true; } wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); irdma_debug(dev, IRDMA_DEBUG_VERBS, "caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n", __builtin_return_address(0), ibqp->qp_num, attr->qp_state, iwqp->ibqp_state, iwqp->iwarp_state, attr_mask); spin_lock_irqsave(&iwqp->lock, flags); if (attr_mask & IB_QP_STATE) { if (!kc_ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, iwqp->ibqp.qp_type, attr_mask, IB_LINK_LAYER_ETHERNET)) { irdma_print("modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n", iwqp->ibqp.qp_num, iwqp->ibqp_state, attr->qp_state); ret = -EINVAL; goto exit; } info.curr_iwarp_state = iwqp->iwarp_state; switch (attr->qp_state) { case IB_QPS_INIT: if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { ret = -EINVAL; goto exit; } if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { info.next_iwarp_state = IRDMA_QP_STATE_IDLE; issue_modify_qp = 1; } break; case IB_QPS_RTR: if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { ret = -EINVAL; goto exit; } info.arp_cache_idx_valid = true; info.cq_num_valid = true; info.next_iwarp_state = IRDMA_QP_STATE_RTR; issue_modify_qp = 1; break; case IB_QPS_RTS: if (iwqp->ibqp_state < IB_QPS_RTR || iwqp->ibqp_state == IB_QPS_ERR) { ret = -EINVAL; goto exit; } info.arp_cache_idx_valid = true; info.cq_num_valid = true; info.ord_valid = true; info.next_iwarp_state = IRDMA_QP_STATE_RTS; issue_modify_qp = 1; if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) iwdev->rf->check_fc(&iwdev->vsi, &iwqp->sc_qp); udp_info->cwnd = iwdev->roce_cwnd; roce_info->ack_credits = iwdev->roce_ackcreds; if (iwdev->push_mode && udata && iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { spin_unlock_irqrestore(&iwqp->lock, flags); irdma_alloc_push_page(iwqp); spin_lock_irqsave(&iwqp->lock, flags); } break; case IB_QPS_SQD: if (iwqp->iwarp_state == IRDMA_QP_STATE_SQD) goto exit; if (iwqp->iwarp_state != IRDMA_QP_STATE_RTS) { ret = -EINVAL; goto exit; } info.next_iwarp_state = IRDMA_QP_STATE_SQD; issue_modify_qp = 1; break; case IB_QPS_SQE: case IB_QPS_ERR: case IB_QPS_RESET: if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) { spin_unlock_irqrestore(&iwqp->lock, flags); info.next_iwarp_state = IRDMA_QP_STATE_SQD; irdma_hw_modify_qp(iwdev, iwqp, &info, true); spin_lock_irqsave(&iwqp->lock, flags); } if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { spin_unlock_irqrestore(&iwqp->lock, flags); if (udata && udata->inlen) { if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen))) return -EINVAL; irdma_flush_wqes(iwqp, (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | IRDMA_REFLUSH); } return 0; } info.next_iwarp_state = IRDMA_QP_STATE_ERROR; issue_modify_qp = 1; break; default: ret = -EINVAL; goto exit; } iwqp->ibqp_state = attr->qp_state; } ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); spin_unlock_irqrestore(&iwqp->lock, flags); if (attr_mask & IB_QP_STATE) { if (issue_modify_qp) { ctx_info->rem_endpoint_idx = udp_info->arp_idx; if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) return -EINVAL; spin_lock_irqsave(&iwqp->lock, flags); if (iwqp->iwarp_state == info.curr_iwarp_state) { iwqp->iwarp_state = info.next_iwarp_state; iwqp->ibqp_state = attr->qp_state; } if (iwqp->ibqp_state > IB_QPS_RTS && !iwqp->flush_issued) { spin_unlock_irqrestore(&iwqp->lock, flags); irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | IRDMA_FLUSH_RQ | IRDMA_FLUSH_WAIT); iwqp->flush_issued = 1; } else { spin_unlock_irqrestore(&iwqp->lock, flags); } } else { iwqp->ibqp_state = attr->qp_state; } if (udata && udata->outlen && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { struct irdma_ucontext *ucontext; -#if __FreeBSD_version >= 1400026 - ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); -#else ucontext = to_ucontext(ibqp->uobject->context); -#endif if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && !iwqp->push_wqe_mmap_entry && !irdma_setup_push_mmap_entries(ucontext, iwqp, &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { uresp.push_valid = 1; uresp.push_offset = iwqp->sc_qp.push_offset; } uresp.rd_fence_rate = iwdev->rd_fence_rate; ret = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)); if (ret) { irdma_remove_push_mmap_entries(iwqp); irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "copy_to_udata failed\n"); return ret; } } } return 0; exit: spin_unlock_irqrestore(&iwqp->lock, flags); return ret; } /** * irdma_modify_qp - modify qp request * @ibqp: qp's pointer for modify * @attr: access attributes * @attr_mask: state mask * @udata: user data */ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { #define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush) #define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_device *iwdev = iwqp->iwdev; struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; struct irdma_qp_host_ctx_info *ctx_info; struct irdma_tcp_offload_info *tcp_info; struct irdma_iwarp_offload_info *offload_info; struct irdma_modify_qp_info info = {0}; struct irdma_modify_qp_resp uresp = {}; struct irdma_modify_qp_req ureq = {}; u8 issue_modify_qp = 0; u8 dont_wait = 0; int err; unsigned long flags; if (udata) { if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) || (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN)) return -EINVAL; } if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) return -EOPNOTSUPP; ctx_info = &iwqp->ctx_info; offload_info = &iwqp->iwarp_info; tcp_info = &iwqp->tcp_info; wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); irdma_debug(dev, IRDMA_DEBUG_VERBS, "caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d last_aeq=%d hw_tcp_state=%d hw_iwarp_state=%d attr_mask=0x%x\n", __builtin_return_address(0), ibqp->qp_num, attr->qp_state, iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq, iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask); spin_lock_irqsave(&iwqp->lock, flags); if (attr_mask & IB_QP_STATE) { info.curr_iwarp_state = iwqp->iwarp_state; switch (attr->qp_state) { case IB_QPS_INIT: case IB_QPS_RTR: if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { err = -EINVAL; goto exit; } if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { info.next_iwarp_state = IRDMA_QP_STATE_IDLE; issue_modify_qp = 1; } if (iwdev->push_mode && udata && iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { spin_unlock_irqrestore(&iwqp->lock, flags); irdma_alloc_push_page(iwqp); spin_lock_irqsave(&iwqp->lock, flags); } break; case IB_QPS_RTS: if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS || !iwqp->cm_id) { err = -EINVAL; goto exit; } issue_modify_qp = 1; iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED; iwqp->hte_added = 1; info.next_iwarp_state = IRDMA_QP_STATE_RTS; info.tcp_ctx_valid = true; info.ord_valid = true; info.arp_cache_idx_valid = true; info.cq_num_valid = true; break; case IB_QPS_SQD: if (iwqp->hw_iwarp_state > IRDMA_QP_STATE_RTS) { err = 0; goto exit; } if (iwqp->iwarp_state == IRDMA_QP_STATE_CLOSING || iwqp->iwarp_state < IRDMA_QP_STATE_RTS) { err = 0; goto exit; } if (iwqp->iwarp_state > IRDMA_QP_STATE_CLOSING) { err = -EINVAL; goto exit; } info.next_iwarp_state = IRDMA_QP_STATE_CLOSING; issue_modify_qp = 1; break; case IB_QPS_SQE: if (iwqp->iwarp_state >= IRDMA_QP_STATE_TERMINATE) { err = -EINVAL; goto exit; } info.next_iwarp_state = IRDMA_QP_STATE_TERMINATE; issue_modify_qp = 1; break; case IB_QPS_ERR: case IB_QPS_RESET: if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { spin_unlock_irqrestore(&iwqp->lock, flags); if (udata && udata->inlen) { if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen))) return -EINVAL; irdma_flush_wqes(iwqp, (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | IRDMA_REFLUSH); } return 0; } if (iwqp->sc_qp.term_flags) { spin_unlock_irqrestore(&iwqp->lock, flags); irdma_terminate_del_timer(&iwqp->sc_qp); spin_lock_irqsave(&iwqp->lock, flags); } info.next_iwarp_state = IRDMA_QP_STATE_ERROR; if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED && iwdev->iw_status && iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT) info.reset_tcp_conn = true; else dont_wait = 1; issue_modify_qp = 1; info.next_iwarp_state = IRDMA_QP_STATE_ERROR; break; default: err = -EINVAL; goto exit; } iwqp->ibqp_state = attr->qp_state; } if (attr_mask & IB_QP_ACCESS_FLAGS) { ctx_info->iwarp_info_valid = true; if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) offload_info->wr_rdresp_en = true; if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) offload_info->wr_rdresp_en = true; if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) offload_info->rd_en = true; } if (ctx_info->iwarp_info_valid) { ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); } spin_unlock_irqrestore(&iwqp->lock, flags); if (attr_mask & IB_QP_STATE) { if (issue_modify_qp) { ctx_info->rem_endpoint_idx = tcp_info->arp_idx; if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) return -EINVAL; } spin_lock_irqsave(&iwqp->lock, flags); if (iwqp->iwarp_state == info.curr_iwarp_state) { iwqp->iwarp_state = info.next_iwarp_state; iwqp->ibqp_state = attr->qp_state; } spin_unlock_irqrestore(&iwqp->lock, flags); } if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) { if (dont_wait) { if (iwqp->hw_tcp_state) { spin_lock_irqsave(&iwqp->lock, flags); iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED; iwqp->last_aeq = IRDMA_AE_RESET_SENT; spin_unlock_irqrestore(&iwqp->lock, flags); } irdma_cm_disconn(iwqp); } else { int close_timer_started; spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags); if (iwqp->cm_node) { atomic_inc(&iwqp->cm_node->refcnt); spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); close_timer_started = atomic_inc_return(&iwqp->close_timer_started); if (iwqp->cm_id && close_timer_started == 1) irdma_schedule_cm_timer(iwqp->cm_node, (struct irdma_puda_buf *)iwqp, IRDMA_TIMER_TYPE_CLOSE, 1, 0); irdma_rem_ref_cm_node(iwqp->cm_node); } else { spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); } } } if (attr_mask & IB_QP_STATE && udata && udata->outlen && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { struct irdma_ucontext *ucontext; -#if __FreeBSD_version >= 1400026 - ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); -#else ucontext = to_ucontext(ibqp->uobject->context); -#endif if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && !iwqp->push_wqe_mmap_entry && !irdma_setup_push_mmap_entries(ucontext, iwqp, &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { uresp.push_valid = 1; uresp.push_offset = iwqp->sc_qp.push_offset; } uresp.rd_fence_rate = iwdev->rd_fence_rate; err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)); if (err) { irdma_remove_push_mmap_entries(iwqp); irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "copy_to_udata failed\n"); return err; } } return 0; exit: spin_unlock_irqrestore(&iwqp->lock, flags); return err; } /** * irdma_cq_free_rsrc - free up resources for cq * @rf: RDMA PCI function * @iwcq: cq ptr */ void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq) { struct irdma_sc_cq *cq = &iwcq->sc_cq; if (!iwcq->user_mode) { irdma_free_dma_mem(rf->sc_dev.hw, &iwcq->kmem); irdma_free_dma_mem(rf->sc_dev.hw, &iwcq->kmem_shadow); } irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id); } /** * irdma_free_cqbuf - worker to free a cq buffer * @work: provides access to the cq buffer to free */ static void irdma_free_cqbuf(struct work_struct *work) { struct irdma_cq_buf *cq_buf = container_of(work, struct irdma_cq_buf, work); irdma_free_dma_mem(cq_buf->hw, &cq_buf->kmem_buf); kfree(cq_buf); } /** * irdma_process_resize_list - remove resized cq buffers from the resize_list * @iwcq: cq which owns the resize_list * @iwdev: irdma device * @lcqe_buf: the buffer where the last cqe is received */ int irdma_process_resize_list(struct irdma_cq *iwcq, struct irdma_device *iwdev, struct irdma_cq_buf *lcqe_buf) { struct list_head *tmp_node, *list_node; struct irdma_cq_buf *cq_buf; int cnt = 0; list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { cq_buf = list_entry(list_node, struct irdma_cq_buf, list); if (cq_buf == lcqe_buf) return cnt; list_del(&cq_buf->list); queue_work(iwdev->cleanup_wq, &cq_buf->work); cnt++; } return cnt; } /** * irdma_resize_cq - resize cq * @ibcq: cq to be resized * @entries: desired cq size * @udata: user data */ static int irdma_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) { #define IRDMA_RESIZE_CQ_MIN_REQ_LEN offsetofend(struct irdma_resize_cq_req, user_cq_buffer) struct irdma_cq *iwcq = to_iwcq(ibcq); struct irdma_sc_dev *dev = iwcq->sc_cq.dev; struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; struct irdma_modify_cq_info *m_info; struct irdma_modify_cq_info info = {0}; struct irdma_dma_mem kmem_buf; struct irdma_cq_mr *cqmr_buf; struct irdma_pbl *iwpbl_buf; struct irdma_device *iwdev; struct irdma_pci_f *rf; struct irdma_cq_buf *cq_buf = NULL; unsigned long flags; int ret; iwdev = to_iwdev(ibcq->device); rf = iwdev->rf; if (!(rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE)) return -EOPNOTSUPP; if (udata && udata->inlen < IRDMA_RESIZE_CQ_MIN_REQ_LEN) return -EINVAL; if (entries > rf->max_cqe) return -EINVAL; if (!iwcq->user_mode) { entries++; if (rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) entries *= 2; } info.cq_size = max(entries, 4); if (info.cq_size == iwcq->sc_cq.cq_uk.cq_size - 1) return 0; if (udata) { struct irdma_resize_cq_req req = {}; struct irdma_ucontext *ucontext = -#if __FreeBSD_version >= 1400026 - rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); -#else to_ucontext(ibcq->uobject->context); -#endif /* CQ resize not supported with legacy GEN_1 libi40iw */ if (ucontext->legacy_mode) return -EOPNOTSUPP; if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) return -EINVAL; spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); iwpbl_buf = irdma_get_pbl((unsigned long)req.user_cq_buffer, &ucontext->cq_reg_mem_list); spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); if (!iwpbl_buf) return -ENOMEM; cqmr_buf = &iwpbl_buf->cq_mr; if (iwpbl_buf->pbl_allocated) { info.virtual_map = true; info.pbl_chunk_size = 1; info.first_pm_pbl_idx = cqmr_buf->cq_pbl.idx; } else { info.cq_pa = cqmr_buf->cq_pbl.addr; } } else { /* Kmode CQ resize */ int rsize; rsize = info.cq_size * sizeof(struct irdma_cqe); kmem_buf.size = round_up(rsize, 256); kmem_buf.va = irdma_allocate_dma_mem(dev->hw, &kmem_buf, kmem_buf.size, 256); if (!kmem_buf.va) return -ENOMEM; info.cq_base = kmem_buf.va; info.cq_pa = kmem_buf.pa; cq_buf = kzalloc(sizeof(*cq_buf), GFP_KERNEL); if (!cq_buf) { ret = -ENOMEM; goto error; } } cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); if (!cqp_request) { ret = -ENOMEM; goto error; } info.shadow_read_threshold = iwcq->sc_cq.shadow_read_threshold; info.cq_resize = true; cqp_info = &cqp_request->info; m_info = &cqp_info->in.u.cq_modify.info; memcpy(m_info, &info, sizeof(*m_info)); cqp_info->cqp_cmd = IRDMA_OP_CQ_MODIFY; cqp_info->in.u.cq_modify.cq = &iwcq->sc_cq; cqp_info->in.u.cq_modify.scratch = (uintptr_t)cqp_request; cqp_info->post_sq = 1; ret = irdma_handle_cqp_op(rf, cqp_request); irdma_put_cqp_request(&rf->cqp, cqp_request); if (ret) goto error; spin_lock_irqsave(&iwcq->lock, flags); if (cq_buf) { cq_buf->kmem_buf = iwcq->kmem; cq_buf->hw = dev->hw; memcpy(&cq_buf->cq_uk, &iwcq->sc_cq.cq_uk, sizeof(cq_buf->cq_uk)); INIT_WORK(&cq_buf->work, irdma_free_cqbuf); list_add_tail(&cq_buf->list, &iwcq->resize_list); iwcq->kmem = kmem_buf; } irdma_sc_cq_resize(&iwcq->sc_cq, &info); ibcq->cqe = info.cq_size - 1; spin_unlock_irqrestore(&iwcq->lock, flags); return 0; error: if (!udata) irdma_free_dma_mem(dev->hw, &kmem_buf); kfree(cq_buf); return ret; } /** * irdma_get_mr_access - get hw MR access permissions from IB access flags * @access: IB access flags */ static inline u16 irdma_get_mr_access(int access){ u16 hw_access = 0; hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ? IRDMA_ACCESS_FLAGS_LOCALWRITE : 0; hw_access |= (access & IB_ACCESS_REMOTE_WRITE) ? IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0; hw_access |= (access & IB_ACCESS_REMOTE_READ) ? IRDMA_ACCESS_FLAGS_REMOTEREAD : 0; hw_access |= (access & IB_ACCESS_MW_BIND) ? IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0; hw_access |= (access & IB_ZERO_BASED) ? IRDMA_ACCESS_FLAGS_ZERO_BASED : 0; hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD; return hw_access; } /** * irdma_free_stag - free stag resource * @iwdev: irdma device * @stag: stag to free */ void irdma_free_stag(struct irdma_device *iwdev, u32 stag) { u32 stag_idx; stag_idx = (stag & iwdev->rf->mr_stagmask) >> IRDMA_CQPSQ_STAG_IDX_S; irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, stag_idx); } /** * irdma_create_stag - create random stag * @iwdev: irdma device */ u32 irdma_create_stag(struct irdma_device *iwdev) { u32 stag; u32 stag_index = 0; u32 next_stag_index; u32 driver_key; u32 random; u8 consumer_key; int ret; get_random_bytes(&random, sizeof(random)); consumer_key = (u8)random; driver_key = random & ~iwdev->rf->mr_stagmask; next_stag_index = (random & iwdev->rf->mr_stagmask) >> 8; next_stag_index %= iwdev->rf->max_mr; ret = irdma_alloc_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, iwdev->rf->max_mr, &stag_index, &next_stag_index); if (ret) return 0; stag = stag_index << IRDMA_CQPSQ_STAG_IDX_S; stag |= driver_key; stag += (u32)consumer_key; return stag; } /** * irdma_check_mem_contiguous - check if pbls stored in arr are contiguous * @arr: lvl1 pbl array * @npages: page count * @pg_size: page size * */ static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size) { u32 pg_idx; for (pg_idx = 0; pg_idx < npages; pg_idx++) { if ((*arr + (pg_size * pg_idx)) != arr[pg_idx]) return false; } return true; } /** * irdma_check_mr_contiguous - check if MR is physically contiguous * @palloc: pbl allocation struct * @pg_size: page size */ static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc, u32 pg_size) { struct irdma_pble_level2 *lvl2 = &palloc->level2; struct irdma_pble_info *leaf = lvl2->leaf; u64 *arr = NULL; u64 *start_addr = NULL; int i; bool ret; if (palloc->level == PBLE_LEVEL_1) { arr = palloc->level1.addr; ret = irdma_check_mem_contiguous(arr, palloc->total_cnt, pg_size); return ret; } start_addr = leaf->addr; for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { arr = leaf->addr; if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr) return false; ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size); if (!ret) return false; } return true; } /** * irdma_setup_pbles - copy user pg address to pble's * @rf: RDMA PCI function * @iwmr: mr pointer for this memory registration * @use_pbles: flag if to use pble's * @lvl_1_only: request only level 1 pble if true */ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, bool use_pbles, bool lvl_1_only) { struct irdma_pbl *iwpbl = &iwmr->iwpbl; struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; struct irdma_pble_info *pinfo; u64 *pbl; int status; enum irdma_pble_level level = PBLE_LEVEL_1; if (use_pbles) { status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, lvl_1_only); if (status) return status; iwpbl->pbl_allocated = true; level = palloc->level; pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 : palloc->level2.leaf; pbl = pinfo->addr; } else { pbl = iwmr->pgaddrmem; } irdma_copy_user_pgaddrs(iwmr, pbl, level); if (use_pbles) iwmr->pgaddrmem[0] = *pbl; return 0; } /** * irdma_handle_q_mem - handle memory for qp and cq * @iwdev: irdma device * @req: information for q memory management * @iwpbl: pble struct * @use_pbles: flag to use pble */ static int irdma_handle_q_mem(struct irdma_device *iwdev, struct irdma_mem_reg_req *req, struct irdma_pbl *iwpbl, bool use_pbles) { struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; struct irdma_mr *iwmr = iwpbl->iwmr; struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; struct irdma_cq_mr *cqmr = &iwpbl->cq_mr; struct irdma_hmc_pble *hmc_p; u64 *arr = iwmr->pgaddrmem; u32 pg_size, total; int err = 0; bool ret = true; pg_size = iwmr->page_size; err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, true); if (err) return err; if (use_pbles) arr = palloc->level1.addr; switch (iwmr->type) { case IRDMA_MEMREG_TYPE_QP: total = req->sq_pages + req->rq_pages; hmc_p = &qpmr->sq_pbl; qpmr->shadow = (dma_addr_t) arr[total]; if (use_pbles) { ret = irdma_check_mem_contiguous(arr, req->sq_pages, pg_size); if (ret) ret = irdma_check_mem_contiguous(&arr[req->sq_pages], req->rq_pages, pg_size); } if (!ret) { hmc_p->idx = palloc->level1.idx; hmc_p = &qpmr->rq_pbl; hmc_p->idx = palloc->level1.idx + req->sq_pages; } else { hmc_p->addr = arr[0]; hmc_p = &qpmr->rq_pbl; hmc_p->addr = arr[req->sq_pages]; } break; case IRDMA_MEMREG_TYPE_CQ: hmc_p = &cqmr->cq_pbl; if (!cqmr->split) cqmr->shadow = (dma_addr_t) arr[req->cq_pages]; if (use_pbles) ret = irdma_check_mem_contiguous(arr, req->cq_pages, pg_size); if (!ret) hmc_p->idx = palloc->level1.idx; else hmc_p->addr = arr[0]; break; default: irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "MR type error\n"); err = -EINVAL; } if (use_pbles && ret) { irdma_free_pble(iwdev->rf->pble_rsrc, palloc); iwpbl->pbl_allocated = false; } return err; } /** * irdma_hw_alloc_mw - create the hw memory window * @iwdev: irdma device * @iwmr: pointer to memory window info */ int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr) { struct irdma_mw_alloc_info *info; struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; int status; cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); if (!cqp_request) return -ENOMEM; cqp_info = &cqp_request->info; info = &cqp_info->in.u.mw_alloc.info; memset(info, 0, sizeof(*info)); if (iwmr->ibmw.type == IB_MW_TYPE_1) info->mw_wide = true; info->page_size = PAGE_SIZE; info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; info->pd_id = iwpd->sc_pd.pd_id; info->remote_access = true; cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC; cqp_info->post_sq = 1; cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev; cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request; status = irdma_handle_cqp_op(iwdev->rf, cqp_request); irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); return status; } /** * irdma_dealloc_mw - Dealloc memory window * @ibmw: memory window structure. */ static int irdma_dealloc_mw(struct ib_mw *ibmw) { struct ib_pd *ibpd = ibmw->pd; struct irdma_pd *iwpd = to_iwpd(ibpd); struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw); struct irdma_device *iwdev = to_iwdev(ibmw->device); struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; struct irdma_dealloc_stag_info *info; cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); if (!cqp_request) return -ENOMEM; cqp_info = &cqp_request->info; info = &cqp_info->in.u.dealloc_stag.info; memset(info, 0, sizeof(*info)); info->pd_id = iwpd->sc_pd.pd_id; info->stag_idx = RS_64_1(ibmw->rkey, IRDMA_CQPSQ_STAG_IDX_S); info->mr = false; cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; cqp_info->post_sq = 1; cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; irdma_handle_cqp_op(iwdev->rf, cqp_request); irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); irdma_free_stag(iwdev, iwmr->stag); kfree(iwmr); return 0; } /** * irdma_hw_alloc_stag - cqp command to allocate stag * @iwdev: irdma device * @iwmr: irdma mr pointer */ int irdma_hw_alloc_stag(struct irdma_device *iwdev, struct irdma_mr *iwmr) { struct irdma_allocate_stag_info *info; struct ib_pd *pd = iwmr->ibmr.pd; struct irdma_pd *iwpd = to_iwpd(pd); struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; int status; cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); if (!cqp_request) return -ENOMEM; cqp_info = &cqp_request->info; info = &cqp_info->in.u.alloc_stag.info; memset(info, 0, sizeof(*info)); info->page_size = PAGE_SIZE; info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; info->pd_id = iwpd->sc_pd.pd_id; info->total_len = iwmr->len; info->all_memory = (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) ? true : false; info->remote_access = true; cqp_info->cqp_cmd = IRDMA_OP_ALLOC_STAG; cqp_info->post_sq = 1; cqp_info->in.u.alloc_stag.dev = &iwdev->rf->sc_dev; cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request; status = irdma_handle_cqp_op(iwdev->rf, cqp_request); irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); if (!status) iwmr->is_hwreg = 1; return status; } /** * irdma_set_page - populate pbl list for fmr * @ibmr: ib mem to access iwarp mr pointer * @addr: page dma address fro pbl list */ static int irdma_set_page(struct ib_mr *ibmr, u64 addr) { struct irdma_mr *iwmr = to_iwmr(ibmr); struct irdma_pbl *iwpbl = &iwmr->iwpbl; struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; u64 *pbl; if (unlikely(iwmr->npages == iwmr->page_cnt)) return -ENOMEM; if (palloc->level == PBLE_LEVEL_2) { struct irdma_pble_info *palloc_info = palloc->level2.leaf + (iwmr->npages >> PBLE_512_SHIFT); palloc_info->addr[iwmr->npages & (PBLE_PER_PAGE - 1)] = addr; } else { pbl = palloc->level1.addr; pbl[iwmr->npages] = addr; } iwmr->npages++; return 0; } /** * irdma_map_mr_sg - map of sg list for fmr * @ibmr: ib mem to access iwarp mr pointer * @sg: scatter gather list * @sg_nents: number of sg pages * @sg_offset: scatter gather list for fmr */ static int irdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset) { struct irdma_mr *iwmr = to_iwmr(ibmr); iwmr->npages = 0; return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, irdma_set_page); } /** * irdma_hwreg_mr - send cqp command for memory registration * @iwdev: irdma device * @iwmr: irdma mr pointer * @access: access for MR */ int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, u16 access) { struct irdma_pbl *iwpbl = &iwmr->iwpbl; struct irdma_reg_ns_stag_info *stag_info; struct ib_pd *pd = iwmr->ibmr.pd; struct irdma_pd *iwpd = to_iwpd(pd); struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; int ret; cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); if (!cqp_request) return -ENOMEM; cqp_info = &cqp_request->info; stag_info = &cqp_info->in.u.mr_reg_non_shared.info; memset(stag_info, 0, sizeof(*stag_info)); stag_info->va = iwpbl->user_base; stag_info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; stag_info->stag_key = (u8)iwmr->stag; stag_info->total_len = iwmr->len; stag_info->all_memory = (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) ? true : false; stag_info->access_rights = irdma_get_mr_access(access); stag_info->pd_id = iwpd->sc_pd.pd_id; if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED) stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED; else stag_info->addr_type = IRDMA_ADDR_TYPE_VA_BASED; stag_info->page_size = iwmr->page_size; if (iwpbl->pbl_allocated) { if (palloc->level == PBLE_LEVEL_1) { stag_info->first_pm_pbl_index = palloc->level1.idx; stag_info->chunk_size = 1; } else { stag_info->first_pm_pbl_index = palloc->level2.root.idx; stag_info->chunk_size = 3; } } else { stag_info->reg_addr_pa = iwmr->pgaddrmem[0]; } cqp_info->cqp_cmd = IRDMA_OP_MR_REG_NON_SHARED; cqp_info->post_sq = 1; cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev; cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request; ret = irdma_handle_cqp_op(iwdev->rf, cqp_request); irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); if (!ret) iwmr->is_hwreg = 1; return ret; } /** * irdma_reg_user_mr - Register a user memory region * @pd: ptr of pd * @start: virtual start address * @len: length of mr * @virt: virtual address * @access: access of mr * @udata: user data */ static struct ib_mr * irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, u64 virt, int access, struct ib_udata *udata) { #define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages) struct irdma_device *iwdev = to_iwdev(pd->device); struct irdma_ucontext *ucontext; struct irdma_pble_alloc *palloc; struct irdma_pbl *iwpbl; struct irdma_mr *iwmr; struct ib_umem *region; struct irdma_mem_reg_req req = {}; u32 total, stag = 0; u8 shadow_pgcnt = 1; bool use_pbles = false; unsigned long flags; int err = -EINVAL; int ret; if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) return ERR_PTR(-EINVAL); if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN) return ERR_PTR(-EINVAL); region = ib_umem_get(pd->uobject->context, start, len, access, 0); if (IS_ERR(region)) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "Failed to create ib_umem region\n"); return (struct ib_mr *)region; } if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) { ib_umem_release(region); return ERR_PTR(-EFAULT); } iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); if (!iwmr) { ib_umem_release(region); return ERR_PTR(-ENOMEM); } iwpbl = &iwmr->iwpbl; iwpbl->iwmr = iwmr; iwmr->region = region; iwmr->ibmr.pd = pd; iwmr->ibmr.device = pd->device; iwmr->ibmr.iova = virt; iwmr->page_size = IRDMA_HW_PAGE_SIZE; iwmr->page_msk = ~(IRDMA_HW_PAGE_SIZE - 1); iwmr->len = region->length; iwpbl->user_base = virt; palloc = &iwpbl->pble_alloc; iwmr->type = req.reg_type; iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size, virt); switch (req.reg_type) { case IRDMA_MEMREG_TYPE_QP: total = req.sq_pages + req.rq_pages + shadow_pgcnt; if (total > iwmr->page_cnt) { err = -EINVAL; goto error; } total = req.sq_pages + req.rq_pages; use_pbles = (total > 2); err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); if (err) goto error; -#if __FreeBSD_version >= 1400026 - ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); -#else ucontext = to_ucontext(pd->uobject->context); -#endif spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); iwpbl->on_list = true; spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); break; case IRDMA_MEMREG_TYPE_CQ: if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) shadow_pgcnt = 0; total = req.cq_pages + shadow_pgcnt; if (total > iwmr->page_cnt) { err = -EINVAL; goto error; } use_pbles = (req.cq_pages > 1); err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); if (err) goto error; -#if __FreeBSD_version >= 1400026 - ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); -#else ucontext = to_ucontext(pd->uobject->context); -#endif spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); iwpbl->on_list = true; spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); break; case IRDMA_MEMREG_TYPE_MEM: use_pbles = (iwmr->page_cnt != 1); err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false); if (err) goto error; if (use_pbles) { ret = irdma_check_mr_contiguous(palloc, iwmr->page_size); if (ret) { irdma_free_pble(iwdev->rf->pble_rsrc, palloc); iwpbl->pbl_allocated = false; } } stag = irdma_create_stag(iwdev); if (!stag) { err = -ENOMEM; goto error; } iwmr->stag = stag; iwmr->ibmr.rkey = stag; iwmr->ibmr.lkey = stag; iwmr->access = access; err = irdma_hwreg_mr(iwdev, iwmr, access); if (err) { irdma_free_stag(iwdev, stag); goto error; } break; default: goto error; } iwmr->type = req.reg_type; return &iwmr->ibmr; error: if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) irdma_free_pble(iwdev->rf->pble_rsrc, palloc); ib_umem_release(region); kfree(iwmr); return ERR_PTR(err); } int irdma_hwdereg_mr(struct ib_mr *ib_mr) { struct irdma_device *iwdev = to_iwdev(ib_mr->device); struct irdma_mr *iwmr = to_iwmr(ib_mr); struct irdma_pd *iwpd = to_iwpd(ib_mr->pd); struct irdma_dealloc_stag_info *info; struct irdma_pbl *iwpbl = &iwmr->iwpbl; struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; int status; /* * Skip HW MR de-register when it is already de-registered during an MR re-reregister and the re-registration * fails */ if (!iwmr->is_hwreg) return 0; cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); if (!cqp_request) return -ENOMEM; cqp_info = &cqp_request->info; info = &cqp_info->in.u.dealloc_stag.info; memset(info, 0, sizeof(*info)); info->pd_id = iwpd->sc_pd.pd_id; info->stag_idx = RS_64_1(ib_mr->rkey, IRDMA_CQPSQ_STAG_IDX_S); info->mr = true; if (iwpbl->pbl_allocated) info->dealloc_pbl = true; cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; cqp_info->post_sq = 1; cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; status = irdma_handle_cqp_op(iwdev->rf, cqp_request); irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); if (!status) iwmr->is_hwreg = 0; return status; } /* * irdma_rereg_mr_trans - Re-register a user MR for a change translation. @iwmr: ptr of iwmr @start: virtual start * address @len: length of mr @virt: virtual address * * Re-register a user memory region when a change translation is requested. Re-register a new region while reusing the * stag from the original registration. */ struct ib_mr * irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, u64 virt, struct ib_udata *udata) { struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); struct irdma_pbl *iwpbl = &iwmr->iwpbl; struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; struct ib_pd *pd = iwmr->ibmr.pd; struct ib_umem *region; bool use_pbles; int err; region = ib_umem_get(pd->uobject->context, start, len, iwmr->access, 0); if (IS_ERR(region)) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "Failed to create ib_umem region\n"); return (struct ib_mr *)region; } iwmr->region = region; iwmr->ibmr.iova = virt; iwmr->ibmr.pd = pd; iwmr->page_size = PAGE_SIZE; iwmr->len = region->length; iwpbl->user_base = virt; iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size, virt); use_pbles = (iwmr->page_cnt != 1); err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false); if (err) goto error; if (use_pbles) { err = irdma_check_mr_contiguous(palloc, iwmr->page_size); if (err) { irdma_free_pble(iwdev->rf->pble_rsrc, palloc); iwpbl->pbl_allocated = false; } } err = irdma_hwreg_mr(iwdev, iwmr, iwmr->access); if (err) goto error; return &iwmr->ibmr; error: if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) { irdma_free_pble(iwdev->rf->pble_rsrc, palloc); iwpbl->pbl_allocated = false; } ib_umem_release(region); iwmr->region = NULL; return ERR_PTR(err); } /** * irdma_reg_phys_mr - register kernel physical memory * @pd: ibpd pointer * @addr: physical address of memory to register * @size: size of memory to register * @access: Access rights * @iova_start: start of virtual address for physical buffers */ struct ib_mr * irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access, u64 *iova_start) { struct irdma_device *iwdev = to_iwdev(pd->device); struct irdma_pbl *iwpbl; struct irdma_mr *iwmr; u32 stag; int ret; iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); if (!iwmr) return ERR_PTR(-ENOMEM); iwmr->ibmr.pd = pd; iwmr->ibmr.device = pd->device; iwpbl = &iwmr->iwpbl; iwpbl->iwmr = iwmr; iwmr->type = IRDMA_MEMREG_TYPE_MEM; iwpbl->user_base = *iova_start; stag = irdma_create_stag(iwdev); if (!stag) { ret = -ENOMEM; goto err; } iwmr->stag = stag; iwmr->ibmr.iova = *iova_start; iwmr->ibmr.rkey = stag; iwmr->ibmr.lkey = stag; iwmr->page_cnt = 1; iwmr->pgaddrmem[0] = addr; iwmr->len = size; iwmr->page_size = SZ_4K; ret = irdma_hwreg_mr(iwdev, iwmr, access); if (ret) { irdma_free_stag(iwdev, stag); goto err; } return &iwmr->ibmr; err: kfree(iwmr); return ERR_PTR(ret); } /** * irdma_get_dma_mr - register physical mem * @pd: ptr of pd * @acc: access for memory */ static struct ib_mr * irdma_get_dma_mr(struct ib_pd *pd, int acc) { u64 kva = 0; return irdma_reg_phys_mr(pd, 0, 0, acc, &kva); } /** * irdma_del_memlist - Deleting pbl list entries for CQ/QP * @iwmr: iwmr for IB's user page addresses * @ucontext: ptr to user context */ void irdma_del_memlist(struct irdma_mr *iwmr, struct irdma_ucontext *ucontext) { struct irdma_pbl *iwpbl = &iwmr->iwpbl; unsigned long flags; switch (iwmr->type) { case IRDMA_MEMREG_TYPE_CQ: spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); if (iwpbl->on_list) { iwpbl->on_list = false; list_del(&iwpbl->list); } spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); break; case IRDMA_MEMREG_TYPE_QP: spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); if (iwpbl->on_list) { iwpbl->on_list = false; list_del(&iwpbl->list); } spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); break; default: break; } } /** * irdma_copy_sg_list - copy sg list for qp * @sg_list: copied into sg_list * @sgl: copy from sgl * @num_sges: count of sg entries */ static void irdma_copy_sg_list(struct irdma_sge *sg_list, struct ib_sge *sgl, int num_sges) { unsigned int i; for (i = 0; i < num_sges; i++) { sg_list[i].tag_off = sgl[i].addr; sg_list[i].len = sgl[i].length; sg_list[i].stag = sgl[i].lkey; } } /** * irdma_post_send - kernel application wr * @ibqp: qp ptr for wr * @ib_wr: work request ptr * @bad_wr: return of bad wr if err */ static int irdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *ib_wr, const struct ib_send_wr **bad_wr) { struct irdma_qp *iwqp; struct irdma_qp_uk *ukqp; struct irdma_sc_dev *dev; struct irdma_post_sq_info info; int err = 0; unsigned long flags; bool inv_stag; struct irdma_ah *ah; iwqp = to_iwqp(ibqp); ukqp = &iwqp->sc_qp.qp_uk; dev = &iwqp->iwdev->rf->sc_dev; spin_lock_irqsave(&iwqp->lock, flags); while (ib_wr) { memset(&info, 0, sizeof(info)); inv_stag = false; info.wr_id = (ib_wr->wr_id); if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all) info.signaled = true; if (ib_wr->send_flags & IB_SEND_FENCE) info.read_fence = true; switch (ib_wr->opcode) { case IB_WR_SEND_WITH_IMM: if (ukqp->qp_caps & IRDMA_SEND_WITH_IMM) { info.imm_data_valid = true; info.imm_data = ntohl(ib_wr->ex.imm_data); } else { err = -EINVAL; break; } /* fallthrough */ case IB_WR_SEND: case IB_WR_SEND_WITH_INV: if (ib_wr->opcode == IB_WR_SEND || ib_wr->opcode == IB_WR_SEND_WITH_IMM) { if (ib_wr->send_flags & IB_SEND_SOLICITED) info.op_type = IRDMA_OP_TYPE_SEND_SOL; else info.op_type = IRDMA_OP_TYPE_SEND; } else { if (ib_wr->send_flags & IB_SEND_SOLICITED) info.op_type = IRDMA_OP_TYPE_SEND_SOL_INV; else info.op_type = IRDMA_OP_TYPE_SEND_INV; info.stag_to_inv = ib_wr->ex.invalidate_rkey; } info.op.send.num_sges = ib_wr->num_sge; info.op.send.sg_list = (struct irdma_sge *)ib_wr->sg_list; if (iwqp->ibqp.qp_type == IB_QPT_UD || iwqp->ibqp.qp_type == IB_QPT_GSI) { ah = to_iwah(ud_wr(ib_wr)->ah); info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx; info.op.send.qkey = ud_wr(ib_wr)->remote_qkey; info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn; } if (ib_wr->send_flags & IB_SEND_INLINE) err = irdma_uk_inline_send(ukqp, &info, false); else err = irdma_uk_send(ukqp, &info, false); break; case IB_WR_RDMA_WRITE_WITH_IMM: if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) { info.imm_data_valid = true; info.imm_data = ntohl(ib_wr->ex.imm_data); } else { err = -EINVAL; break; } /* fallthrough */ case IB_WR_RDMA_WRITE: if (ib_wr->send_flags & IB_SEND_SOLICITED) info.op_type = IRDMA_OP_TYPE_RDMA_WRITE_SOL; else info.op_type = IRDMA_OP_TYPE_RDMA_WRITE; info.op.rdma_write.num_lo_sges = ib_wr->num_sge; info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list; info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey; if (ib_wr->send_flags & IB_SEND_INLINE) err = irdma_uk_inline_rdma_write(ukqp, &info, false); else err = irdma_uk_rdma_write(ukqp, &info, false); break; case IB_WR_RDMA_READ_WITH_INV: inv_stag = true; /* fallthrough */ case IB_WR_RDMA_READ: if (ib_wr->num_sge > dev->hw_attrs.uk_attrs.max_hw_read_sges) { err = -EINVAL; break; } info.op_type = IRDMA_OP_TYPE_RDMA_READ; info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey; info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list; info.op.rdma_read.num_lo_sges = ib_wr->num_sge; err = irdma_uk_rdma_read(ukqp, &info, inv_stag, false); break; case IB_WR_LOCAL_INV: info.op_type = IRDMA_OP_TYPE_INV_STAG; info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey; err = irdma_uk_stag_local_invalidate(ukqp, &info, true); break; case IB_WR_REG_MR:{ struct irdma_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr); struct irdma_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc; struct irdma_fast_reg_stag_info stag_info = {0}; stag_info.signaled = info.signaled; stag_info.read_fence = info.read_fence; stag_info.access_rights = irdma_get_mr_access(reg_wr(ib_wr)->access); stag_info.stag_key = reg_wr(ib_wr)->key & 0xff; stag_info.stag_idx = reg_wr(ib_wr)->key >> 8; stag_info.page_size = reg_wr(ib_wr)->mr->page_size; stag_info.wr_id = ib_wr->wr_id; stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED; stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova; stag_info.total_len = iwmr->ibmr.length; if (palloc->level == PBLE_LEVEL_2) { stag_info.chunk_size = 3; stag_info.first_pm_pbl_index = palloc->level2.root.idx; } else { stag_info.chunk_size = 1; stag_info.first_pm_pbl_index = palloc->level1.idx; } stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE; err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info, true); break; } default: err = -EINVAL; irdma_debug(iwdev_to_idev(iwqp->iwdev), IRDMA_DEBUG_VERBS, "upost_send bad opcode = 0x%x\n", ib_wr->opcode); break; } if (err) break; ib_wr = ib_wr->next; } if (!iwqp->flush_issued) { if (iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS) irdma_uk_qp_post_wr(ukqp); spin_unlock_irqrestore(&iwqp->lock, flags); } else { spin_unlock_irqrestore(&iwqp->lock, flags); irdma_sched_qp_flush_work(iwqp); } if (err) *bad_wr = ib_wr; return err; } /** * irdma_post_recv - post receive wr for kernel application * @ibqp: ib qp pointer * @ib_wr: work request for receive * @bad_wr: bad wr caused an error */ static int irdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *ib_wr, const struct ib_recv_wr **bad_wr) { struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_qp_uk *ukqp = &iwqp->sc_qp.qp_uk; struct irdma_post_rq_info post_recv = {0}; struct irdma_sge *sg_list = iwqp->sg_list; unsigned long flags; int err = 0; spin_lock_irqsave(&iwqp->lock, flags); while (ib_wr) { if (ib_wr->num_sge > ukqp->max_rq_frag_cnt) { err = -EINVAL; goto out; } post_recv.num_sges = ib_wr->num_sge; post_recv.wr_id = ib_wr->wr_id; irdma_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge); post_recv.sg_list = sg_list; err = irdma_uk_post_receive(ukqp, &post_recv); if (err) { irdma_debug(iwdev_to_idev(iwqp->iwdev), IRDMA_DEBUG_VERBS, "post_recv err %d\n", err); goto out; } ib_wr = ib_wr->next; } out: spin_unlock_irqrestore(&iwqp->lock, flags); if (iwqp->flush_issued) irdma_sched_qp_flush_work(iwqp); if (err) *bad_wr = ib_wr; return err; } /** * irdma_flush_err_to_ib_wc_status - return change flush error code to IB status * @opcode: iwarp flush code */ static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode opcode) { switch (opcode) { case FLUSH_PROT_ERR: return IB_WC_LOC_PROT_ERR; case FLUSH_REM_ACCESS_ERR: return IB_WC_REM_ACCESS_ERR; case FLUSH_LOC_QP_OP_ERR: return IB_WC_LOC_QP_OP_ERR; case FLUSH_REM_OP_ERR: return IB_WC_REM_OP_ERR; case FLUSH_LOC_LEN_ERR: return IB_WC_LOC_LEN_ERR; case FLUSH_GENERAL_ERR: return IB_WC_WR_FLUSH_ERR; case FLUSH_MW_BIND_ERR: return IB_WC_MW_BIND_ERR; case FLUSH_REM_INV_REQ_ERR: return IB_WC_REM_INV_REQ_ERR; case FLUSH_RETRY_EXC_ERR: return IB_WC_RETRY_EXC_ERR; case FLUSH_FATAL_ERR: default: return IB_WC_FATAL_ERR; } } static inline void set_ib_wc_op_sq(struct irdma_cq_poll_info *cq_poll_info, struct ib_wc *entry) { struct irdma_sc_qp *qp; switch (cq_poll_info->op_type) { case IRDMA_OP_TYPE_RDMA_WRITE: case IRDMA_OP_TYPE_RDMA_WRITE_SOL: entry->opcode = IB_WC_RDMA_WRITE; break; case IRDMA_OP_TYPE_RDMA_READ_INV_STAG: case IRDMA_OP_TYPE_RDMA_READ: entry->opcode = IB_WC_RDMA_READ; break; case IRDMA_OP_TYPE_SEND_SOL: case IRDMA_OP_TYPE_SEND_SOL_INV: case IRDMA_OP_TYPE_SEND_INV: case IRDMA_OP_TYPE_SEND: entry->opcode = IB_WC_SEND; break; case IRDMA_OP_TYPE_FAST_REG_NSMR: entry->opcode = IB_WC_REG_MR; break; case IRDMA_OP_TYPE_INV_STAG: entry->opcode = IB_WC_LOCAL_INV; break; default: qp = cq_poll_info->qp_handle; ibdev_err(irdma_get_ibdev(qp->dev), "Invalid opcode = %d in CQE\n", cq_poll_info->op_type); entry->status = IB_WC_GENERAL_ERR; } } static inline void set_ib_wc_op_rq(struct irdma_cq_poll_info *cq_poll_info, struct ib_wc *entry, bool send_imm_support) { /** * iWARP does not support sendImm, so the presence of Imm data * must be WriteImm. */ if (!send_imm_support) { entry->opcode = cq_poll_info->imm_valid ? IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV; return; } switch (cq_poll_info->op_type) { case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE: case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE: entry->opcode = IB_WC_RECV_RDMA_WITH_IMM; break; default: entry->opcode = IB_WC_RECV; } } /** * irdma_process_cqe - process cqe info * @entry: processed cqe * @cq_poll_info: cqe info */ static void irdma_process_cqe(struct ib_wc *entry, struct irdma_cq_poll_info *cq_poll_info) { struct irdma_sc_qp *qp; entry->wc_flags = 0; entry->pkey_index = 0; entry->wr_id = cq_poll_info->wr_id; qp = cq_poll_info->qp_handle; entry->qp = qp->qp_uk.back_qp; if (cq_poll_info->error) { entry->status = (cq_poll_info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ? irdma_flush_err_to_ib_wc_status(cq_poll_info->minor_err) : IB_WC_GENERAL_ERR; entry->vendor_err = cq_poll_info->major_err << 16 | cq_poll_info->minor_err; } else { entry->status = IB_WC_SUCCESS; if (cq_poll_info->imm_valid) { entry->ex.imm_data = htonl(cq_poll_info->imm_data); entry->wc_flags |= IB_WC_WITH_IMM; } if (cq_poll_info->ud_smac_valid) { ether_addr_copy(entry->smac, cq_poll_info->ud_smac); entry->wc_flags |= IB_WC_WITH_SMAC; } if (cq_poll_info->ud_vlan_valid) { u16 vlan = cq_poll_info->ud_vlan & EVL_VLID_MASK; entry->sl = cq_poll_info->ud_vlan >> VLAN_PRIO_SHIFT; if (vlan) { entry->vlan_id = vlan; entry->wc_flags |= IB_WC_WITH_VLAN; } } else { entry->sl = 0; } } if (cq_poll_info->q_type == IRDMA_CQE_QTYPE_SQ) { set_ib_wc_op_sq(cq_poll_info, entry); } else { set_ib_wc_op_rq(cq_poll_info, entry, qp->qp_uk.qp_caps & IRDMA_SEND_WITH_IMM ? true : false); if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD && cq_poll_info->stag_invalid_set) { entry->ex.invalidate_rkey = cq_poll_info->inv_stag; entry->wc_flags |= IB_WC_WITH_INVALIDATE; } } if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) { entry->src_qp = cq_poll_info->ud_src_qpn; entry->slid = 0; entry->wc_flags |= (IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE); entry->network_hdr_type = cq_poll_info->ipv4 ? RDMA_NETWORK_IPV4 : RDMA_NETWORK_IPV6; } else { entry->src_qp = cq_poll_info->qp_id; } entry->byte_len = cq_poll_info->bytes_xfered; } /** * irdma_poll_one - poll one entry of the CQ * @ukcq: ukcq to poll * @cur_cqe: current CQE info to be filled in * @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ * * Returns the internal irdma device error code or 0 on success */ static inline int irdma_poll_one(struct irdma_cq_uk *ukcq, struct irdma_cq_poll_info *cur_cqe, struct ib_wc *entry) { int ret = irdma_uk_cq_poll_cmpl(ukcq, cur_cqe); if (ret) return ret; irdma_process_cqe(entry, cur_cqe); return 0; } /** * __irdma_poll_cq - poll cq for completion (kernel apps) * @iwcq: cq to poll * @num_entries: number of entries to poll * @entry: wr of a completed entry */ static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc *entry) { struct list_head *tmp_node, *list_node; struct irdma_cq_buf *last_buf = NULL; struct irdma_cq_poll_info *cur_cqe = &iwcq->cur_cqe; struct irdma_cq_buf *cq_buf; int ret; struct irdma_device *iwdev; struct irdma_cq_uk *ukcq; bool cq_new_cqe = false; int resized_bufs = 0; int npolled = 0; iwdev = to_iwdev(iwcq->ibcq.device); ukcq = &iwcq->sc_cq.cq_uk; /* go through the list of previously resized CQ buffers */ list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { cq_buf = container_of(list_node, struct irdma_cq_buf, list); while (npolled < num_entries) { ret = irdma_poll_one(&cq_buf->cq_uk, cur_cqe, entry + npolled); if (!ret) { ++npolled; cq_new_cqe = true; continue; } if (ret == -ENOENT) break; /* QP using the CQ is destroyed. Skip reporting this CQE */ if (ret == -EFAULT) { cq_new_cqe = true; continue; } goto error; } /* save the resized CQ buffer which received the last cqe */ if (cq_new_cqe) last_buf = cq_buf; cq_new_cqe = false; } /* check the current CQ for new cqes */ while (npolled < num_entries) { ret = irdma_poll_one(ukcq, cur_cqe, entry + npolled); if (ret == -ENOENT) { ret = irdma_generated_cmpls(iwcq, cur_cqe); if (!ret) irdma_process_cqe(entry + npolled, cur_cqe); } if (!ret) { ++npolled; cq_new_cqe = true; continue; } if (ret == -ENOENT) break; /* QP using the CQ is destroyed. Skip reporting this CQE */ if (ret == -EFAULT) { cq_new_cqe = true; continue; } goto error; } if (cq_new_cqe) /* all previous CQ resizes are complete */ resized_bufs = irdma_process_resize_list(iwcq, iwdev, NULL); else if (last_buf) /* only CQ resizes up to the last_buf are complete */ resized_bufs = irdma_process_resize_list(iwcq, iwdev, last_buf); if (resized_bufs) /* report to the HW the number of complete CQ resizes */ irdma_uk_cq_set_resized_cnt(ukcq, resized_bufs); return npolled; error: irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "%s: Error polling CQ, irdma_err: %d\n", __func__, ret); return ret; } /** * irdma_poll_cq - poll cq for completion (kernel apps) * @ibcq: cq to poll * @num_entries: number of entries to poll * @entry: wr of a completed entry */ static int irdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) { struct irdma_cq *iwcq; unsigned long flags; int ret; iwcq = to_iwcq(ibcq); spin_lock_irqsave(&iwcq->lock, flags); ret = __irdma_poll_cq(iwcq, num_entries, entry); spin_unlock_irqrestore(&iwcq->lock, flags); return ret; } /** * irdma_req_notify_cq - arm cq kernel application * @ibcq: cq to arm * @notify_flags: notofication flags */ static int irdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags) { struct irdma_cq *iwcq; struct irdma_cq_uk *ukcq; unsigned long flags; enum irdma_cmpl_notify cq_notify = IRDMA_CQ_COMPL_EVENT; bool promo_event = false; int ret = 0; iwcq = to_iwcq(ibcq); ukcq = &iwcq->sc_cq.cq_uk; spin_lock_irqsave(&iwcq->lock, flags); if (notify_flags == IB_CQ_SOLICITED) { cq_notify = IRDMA_CQ_COMPL_SOLICITED; } else { if (iwcq->last_notify == IRDMA_CQ_COMPL_SOLICITED) promo_event = true; } if (!atomic_cmpxchg(&iwcq->armed, 0, 1) || promo_event) { iwcq->last_notify = cq_notify; irdma_uk_cq_request_notification(ukcq, cq_notify); } if ((notify_flags & IB_CQ_REPORT_MISSED_EVENTS) && (!irdma_cq_empty(iwcq) || !list_empty(&iwcq->cmpl_generated))) ret = 1; spin_unlock_irqrestore(&iwcq->lock, flags); return ret; } /** * mcast_list_add - Add a new mcast item to list * @rf: RDMA PCI function * @new_elem: pointer to element to add */ static void mcast_list_add(struct irdma_pci_f *rf, struct mc_table_list *new_elem) { list_add(&new_elem->list, &rf->mc_qht_list.list); } /** * mcast_list_del - Remove an mcast item from list * @mc_qht_elem: pointer to mcast table list element */ static void mcast_list_del(struct mc_table_list *mc_qht_elem) { if (mc_qht_elem) list_del(&mc_qht_elem->list); } /** * mcast_list_lookup_ip - Search mcast list for address * @rf: RDMA PCI function * @ip_mcast: pointer to mcast IP address */ static struct mc_table_list * mcast_list_lookup_ip(struct irdma_pci_f *rf, u32 *ip_mcast) { struct mc_table_list *mc_qht_el; struct list_head *pos, *q; list_for_each_safe(pos, q, &rf->mc_qht_list.list) { mc_qht_el = list_entry(pos, struct mc_table_list, list); if (!memcmp(mc_qht_el->mc_info.dest_ip, ip_mcast, sizeof(mc_qht_el->mc_info.dest_ip))) return mc_qht_el; } return NULL; } /** * irdma_mcast_cqp_op - perform a mcast cqp operation * @iwdev: irdma device * @mc_grp_ctx: mcast group info * @op: operation * * returns error status */ static int irdma_mcast_cqp_op(struct irdma_device *iwdev, struct irdma_mcast_grp_info *mc_grp_ctx, u8 op) { struct cqp_cmds_info *cqp_info; struct irdma_cqp_request *cqp_request; int status; cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); if (!cqp_request) return -ENOMEM; cqp_request->info.in.u.mc_create.info = *mc_grp_ctx; cqp_info = &cqp_request->info; cqp_info->cqp_cmd = op; cqp_info->post_sq = 1; cqp_info->in.u.mc_create.scratch = (uintptr_t)cqp_request; cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp; status = irdma_handle_cqp_op(iwdev->rf, cqp_request); irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); return status; } /** * irdma_attach_mcast - attach a qp to a multicast group * @ibqp: ptr to qp * @ibgid: pointer to global ID * @lid: local ID * * returns error status */ static int irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) { struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_device *iwdev = iwqp->iwdev; struct irdma_pci_f *rf = iwdev->rf; struct mc_table_list *mc_qht_elem; struct irdma_mcast_grp_ctx_entry_info mcg_info = {0}; unsigned long flags; u32 ip_addr[4] = {0}; u32 mgn; u32 no_mgs; int ret = 0; bool ipv4; u16 vlan_id; union { struct sockaddr saddr; struct sockaddr_in saddr_in; struct sockaddr_in6 saddr_in6; } sgid_addr; unsigned char dmac[ETH_ALEN]; rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) { irdma_copy_ip_ntohl(ip_addr, sgid_addr.saddr_in6.sin6_addr.__u6_addr.__u6_addr32); irdma_netdev_vlan_ipv6(ip_addr, &vlan_id, NULL); ipv4 = false; irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "qp_id=%d, IP6address=%pI6\n", ibqp->qp_num, ip_addr); irdma_mcast_mac_v6(ip_addr, dmac); } else { ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); ipv4 = true; vlan_id = irdma_get_vlan_ipv4(ip_addr); irdma_mcast_mac_v4(ip_addr, dmac); irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "qp_id=%d, IP4address=%pI4, MAC=%pM\n", ibqp->qp_num, ip_addr, dmac); } spin_lock_irqsave(&rf->qh_list_lock, flags); mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); if (!mc_qht_elem) { struct irdma_dma_mem *dma_mem_mc; spin_unlock_irqrestore(&rf->qh_list_lock, flags); mc_qht_elem = kzalloc(sizeof(*mc_qht_elem), GFP_KERNEL); if (!mc_qht_elem) return -ENOMEM; mc_qht_elem->mc_info.ipv4_valid = ipv4; memcpy(mc_qht_elem->mc_info.dest_ip, ip_addr, sizeof(mc_qht_elem->mc_info.dest_ip)); ret = irdma_alloc_rsrc(rf, rf->allocated_mcgs, rf->max_mcg, &mgn, &rf->next_mcg); if (ret) { kfree(mc_qht_elem); return -ENOMEM; } mc_qht_elem->mc_info.mgn = mgn; dma_mem_mc = &mc_qht_elem->mc_grp_ctx.dma_mem_mc; dma_mem_mc->size = sizeof(u64)* IRDMA_MAX_MGS_PER_CTX; dma_mem_mc->va = irdma_allocate_dma_mem(&rf->hw, dma_mem_mc, dma_mem_mc->size, IRDMA_HW_PAGE_SIZE); if (!dma_mem_mc->va) { irdma_free_rsrc(rf, rf->allocated_mcgs, mgn); kfree(mc_qht_elem); return -ENOMEM; } mc_qht_elem->mc_grp_ctx.mg_id = (u16)mgn; memcpy(mc_qht_elem->mc_grp_ctx.dest_ip_addr, ip_addr, sizeof(mc_qht_elem->mc_grp_ctx.dest_ip_addr)); mc_qht_elem->mc_grp_ctx.ipv4_valid = ipv4; mc_qht_elem->mc_grp_ctx.vlan_id = vlan_id; if (vlan_id < VLAN_N_VID) mc_qht_elem->mc_grp_ctx.vlan_valid = true; mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->rf->sc_dev.hmc_fn_id; mc_qht_elem->mc_grp_ctx.qs_handle = iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle; ether_addr_copy(mc_qht_elem->mc_grp_ctx.dest_mac_addr, dmac); spin_lock_irqsave(&rf->qh_list_lock, flags); mcast_list_add(rf, mc_qht_elem); } else { if (mc_qht_elem->mc_grp_ctx.no_of_mgs == IRDMA_MAX_MGS_PER_CTX) { spin_unlock_irqrestore(&rf->qh_list_lock, flags); return -ENOMEM; } } mcg_info.qp_id = iwqp->ibqp.qp_num; no_mgs = mc_qht_elem->mc_grp_ctx.no_of_mgs; irdma_sc_add_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); spin_unlock_irqrestore(&rf->qh_list_lock, flags); /* Only if there is a change do we need to modify or create */ if (!no_mgs) { ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, IRDMA_OP_MC_CREATE); } else if (no_mgs != mc_qht_elem->mc_grp_ctx.no_of_mgs) { ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, IRDMA_OP_MC_MODIFY); } else { return 0; } if (ret) goto error; return 0; error: irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { mcast_list_del(mc_qht_elem); irdma_free_dma_mem(&rf->hw, &mc_qht_elem->mc_grp_ctx.dma_mem_mc); irdma_free_rsrc(rf, rf->allocated_mcgs, mc_qht_elem->mc_grp_ctx.mg_id); kfree(mc_qht_elem); } return ret; } /** * irdma_detach_mcast - detach a qp from a multicast group * @ibqp: ptr to qp * @ibgid: pointer to global ID * @lid: local ID * * returns error status */ static int irdma_detach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) { struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_device *iwdev = iwqp->iwdev; struct irdma_pci_f *rf = iwdev->rf; u32 ip_addr[4] = {0}; struct mc_table_list *mc_qht_elem; struct irdma_mcast_grp_ctx_entry_info mcg_info = {0}; int ret; unsigned long flags; union { struct sockaddr saddr; struct sockaddr_in saddr_in; struct sockaddr_in6 saddr_in6; } sgid_addr; rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) irdma_copy_ip_ntohl(ip_addr, sgid_addr.saddr_in6.sin6_addr.__u6_addr.__u6_addr32); else ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); spin_lock_irqsave(&rf->qh_list_lock, flags); mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); if (!mc_qht_elem) { spin_unlock_irqrestore(&rf->qh_list_lock, flags); irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "address not found MCG\n"); return 0; } mcg_info.qp_id = iwqp->ibqp.qp_num; irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { mcast_list_del(mc_qht_elem); spin_unlock_irqrestore(&rf->qh_list_lock, flags); ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, IRDMA_OP_MC_DESTROY); if (ret) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "failed MC_DESTROY MCG\n"); spin_lock_irqsave(&rf->qh_list_lock, flags); mcast_list_add(rf, mc_qht_elem); spin_unlock_irqrestore(&rf->qh_list_lock, flags); return -EAGAIN; } irdma_free_dma_mem(&rf->hw, &mc_qht_elem->mc_grp_ctx.dma_mem_mc); irdma_free_rsrc(rf, rf->allocated_mcgs, mc_qht_elem->mc_grp_ctx.mg_id); kfree(mc_qht_elem); } else { spin_unlock_irqrestore(&rf->qh_list_lock, flags); ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, IRDMA_OP_MC_MODIFY); if (ret) { irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, "failed Modify MCG\n"); return ret; } } return 0; } /** * irdma_query_ah - Query address handle * @ibah: pointer to address handle * @ah_attr: address handle attributes */ static int irdma_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr) { struct irdma_ah *ah = to_iwah(ibah); memset(ah_attr, 0, sizeof(*ah_attr)); if (ah->av.attrs.ah_flags & IB_AH_GRH) { ah_attr->ah_flags = IB_AH_GRH; ah_attr->grh.flow_label = ah->sc_ah.ah_info.flow_label; ah_attr->grh.traffic_class = ah->sc_ah.ah_info.tc_tos; ah_attr->grh.hop_limit = ah->sc_ah.ah_info.hop_ttl; ah_attr->grh.sgid_index = ah->sgid_index; ah_attr->grh.sgid_index = ah->sgid_index; memcpy(&ah_attr->grh.dgid, &ah->dgid, sizeof(ah_attr->grh.dgid)); } return 0; } static __be64 irdma_mac_to_guid(struct ifnet *ndev){ const unsigned char *mac = IF_LLADDR(ndev); __be64 guid; unsigned char *dst = (unsigned char *)&guid; dst[0] = mac[0] ^ 2; dst[1] = mac[1]; dst[2] = mac[2]; dst[3] = 0xff; dst[4] = 0xfe; dst[5] = mac[3]; dst[6] = mac[4]; dst[7] = mac[5]; return guid; } static struct ifnet * irdma_get_netdev(struct ib_device *ibdev, u8 port_num) { struct irdma_device *iwdev = to_iwdev(ibdev); if (iwdev->netdev) { dev_hold(iwdev->netdev); return iwdev->netdev; } return NULL; } static void irdma_set_device_ops(struct ib_device *ibdev) { struct ib_device *dev_ops = ibdev; -#if __FreeBSD_version >= 1400000 - dev_ops->ops.driver_id = RDMA_DRIVER_I40IW; - dev_ops->ops.size_ib_ah = IRDMA_SET_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah); - dev_ops->ops.size_ib_cq = IRDMA_SET_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq); - dev_ops->ops.size_ib_pd = IRDMA_SET_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd); - dev_ops->ops.size_ib_ucontext = IRDMA_SET_RDMA_OBJ_SIZE(ib_ucontext, - irdma_ucontext, - ibucontext); - -#endif /* __FreeBSD_version >= 1400000 */ dev_ops->alloc_hw_stats = irdma_alloc_hw_stats; dev_ops->alloc_mr = irdma_alloc_mr; dev_ops->alloc_mw = irdma_alloc_mw; dev_ops->alloc_pd = irdma_alloc_pd; dev_ops->alloc_ucontext = irdma_alloc_ucontext; dev_ops->create_cq = irdma_create_cq; dev_ops->create_qp = irdma_create_qp; dev_ops->dealloc_mw = irdma_dealloc_mw; dev_ops->dealloc_pd = irdma_dealloc_pd; dev_ops->dealloc_ucontext = irdma_dealloc_ucontext; dev_ops->dereg_mr = irdma_dereg_mr; dev_ops->destroy_cq = irdma_destroy_cq; dev_ops->destroy_qp = irdma_destroy_qp; dev_ops->disassociate_ucontext = irdma_disassociate_ucontext; dev_ops->get_dev_fw_str = irdma_get_dev_fw_str; dev_ops->get_dma_mr = irdma_get_dma_mr; dev_ops->get_hw_stats = irdma_get_hw_stats; dev_ops->get_netdev = irdma_get_netdev; dev_ops->map_mr_sg = irdma_map_mr_sg; dev_ops->mmap = irdma_mmap; -#if __FreeBSD_version >= 1400026 - dev_ops->mmap_free = irdma_mmap_free; -#endif dev_ops->poll_cq = irdma_poll_cq; dev_ops->post_recv = irdma_post_recv; dev_ops->post_send = irdma_post_send; dev_ops->query_device = irdma_query_device; dev_ops->query_port = irdma_query_port; dev_ops->modify_port = irdma_modify_port; dev_ops->query_qp = irdma_query_qp; dev_ops->reg_user_mr = irdma_reg_user_mr; dev_ops->rereg_user_mr = irdma_rereg_user_mr; dev_ops->req_notify_cq = irdma_req_notify_cq; dev_ops->resize_cq = irdma_resize_cq; } static void irdma_set_device_mcast_ops(struct ib_device *ibdev) { struct ib_device *dev_ops = ibdev; dev_ops->attach_mcast = irdma_attach_mcast; dev_ops->detach_mcast = irdma_detach_mcast; } static void irdma_set_device_roce_ops(struct ib_device *ibdev) { struct ib_device *dev_ops = ibdev; dev_ops->create_ah = irdma_create_ah; dev_ops->destroy_ah = irdma_destroy_ah; dev_ops->get_link_layer = irdma_get_link_layer; dev_ops->get_port_immutable = irdma_roce_port_immutable; dev_ops->modify_qp = irdma_modify_qp_roce; dev_ops->query_ah = irdma_query_ah; dev_ops->query_gid = irdma_query_gid_roce; dev_ops->query_pkey = irdma_query_pkey; ibdev->add_gid = irdma_add_gid; ibdev->del_gid = irdma_del_gid; } static void irdma_set_device_iw_ops(struct ib_device *ibdev) { struct ib_device *dev_ops = ibdev; ibdev->uverbs_cmd_mask |= (1ull << IB_USER_VERBS_CMD_CREATE_AH) | (1ull << IB_USER_VERBS_CMD_DESTROY_AH); dev_ops->create_ah = irdma_create_ah_stub; dev_ops->destroy_ah = irdma_destroy_ah_stub; dev_ops->get_port_immutable = irdma_iw_port_immutable; dev_ops->modify_qp = irdma_modify_qp; dev_ops->query_gid = irdma_query_gid; dev_ops->query_pkey = irdma_iw_query_pkey; } static inline void irdma_set_device_gen1_ops(struct ib_device *ibdev) { } /** * irdma_init_roce_device - initialization of roce rdma device * @iwdev: irdma device */ static void irdma_init_roce_device(struct irdma_device *iwdev) { kc_set_roce_uverbs_cmd_mask(iwdev); iwdev->ibdev.node_type = RDMA_NODE_IB_CA; iwdev->ibdev.node_guid = irdma_mac_to_guid(iwdev->netdev); irdma_set_device_roce_ops(&iwdev->ibdev); if (iwdev->rf->rdma_ver == IRDMA_GEN_2) irdma_set_device_mcast_ops(&iwdev->ibdev); } /** * irdma_init_iw_device - initialization of iwarp rdma device * @iwdev: irdma device */ static int irdma_init_iw_device(struct irdma_device *iwdev) { struct ifnet *netdev = iwdev->netdev; iwdev->ibdev.node_type = RDMA_NODE_RNIC; ether_addr_copy((u8 *)&iwdev->ibdev.node_guid, IF_LLADDR(netdev)); iwdev->ibdev.iwcm = kzalloc(sizeof(*iwdev->ibdev.iwcm), GFP_KERNEL); if (!iwdev->ibdev.iwcm) return -ENOMEM; iwdev->ibdev.iwcm->add_ref = irdma_qp_add_ref; iwdev->ibdev.iwcm->rem_ref = irdma_qp_rem_ref; iwdev->ibdev.iwcm->get_qp = irdma_get_qp; iwdev->ibdev.iwcm->connect = irdma_connect; iwdev->ibdev.iwcm->accept = irdma_accept; iwdev->ibdev.iwcm->reject = irdma_reject; iwdev->ibdev.iwcm->create_listen = irdma_create_listen; iwdev->ibdev.iwcm->destroy_listen = irdma_destroy_listen; memcpy(iwdev->ibdev.iwcm->ifname, if_name(netdev), sizeof(iwdev->ibdev.iwcm->ifname)); irdma_set_device_iw_ops(&iwdev->ibdev); return 0; } /** * irdma_init_rdma_device - initialization of rdma device * @iwdev: irdma device */ static int irdma_init_rdma_device(struct irdma_device *iwdev) { struct pci_dev *pcidev = iwdev->rf->pcidev; int ret; iwdev->ibdev.owner = THIS_MODULE; iwdev->ibdev.uverbs_abi_ver = IRDMA_ABI_VER; kc_set_rdma_uverbs_cmd_mask(iwdev); if (iwdev->roce_mode) { irdma_init_roce_device(iwdev); } else { ret = irdma_init_iw_device(iwdev); if (ret) return ret; } iwdev->ibdev.phys_port_cnt = 1; iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count; iwdev->ibdev.dev.parent = iwdev->rf->dev_ctx.dev; set_ibdev_dma_device(iwdev->ibdev, &pcidev->dev); irdma_set_device_ops(&iwdev->ibdev); if (iwdev->rf->rdma_ver == IRDMA_GEN_1) irdma_set_device_gen1_ops(&iwdev->ibdev); return 0; } /** * irdma_port_ibevent - indicate port event * @iwdev: irdma device */ void irdma_port_ibevent(struct irdma_device *iwdev) { struct ib_event event; event.device = &iwdev->ibdev; event.element.port_num = 1; event.event = iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; ib_dispatch_event(&event); } /** * irdma_ib_unregister_device - unregister rdma device from IB * core * @iwdev: irdma device */ void irdma_ib_unregister_device(struct irdma_device *iwdev) { iwdev->iw_status = 0; irdma_port_ibevent(iwdev); ib_unregister_device(&iwdev->ibdev); dev_put(iwdev->netdev); kfree(iwdev->ibdev.iwcm); iwdev->ibdev.iwcm = NULL; } /** * irdma_ib_register_device - register irdma device to IB core * @iwdev: irdma device */ int irdma_ib_register_device(struct irdma_device *iwdev) { int ret; ret = irdma_init_rdma_device(iwdev); if (ret) return ret; dev_hold(iwdev->netdev); sprintf(iwdev->ibdev.name, "irdma-%s", if_name(iwdev->netdev)); ret = ib_register_device(&iwdev->ibdev, NULL); if (ret) goto error; iwdev->iw_status = 1; irdma_port_ibevent(iwdev); return 0; error: kfree(iwdev->ibdev.iwcm); iwdev->ibdev.iwcm = NULL; irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "Register RDMA device fail\n"); return ret; } diff --git a/sys/dev/irdma/irdma_verbs.h b/sys/dev/irdma/irdma_verbs.h index c4e778706106..dd0024617aba 100644 --- a/sys/dev/irdma/irdma_verbs.h +++ b/sys/dev/irdma/irdma_verbs.h @@ -1,338 +1,319 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * * Copyright (c) 2015 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenFabrics.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /*$FreeBSD$*/ #ifndef IRDMA_VERBS_H #define IRDMA_VERBS_H #define IRDMA_MAX_SAVED_PHY_PGADDR 4 #define IRDMA_FLUSH_DELAY_MS 20 #define IRDMA_PKEY_TBL_SZ 1 #define IRDMA_DEFAULT_PKEY 0xFFFF #define iwdev_to_idev(iwdev) (&(iwdev)->rf->sc_dev) struct irdma_ucontext { struct ib_ucontext ibucontext; struct irdma_device *iwdev; -#if __FreeBSD_version >= 1400026 - struct rdma_user_mmap_entry *db_mmap_entry; -#else struct irdma_user_mmap_entry *db_mmap_entry; DECLARE_HASHTABLE(mmap_hash_tbl, 6); spinlock_t mmap_tbl_lock; /* protect mmap hash table entries */ -#endif struct list_head cq_reg_mem_list; spinlock_t cq_reg_mem_list_lock; /* protect CQ memory list */ struct list_head qp_reg_mem_list; spinlock_t qp_reg_mem_list_lock; /* protect QP memory list */ /* FIXME: Move to kcompat ideally. Used < 4.20.0 for old diassasscoaite flow */ struct list_head vma_list; struct mutex vma_list_mutex; /* protect the vma_list */ int abi_ver; bool legacy_mode; }; struct irdma_pd { struct ib_pd ibpd; struct irdma_sc_pd sc_pd; }; struct irdma_av { u8 macaddr[16]; struct ib_ah_attr attrs; union { struct sockaddr saddr; struct sockaddr_in saddr_in; struct sockaddr_in6 saddr_in6; } sgid_addr, dgid_addr; u8 net_type; }; struct irdma_ah { struct ib_ah ibah; struct irdma_sc_ah sc_ah; struct irdma_pd *pd; struct irdma_av av; u8 sgid_index; union ib_gid dgid; }; struct irdma_hmc_pble { union { u32 idx; dma_addr_t addr; }; }; struct irdma_cq_mr { struct irdma_hmc_pble cq_pbl; dma_addr_t shadow; bool split; }; struct irdma_qp_mr { struct irdma_hmc_pble sq_pbl; struct irdma_hmc_pble rq_pbl; dma_addr_t shadow; struct page *sq_page; }; struct irdma_cq_buf { struct irdma_dma_mem kmem_buf; struct irdma_cq_uk cq_uk; struct irdma_hw *hw; struct list_head list; struct work_struct work; }; struct irdma_pbl { struct list_head list; union { struct irdma_qp_mr qp_mr; struct irdma_cq_mr cq_mr; }; bool pbl_allocated:1; bool on_list:1; u64 user_base; struct irdma_pble_alloc pble_alloc; struct irdma_mr *iwmr; }; struct irdma_mr { union { struct ib_mr ibmr; struct ib_mw ibmw; }; struct ib_umem *region; int access; u8 is_hwreg; u16 type; u32 page_cnt; u64 page_size; u64 page_msk; u32 npages; u32 stag; u64 len; u64 pgaddrmem[IRDMA_MAX_SAVED_PHY_PGADDR]; struct irdma_pbl iwpbl; }; struct irdma_cq { struct ib_cq ibcq; struct irdma_sc_cq sc_cq; u16 cq_head; u16 cq_size; u16 cq_num; bool user_mode; atomic_t armed; enum irdma_cmpl_notify last_notify; u32 polled_cmpls; u32 cq_mem_size; struct irdma_dma_mem kmem; struct irdma_dma_mem kmem_shadow; struct completion free_cq; atomic_t refcnt; spinlock_t lock; /* for poll cq */ struct irdma_pbl *iwpbl; struct irdma_pbl *iwpbl_shadow; struct list_head resize_list; struct irdma_cq_poll_info cur_cqe; struct list_head cmpl_generated; }; struct irdma_cmpl_gen { struct list_head list; struct irdma_cq_poll_info cpi; }; struct disconn_work { struct work_struct work; struct irdma_qp *iwqp; }; struct iw_cm_id; struct irdma_qp_kmode { struct irdma_dma_mem dma_mem; u32 *sig_trk_mem; struct irdma_sq_uk_wr_trk_info *sq_wrid_mem; u64 *rq_wrid_mem; }; struct irdma_qp { struct ib_qp ibqp; struct irdma_sc_qp sc_qp; struct irdma_device *iwdev; struct irdma_cq *iwscq; struct irdma_cq *iwrcq; struct irdma_pd *iwpd; -#if __FreeBSD_version >= 1400026 - struct rdma_user_mmap_entry *push_wqe_mmap_entry; - struct rdma_user_mmap_entry *push_db_mmap_entry; -#else struct irdma_user_mmap_entry *push_wqe_mmap_entry; struct irdma_user_mmap_entry *push_db_mmap_entry; -#endif struct irdma_qp_host_ctx_info ctx_info; union { struct irdma_iwarp_offload_info iwarp_info; struct irdma_roce_offload_info roce_info; }; union { struct irdma_tcp_offload_info tcp_info; struct irdma_udp_offload_info udp_info; }; struct irdma_ah roce_ah; struct list_head teardown_entry; atomic_t refcnt; struct iw_cm_id *cm_id; struct irdma_cm_node *cm_node; struct delayed_work dwork_flush; struct ib_mr *lsmm_mr; atomic_t hw_mod_qp_pend; enum ib_qp_state ibqp_state; u32 qp_mem_size; u32 last_aeq; int max_send_wr; int max_recv_wr; atomic_t close_timer_started; spinlock_t lock; /* serialize posting WRs to SQ/RQ */ struct irdma_qp_context *iwqp_context; void *pbl_vbase; dma_addr_t pbl_pbase; struct page *page; u8 iwarp_state; u16 term_sq_flush_code; u16 term_rq_flush_code; u8 hw_iwarp_state; u8 hw_tcp_state; struct irdma_qp_kmode kqp; struct irdma_dma_mem host_ctx; struct timer_list terminate_timer; struct irdma_pbl *iwpbl; struct irdma_sge *sg_list; struct irdma_dma_mem q2_ctx_mem; struct irdma_dma_mem ietf_mem; struct completion free_qp; wait_queue_head_t waitq; wait_queue_head_t mod_qp_waitq; u8 rts_ae_rcvd; u8 active_conn : 1; u8 user_mode : 1; u8 hte_added : 1; u8 flush_issued : 1; u8 sig_all : 1; u8 pau_mode : 1; }; enum irdma_mmap_flag { IRDMA_MMAP_IO_NC, IRDMA_MMAP_IO_WC, }; struct irdma_user_mmap_entry { -#if __FreeBSD_version >= 1400026 - struct rdma_user_mmap_entry rdma_entry; -#else struct irdma_ucontext *ucontext; struct hlist_node hlist; u64 pgoff_key; /* Used to compute offset (in bytes) returned to user libc's mmap */ -#endif u64 bar_offset; u8 mmap_flag; }; static inline u16 irdma_fw_major_ver(struct irdma_sc_dev *dev) { return (u16)FIELD_GET(IRDMA_FW_VER_MAJOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]); } static inline u16 irdma_fw_minor_ver(struct irdma_sc_dev *dev) { return (u16)FIELD_GET(IRDMA_FW_VER_MINOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]); } /** * irdma_mcast_mac_v4 - Get the multicast MAC for an IP address * @ip_addr: IPv4 address * @mac: pointer to result MAC address * */ static inline void irdma_mcast_mac_v4(u32 *ip_addr, u8 *mac) { u8 *ip = (u8 *)ip_addr; unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, ip[2] & 0x7F, ip[1], ip[0]}; ether_addr_copy(mac, mac4); } /** * irdma_mcast_mac_v6 - Get the multicast MAC for an IP address * @ip_addr: IPv6 address * @mac: pointer to result MAC address * */ static inline void irdma_mcast_mac_v6(u32 *ip_addr, u8 *mac) { u8 *ip = (u8 *)ip_addr; unsigned char mac6[ETH_ALEN] = {0x33, 0x33, ip[3], ip[2], ip[1], ip[0]}; ether_addr_copy(mac, mac6); } -#if __FreeBSD_version >= 1400026 -struct rdma_user_mmap_entry* -irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset, - enum irdma_mmap_flag mmap_flag, u64 *mmap_offset); -#else struct irdma_user_mmap_entry * irdma_user_mmap_entry_add_hash(struct irdma_ucontext *ucontext, u64 bar_offset, enum irdma_mmap_flag mmap_flag, u64 *mmap_offset); void irdma_user_mmap_entry_del_hash(struct irdma_user_mmap_entry *entry); -#endif int irdma_ib_register_device(struct irdma_device *iwdev); void irdma_ib_unregister_device(struct irdma_device *iwdev); void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event); void irdma_generate_flush_completions(struct irdma_qp *iwqp); void irdma_remove_cmpls_list(struct irdma_cq *iwcq); int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info); void irdma_sched_qp_flush_work(struct irdma_qp *iwqp); void irdma_flush_worker(struct work_struct *work); #endif /* IRDMA_VERBS_H */ diff --git a/sys/dev/irdma/osdep.h b/sys/dev/irdma/osdep.h index 834af04cc838..5cae2b9fdc33 100644 --- a/sys/dev/irdma/osdep.h +++ b/sys/dev/irdma/osdep.h @@ -1,252 +1,250 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * * Copyright (c) 2021 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenFabrics.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /*$FreeBSD$*/ #ifndef _ICRDMA_OSDEP_H_ #define _ICRDMA_OSDEP_H_ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define ATOMIC atomic_t #define IOMEM #define IRDMA_NTOHS(a) ntohs(a) #define MAKEMASK(m, s) ((m) << (s)) #define OS_TIMER timer_list #define DECLARE_HASHTABLE(n, b) struct hlist_head (n)[1 << (b)] #define HASH_MIN(v, b) (sizeof(v) <= 4 ? hash_32(v, b) : hash_long(v, b)) #define HASH_FOR_EACH_RCU(n, b, o, m) for ((b) = 0, o = NULL; o == NULL && (b) < ARRAY_SIZE(n);\ (b)++)\ hlist_for_each_entry_rcu(o, &n[(b)], m) #define HASH_FOR_EACH_POSSIBLE_RCU(n, o, m, k) \ hlist_for_each_entry_rcu(o, &n[jhash(&k, sizeof(k), 0) >> (32 - ilog2(ARRAY_SIZE(n)))],\ m) #define HASH_FOR_EACH_POSSIBLE(n, o, m, k) \ hlist_for_each_entry(o, &n[jhash(&k, sizeof(k), 0) >> (32 - ilog2(ARRAY_SIZE(n)))],\ m) #define HASH_ADD_RCU(h, n, k) \ hlist_add_head_rcu(n, &h[jhash(&k, sizeof(k), 0) >> (32 - ilog2(ARRAY_SIZE(h)))]) #define HASH_DEL_RCU(tbl, node) hlist_del_rcu(node) #define HASH_ADD(h, n, k) \ hlist_add_head(n, &h[jhash(&k, sizeof(k), 0) >> (32 - ilog2(ARRAY_SIZE(h)))]) #define HASH_DEL(tbl, node) hlist_del(node) #define WQ_UNBOUND_MAX_ACTIVE max_t(int, 512, num_possible_cpus() * 4) #define if_addr_rlock(x) #define if_addr_runlock(x) /* constants */ #define STATS_TIMER_DELAY 60000 /* a couple of linux size defines */ #define SZ_128 128 #define SZ_2K SZ_128 * 16 #define SZ_1G (SZ_1K * SZ_1K * SZ_1K) #define SPEED_1000 1000 #define SPEED_10000 10000 #define SPEED_20000 20000 #define SPEED_25000 25000 #define SPEED_40000 40000 #define SPEED_100000 100000 #define irdma_mb() mb() #define irdma_wmb() wmb() #define irdma_get_virt_to_phy vtophys #define __aligned_u64 uint64_t __aligned(8) #define VLAN_PRIO_SHIFT 13 -#if __FreeBSD_version < 1400000 #define IB_USER_VERBS_EX_CMD_MODIFY_QP IB_USER_VERBS_CMD_MODIFY_QP -#endif /* * debug definition section */ #define irdma_print(S, ...) printf("%s:%d "S, __FUNCTION__, __LINE__, ##__VA_ARGS__) #define irdma_debug_buf(dev, mask, desc, buf, size) \ do { \ u32 i; \ if (!((mask) & (dev)->debug_mask)) { \ break; \ } \ irdma_debug(dev, mask, "%s\n", desc); \ irdma_debug(dev, mask, "starting address virt=%p phy=%lxh\n", buf, irdma_get_virt_to_phy(buf)); \ for (i = 0; i < size ; i += 8) \ irdma_debug(dev, mask, "index %03d val: %016lx\n", i, ((unsigned long *)buf)[i / 8]); \ } while(0) #define irdma_debug(h, m, s, ...) \ do { \ if (!(h)) { \ if ((m) == IRDMA_DEBUG_INIT) \ printf("irdma INIT " s, ##__VA_ARGS__); \ } else if (((m) & (h)->debug_mask)) { \ printf("irdma " s, ##__VA_ARGS__); \ } \ } while (0) #define irdma_dev_err(a, b, ...) printf(b, ##__VA_ARGS__) #define irdma_dev_warn(a, b, ...) printf(b, ##__VA_ARGS__) /*dev_warn(a, b)*/ #define irdma_dev_info(a, b, ...) printf(b, ##__VA_ARGS__) #define irdma_pr_warn printf #define ibdev_err(ibdev, fmt, ...) printf("%s:"fmt, (ibdev)->name, ##__VA_ARGS__) #define dump_struct(s, sz, name) \ do { \ unsigned char *a; \ printf("%s %u", (name), (unsigned int)(sz)); \ for (a = (unsigned char*)(s); a < (unsigned char *)(s) + (sz) ; a ++) { \ if ((u64)a % 8 == 0) \ printf("\n%p ", a); \ printf("%2x ", *a); \ } \ printf("\n"); \ }while(0) /* * debug definition end */ typedef __be16 BE16; typedef __be32 BE32; typedef uintptr_t irdma_uintptr; struct irdma_hw; struct irdma_pci_f; struct irdma_sc_dev; struct irdma_sc_qp; struct irdma_sc_vsi; struct irdma_task_arg { struct irdma_device *iwdev; struct ice_rdma_peer *peer; atomic_t open_ongoing; atomic_t close_ongoing; }; struct irdma_dev_ctx { bus_space_tag_t mem_bus_space_tag; bus_space_handle_t mem_bus_space_handle; bus_size_t mem_bus_space_size; void *dev; struct irdma_task_arg task_arg; }; #define irdma_pr_info(fmt, args ...) printf("%s: WARN "fmt, __func__, ## args) #define irdma_pr_err(fmt, args ...) printf("%s: ERR "fmt, __func__, ## args) #define irdma_memcpy(a, b, c) memcpy((a), (b), (c)) #define irdma_memset(a, b, c) memset((a), (b), (c)) #define irdma_usec_delay(x) DELAY(x) #define mdelay(x) DELAY((x) * 1000) #define rt_tos2priority(tos) (tos >> 5) #define ah_attr_to_dmac(attr) ((attr).dmac) #define kc_ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll) \ ib_modify_qp_is_ok(cur_state, next_state, type, mask) #define kc_rdma_gid_attr_network_type(sgid_attr, gid_type, gid) \ ib_gid_to_network_type(gid_type, gid) #define irdma_del_timer_compat(tt) del_timer((tt)) #define IRDMA_TAILQ_FOREACH CK_STAILQ_FOREACH #define IRDMA_TAILQ_FOREACH_SAFE CK_STAILQ_FOREACH_SAFE #define between(a, b, c) (bool)(c-a >= b-a) #define rd32(a, reg) irdma_rd32((a)->dev_context, (reg)) #define wr32(a, reg, value) irdma_wr32((a)->dev_context, (reg), (value)) #define rd64(a, reg) irdma_rd64((a)->dev_context, (reg)) #define wr64(a, reg, value) irdma_wr64((a)->dev_context, (reg), (value)) #define db_wr32(value, a) writel((value), (a)) void *hw_to_dev(struct irdma_hw *hw); struct irdma_dma_mem { void *va; u64 pa; bus_dma_tag_t tag; bus_dmamap_t map; bus_dma_segment_t seg; bus_size_t size; int nseg; int flags; }; struct irdma_virt_mem { void *va; u32 size; }; struct irdma_dma_info { dma_addr_t *dmaaddrs; }; struct list_head; u32 irdma_rd32(struct irdma_dev_ctx *dev_ctx, u32 reg); void irdma_wr32(struct irdma_dev_ctx *dev_ctx, u32 reg, u32 value); u64 irdma_rd64(struct irdma_dev_ctx *dev_ctx, u32 reg); void irdma_wr64(struct irdma_dev_ctx *dev_ctx, u32 reg, u64 value); void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term, u8 term_len); void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred); void irdma_terminate_start_timer(struct irdma_sc_qp *qp); void irdma_terminate_del_timer(struct irdma_sc_qp *qp); void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi); void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi); void irdma_send_ieq_ack(struct irdma_sc_qp *qp); u8* irdma_get_hw_addr(void *par); void irdma_unmap_vm_page_list(struct irdma_hw *hw, u64 *pg_arr, u32 pg_cnt); int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, u64 *pg_arr, u32 pg_cnt); struct ib_device *irdma_get_ibdev(struct irdma_sc_dev *dev); #endif /* _ICRDMA_OSDEP_H_ */