Index: stable/9/contrib/ofed/libibverbs/examples/asyncwatch.c =================================================================== --- stable/9/contrib/ofed/libibverbs/examples/asyncwatch.c (revision 273863) +++ stable/9/contrib/ofed/libibverbs/examples/asyncwatch.c (revision 273864) @@ -1,121 +1,119 @@ /* * Copyright (c) 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #if HAVE_CONFIG_H # include #endif /* HAVE_CONFIG_H */ #include -#include -#include #include static const char *event_name_str(enum ibv_event_type event_type) { switch (event_type) { case IBV_EVENT_DEVICE_FATAL: return "IBV_EVENT_DEVICE_FATAL"; case IBV_EVENT_PORT_ACTIVE: return "IBV_EVENT_PORT_ACTIVE"; case IBV_EVENT_PORT_ERR: return "IBV_EVENT_PORT_ERR"; case IBV_EVENT_LID_CHANGE: return "IBV_EVENT_LID_CHANGE"; case IBV_EVENT_PKEY_CHANGE: return "IBV_EVENT_PKEY_CHANGE"; case IBV_EVENT_SM_CHANGE: return "IBV_EVENT_SM_CHANGE"; case IBV_EVENT_CLIENT_REREGISTER: return "IBV_EVENT_CLIENT_REREGISTER"; case IBV_EVENT_GID_CHANGE: return "IBV_EVENT_GID_CHANGE"; case IBV_EVENT_CQ_ERR: case IBV_EVENT_QP_FATAL: case IBV_EVENT_QP_REQ_ERR: case IBV_EVENT_QP_ACCESS_ERR: case IBV_EVENT_COMM_EST: case IBV_EVENT_SQ_DRAINED: case IBV_EVENT_PATH_MIG: case IBV_EVENT_PATH_MIG_ERR: case IBV_EVENT_SRQ_ERR: case IBV_EVENT_SRQ_LIMIT_REACHED: case IBV_EVENT_QP_LAST_WQE_REACHED: default: return "unexpected"; } } int main(int argc, char *argv[]) { struct ibv_device **dev_list; struct ibv_context *context; struct ibv_async_event event; /* Force line-buffering in case stdout is redirected */ setvbuf(stdout, NULL, _IOLBF, 0); dev_list = ibv_get_device_list(NULL); if (!dev_list) { perror("Failed to get IB devices list"); return 1; } if (!*dev_list) { fprintf(stderr, "No IB devices found\n"); return 1; } context = ibv_open_device(*dev_list); if (!context) { fprintf(stderr, "Couldn't get context for %s\n", ibv_get_device_name(*dev_list)); return 1; } printf("%s: async event FD %d\n", ibv_get_device_name(*dev_list), context->async_fd); while (1) { if (ibv_get_async_event(context, &event)) return 1; printf(" event_type %s (%d), port %d\n", event_name_str(event.event_type), event.event_type, event.element.port_num); ibv_ack_async_event(&event); } return 0; } Index: stable/9/contrib/ofed/libibverbs/examples/device_list.c =================================================================== --- stable/9/contrib/ofed/libibverbs/examples/device_list.c (revision 273863) +++ stable/9/contrib/ofed/libibverbs/examples/device_list.c (revision 273864) @@ -1,68 +1,65 @@ /* * Copyright (c) 2004 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #if HAVE_CONFIG_H # include #endif /* HAVE_CONFIG_H */ #include -#include -#include - #include #include int main(int argc, char *argv[]) { struct ibv_device **dev_list; int num_devices, i; dev_list = ibv_get_device_list(&num_devices); if (!dev_list) { perror("Failed to get IB devices list"); return 1; } printf(" %-16s\t node GUID\n", "device"); printf(" %-16s\t----------------\n", "------"); for (i = 0; i < num_devices; ++i) { printf(" %-16s\t%016llx\n", ibv_get_device_name(dev_list[i]), (unsigned long long) ntohll(ibv_get_device_guid(dev_list[i]))); } ibv_free_device_list(dev_list); return 0; } Index: stable/9/contrib/ofed/libibverbs/examples/devinfo.c =================================================================== --- stable/9/contrib/ofed/libibverbs/examples/devinfo.c (revision 273863) +++ stable/9/contrib/ofed/libibverbs/examples/devinfo.c (revision 273864) @@ -1,450 +1,448 @@ /* * Copyright (c) 2005 Cisco Systems. All rights reserved. * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #if HAVE_CONFIG_H # include #endif /* HAVE_CONFIG_H */ #include #include #include #include #include #include -#include -#include #include #include #include static int verbose; static int null_gid(union ibv_gid *gid) { return !(gid->raw[8] | gid->raw[9] | gid->raw[10] | gid->raw[11] | gid->raw[12] | gid->raw[13] | gid->raw[14] | gid->raw[15]); } static const char *guid_str(uint64_t node_guid, char *str) { node_guid = ntohll(node_guid); sprintf(str, "%04x:%04x:%04x:%04x", (unsigned) (node_guid >> 48) & 0xffff, (unsigned) (node_guid >> 32) & 0xffff, (unsigned) (node_guid >> 16) & 0xffff, (unsigned) (node_guid >> 0) & 0xffff); return str; } static const char *transport_str(enum ibv_transport_type transport) { switch (transport) { case IBV_TRANSPORT_IB: return "InfiniBand"; case IBV_TRANSPORT_IWARP: return "iWARP"; default: return "invalid transport"; } } static const char *port_state_str(enum ibv_port_state pstate) { switch (pstate) { case IBV_PORT_DOWN: return "PORT_DOWN"; case IBV_PORT_INIT: return "PORT_INIT"; case IBV_PORT_ARMED: return "PORT_ARMED"; case IBV_PORT_ACTIVE: return "PORT_ACTIVE"; default: return "invalid state"; } } static const char *port_phy_state_str(uint8_t phys_state) { switch (phys_state) { case 1: return "SLEEP"; case 2: return "POLLING"; case 3: return "DISABLED"; case 4: return "PORT_CONFIGURATION TRAINNING"; case 5: return "LINK_UP"; case 6: return "LINK_ERROR_RECOVERY"; case 7: return "PHY TEST"; default: return "invalid physical state"; } } static const char *atomic_cap_str(enum ibv_atomic_cap atom_cap) { switch (atom_cap) { case IBV_ATOMIC_NONE: return "ATOMIC_NONE"; case IBV_ATOMIC_HCA: return "ATOMIC_HCA"; case IBV_ATOMIC_GLOB: return "ATOMIC_GLOB"; default: return "invalid atomic capability"; } } static const char *mtu_str(enum ibv_mtu max_mtu) { switch (max_mtu) { case IBV_MTU_256: return "256"; case IBV_MTU_512: return "512"; case IBV_MTU_1024: return "1024"; case IBV_MTU_2048: return "2048"; case IBV_MTU_4096: return "4096"; default: return "invalid MTU"; } } static const char *width_str(uint8_t width) { switch (width) { case 1: return "1"; case 2: return "4"; case 4: return "8"; case 8: return "12"; default: return "invalid width"; } } static const char *speed_str(uint8_t speed) { switch (speed) { case 1: return "2.5 Gbps"; case 2: return "5.0 Gbps"; case 4: return "10.0 Gbps"; default: return "invalid speed"; } } static const char *vl_str(uint8_t vl_num) { switch (vl_num) { case 1: return "1"; case 2: return "2"; case 3: return "4"; case 4: return "8"; case 5: return "15"; default: return "invalid value"; } } static int print_all_port_gids(struct ibv_context *ctx, uint8_t port_num, int tbl_len) { union ibv_gid gid; int rc = 0; int i; for (i = 0; i < tbl_len; i++) { rc = ibv_query_gid(ctx, port_num, i, &gid); if (rc) { fprintf(stderr, "Failed to query gid to port %d, index %d\n", port_num, i); return rc; } if (!null_gid(&gid)) printf("\t\t\tGID[%3d]:\t\t%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x\n", i, gid.raw[ 0], gid.raw[ 1], gid.raw[ 2], gid.raw[ 3], gid.raw[ 4], gid.raw[ 5], gid.raw[ 6], gid.raw[ 7], gid.raw[ 8], gid.raw[ 9], gid.raw[10], gid.raw[11], gid.raw[12], gid.raw[13], gid.raw[14], gid.raw[15]); } return rc; } static const char *link_layer_str(uint8_t link_layer) { switch (link_layer) { case IBV_LINK_LAYER_UNSPECIFIED: case IBV_LINK_LAYER_INFINIBAND: return "IB"; case IBV_LINK_LAYER_ETHERNET: return "Ethernet"; default: return "Unknown"; } } static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port) { struct ibv_context *ctx; struct ibv_device_attr device_attr; struct ibv_port_attr port_attr; int rc = 0; uint8_t port; char buf[256]; ctx = ibv_open_device(ib_dev); if (!ctx) { fprintf(stderr, "Failed to open device\n"); rc = 1; goto cleanup; } if (ibv_query_device(ctx, &device_attr)) { fprintf(stderr, "Failed to query device props"); rc = 2; goto cleanup; } printf("hca_id:\t%s\n", ibv_get_device_name(ib_dev)); printf("\ttransport:\t\t\t%s (%d)\n", transport_str(ib_dev->transport_type), ib_dev->transport_type); if (strlen(device_attr.fw_ver)) printf("\tfw_ver:\t\t\t\t%s\n", device_attr.fw_ver); printf("\tnode_guid:\t\t\t%s\n", guid_str(device_attr.node_guid, buf)); printf("\tsys_image_guid:\t\t\t%s\n", guid_str(device_attr.sys_image_guid, buf)); printf("\tvendor_id:\t\t\t0x%04x\n", device_attr.vendor_id); printf("\tvendor_part_id:\t\t\t%d\n", device_attr.vendor_part_id); printf("\thw_ver:\t\t\t\t0x%X\n", device_attr.hw_ver); if (ibv_read_sysfs_file(ib_dev->ibdev_path, "board_id", buf, sizeof buf) > 0) printf("\tboard_id:\t\t\t%s\n", buf); printf("\tphys_port_cnt:\t\t\t%d\n", device_attr.phys_port_cnt); if (verbose) { printf("\tmax_mr_size:\t\t\t0x%llx\n", (unsigned long long) device_attr.max_mr_size); printf("\tpage_size_cap:\t\t\t0x%llx\n", (unsigned long long) device_attr.page_size_cap); printf("\tmax_qp:\t\t\t\t%d\n", device_attr.max_qp); printf("\tmax_qp_wr:\t\t\t%d\n", device_attr.max_qp_wr); printf("\tdevice_cap_flags:\t\t0x%08x\n", device_attr.device_cap_flags); printf("\tmax_sge:\t\t\t%d\n", device_attr.max_sge); printf("\tmax_sge_rd:\t\t\t%d\n", device_attr.max_sge_rd); printf("\tmax_cq:\t\t\t\t%d\n", device_attr.max_cq); printf("\tmax_cqe:\t\t\t%d\n", device_attr.max_cqe); printf("\tmax_mr:\t\t\t\t%d\n", device_attr.max_mr); printf("\tmax_pd:\t\t\t\t%d\n", device_attr.max_pd); printf("\tmax_qp_rd_atom:\t\t\t%d\n", device_attr.max_qp_rd_atom); printf("\tmax_ee_rd_atom:\t\t\t%d\n", device_attr.max_ee_rd_atom); printf("\tmax_res_rd_atom:\t\t%d\n", device_attr.max_res_rd_atom); printf("\tmax_qp_init_rd_atom:\t\t%d\n", device_attr.max_qp_init_rd_atom); printf("\tmax_ee_init_rd_atom:\t\t%d\n", device_attr.max_ee_init_rd_atom); printf("\tatomic_cap:\t\t\t%s (%d)\n", atomic_cap_str(device_attr.atomic_cap), device_attr.atomic_cap); printf("\tmax_ee:\t\t\t\t%d\n", device_attr.max_ee); printf("\tmax_rdd:\t\t\t%d\n", device_attr.max_rdd); printf("\tmax_mw:\t\t\t\t%d\n", device_attr.max_mw); printf("\tmax_raw_ipv6_qp:\t\t%d\n", device_attr.max_raw_ipv6_qp); printf("\tmax_raw_ethy_qp:\t\t%d\n", device_attr.max_raw_ethy_qp); printf("\tmax_mcast_grp:\t\t\t%d\n", device_attr.max_mcast_grp); printf("\tmax_mcast_qp_attach:\t\t%d\n", device_attr.max_mcast_qp_attach); printf("\tmax_total_mcast_qp_attach:\t%d\n", device_attr.max_total_mcast_qp_attach); printf("\tmax_ah:\t\t\t\t%d\n", device_attr.max_ah); printf("\tmax_fmr:\t\t\t%d\n", device_attr.max_fmr); if (device_attr.max_fmr) printf("\tmax_map_per_fmr:\t\t%d\n", device_attr.max_map_per_fmr); printf("\tmax_srq:\t\t\t%d\n", device_attr.max_srq); if (device_attr.max_srq) { printf("\tmax_srq_wr:\t\t\t%d\n", device_attr.max_srq_wr); printf("\tmax_srq_sge:\t\t\t%d\n", device_attr.max_srq_sge); } printf("\tmax_pkeys:\t\t\t%d\n", device_attr.max_pkeys); printf("\tlocal_ca_ack_delay:\t\t%d\n", device_attr.local_ca_ack_delay); } for (port = 1; port <= device_attr.phys_port_cnt; ++port) { /* if in the command line the user didn't ask for info about this port */ if ((ib_port) && (port != ib_port)) continue; rc = ibv_query_port(ctx, port, &port_attr); if (rc) { fprintf(stderr, "Failed to query port %u props\n", port); goto cleanup; } printf("\t\tport:\t%d\n", port); printf("\t\t\tstate:\t\t\t%s (%d)\n", port_state_str(port_attr.state), port_attr.state); printf("\t\t\tmax_mtu:\t\t%s (%d)\n", mtu_str(port_attr.max_mtu), port_attr.max_mtu); printf("\t\t\tactive_mtu:\t\t%s (%d)\n", mtu_str(port_attr.active_mtu), port_attr.active_mtu); printf("\t\t\tsm_lid:\t\t\t%d\n", port_attr.sm_lid); printf("\t\t\tport_lid:\t\t%d\n", port_attr.lid); printf("\t\t\tport_lmc:\t\t0x%02x\n", port_attr.lmc); printf("\t\t\tlink_layer:\t\t%s\n", link_layer_str(port_attr.link_layer)); if (verbose) { printf("\t\t\tmax_msg_sz:\t\t0x%x\n", port_attr.max_msg_sz); printf("\t\t\tport_cap_flags:\t\t0x%08x\n", port_attr.port_cap_flags); printf("\t\t\tmax_vl_num:\t\t%s (%d)\n", vl_str(port_attr.max_vl_num), port_attr.max_vl_num); printf("\t\t\tbad_pkey_cntr:\t\t0x%x\n", port_attr.bad_pkey_cntr); printf("\t\t\tqkey_viol_cntr:\t\t0x%x\n", port_attr.qkey_viol_cntr); printf("\t\t\tsm_sl:\t\t\t%d\n", port_attr.sm_sl); printf("\t\t\tpkey_tbl_len:\t\t%d\n", port_attr.pkey_tbl_len); printf("\t\t\tgid_tbl_len:\t\t%d\n", port_attr.gid_tbl_len); printf("\t\t\tsubnet_timeout:\t\t%d\n", port_attr.subnet_timeout); printf("\t\t\tinit_type_reply:\t%d\n", port_attr.init_type_reply); printf("\t\t\tactive_width:\t\t%sX (%d)\n", width_str(port_attr.active_width), port_attr.active_width); printf("\t\t\tactive_speed:\t\t%s (%d)\n", speed_str(port_attr.active_speed), port_attr.active_speed); printf("\t\t\tphys_state:\t\t%s (%d)\n", port_phy_state_str(port_attr.phys_state), port_attr.phys_state); if (print_all_port_gids(ctx, port, port_attr.gid_tbl_len)) goto cleanup; } printf("\n"); } cleanup: if (ctx) if (ibv_close_device(ctx)) { fprintf(stderr, "Failed to close device"); rc = 3; } return rc; } static void usage(const char *argv0) { printf("Usage: %s print the ca attributes\n", argv0); printf("\n"); printf("Options:\n"); printf(" -d, --ib-dev= use IB device (default first device found)\n"); printf(" -i, --ib-port= use port of IB device (default all ports)\n"); printf(" -l, --list print only the IB devices names\n"); printf(" -v, --verbose print all the attributes of the IB device(s)\n"); } int main(int argc, char *argv[]) { char *ib_devname = NULL; int ret = 0; struct ibv_device **dev_list, **orig_dev_list; int num_of_hcas; int ib_port = 0; /* parse command line options */ while (1) { int c; static struct option long_options[] = { { .name = "ib-dev", .has_arg = 1, .val = 'd' }, { .name = "ib-port", .has_arg = 1, .val = 'i' }, { .name = "list", .has_arg = 0, .val = 'l' }, { .name = "verbose", .has_arg = 0, .val = 'v' }, { 0, 0, 0, 0} }; c = getopt_long(argc, argv, "d:i:lv", long_options, NULL); if (c == -1) break; switch (c) { case 'd': ib_devname = strdup(optarg); break; case 'i': ib_port = strtol(optarg, NULL, 0); if (ib_port < 0) { usage(argv[0]); return 1; } break; case 'v': verbose = 1; break; case 'l': dev_list = orig_dev_list = ibv_get_device_list(&num_of_hcas); if (!dev_list) { perror("Failed to get IB devices list"); return -1; } printf("%d HCA%s found:\n", num_of_hcas, num_of_hcas != 1 ? "s" : ""); while (*dev_list) { printf("\t%s\n", ibv_get_device_name(*dev_list)); ++dev_list; } printf("\n"); ibv_free_device_list(orig_dev_list); return 0; default: usage(argv[0]); return -1; } } dev_list = orig_dev_list = ibv_get_device_list(NULL); if (!dev_list) { perror("Failed to get IB devices list"); return -1; } if (ib_devname) { while (*dev_list) { if (!strcmp(ibv_get_device_name(*dev_list), ib_devname)) break; ++dev_list; } if (!*dev_list) { fprintf(stderr, "IB device '%s' wasn't found\n", ib_devname); return -1; } ret |= print_hca_cap(*dev_list, ib_port); } else { if (!*dev_list) { fprintf(stderr, "No IB devices found\n"); return -1; } while (*dev_list) { ret |= print_hca_cap(*dev_list, ib_port); ++dev_list; } } if (ib_devname) free(ib_devname); ibv_free_device_list(orig_dev_list); return ret; } Index: stable/9/contrib/ofed/libmlx4/src/mlx4-abi.h =================================================================== --- stable/9/contrib/ofed/libmlx4/src/mlx4-abi.h (revision 273863) +++ stable/9/contrib/ofed/libmlx4/src/mlx4-abi.h (revision 273864) @@ -1,109 +1,109 @@ /* * Copyright (c) 2007 Cisco, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef MLX4_ABI_H #define MLX4_ABI_H #include #define MLX4_UVERBS_MIN_ABI_VERSION 2 -#define MLX4_UVERBS_MAX_ABI_VERSION 3 +#define MLX4_UVERBS_MAX_ABI_VERSION 4 struct mlx4_alloc_ucontext_resp { struct ibv_get_context_resp ibv_resp; __u32 qp_tab_size; __u16 bf_reg_size; __u16 bf_regs_per_page; }; struct mlx4_alloc_pd_resp { struct ibv_alloc_pd_resp ibv_resp; __u32 pdn; __u32 reserved; }; struct mlx4_create_cq { struct ibv_create_cq ibv_cmd; __u64 buf_addr; __u64 db_addr; }; struct mlx4_create_cq_resp { struct ibv_create_cq_resp ibv_resp; __u32 cqn; __u32 reserved; }; struct mlx4_resize_cq { struct ibv_resize_cq ibv_cmd; __u64 buf_addr; }; #ifdef HAVE_IBV_XRC_OPS struct mlx4_create_xrc_srq { struct ibv_create_xrc_srq ibv_cmd; __u64 buf_addr; __u64 db_addr; }; #endif struct mlx4_create_srq { struct ibv_create_srq ibv_cmd; __u64 buf_addr; __u64 db_addr; }; struct mlx4_create_srq_resp { struct ibv_create_srq_resp ibv_resp; __u32 srqn; __u32 reserved; }; struct mlx4_create_qp { struct ibv_create_qp ibv_cmd; __u64 buf_addr; __u64 db_addr; __u8 log_sq_bb_count; __u8 log_sq_stride; __u8 sq_no_prefetch; /* was reserved in ABI 2 */ __u8 reserved[5]; }; #ifdef HAVE_IBV_XRC_OPS struct mlx4_open_xrc_domain_resp { struct ibv_open_xrc_domain_resp ibv_resp; __u32 xrcdn; __u32 reserved; }; #endif #endif /* MLX4_ABI_H */ Index: stable/9/contrib/ofed/libsdp/src/linux/sdp_inet.h =================================================================== --- stable/9/contrib/ofed/libsdp/src/linux/sdp_inet.h (revision 273863) +++ stable/9/contrib/ofed/libsdp/src/linux/sdp_inet.h (revision 273864) @@ -1,52 +1,56 @@ /* This software is available to you under a choice of one of two licenses. You may choose to be licensed under the terms of the GNU General Public License (GPL) Version 2, available at , or the OpenIB.org BSD license, available in the LICENSE.TXT file accompanying this software. These details are also available at . THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Copyright (c) 2004 Topspin Communications. All rights reserved. $Id$ */ #ifndef _SDP_INET_H #define _SDP_INET_H /* * constants shared between user and kernel space. */ #ifndef SOLARIS_BUILD +#ifdef __FreeBSD__ +#include +#else #define AF_INET_SDP 27 /* SDP socket protocol family */ #define AF_INET6_SDP 28 /* SDP socket protocol family */ +#endif #else #define AF_INET_SDP 31 /* This is an invalid family on native solaris * and will only work using QuickTransit */ //TODO XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXx #define AF_INET6_SDP 32 /* SDP socket protocol family */ #endif #define AF_INET_STR "AF_INET_SDP" /* SDP enabled environment variable */ #define AF_INET6_STR "AF_INET6_SDP" /* SDP enabled environment variable */ #ifndef SDP_ZCOPY_THRESH #define SDP_ZCOPY_THRESH 80 #endif #ifndef SDP_LAST_BIND_ERR #define SDP_LAST_BIND_ERR 81 #endif #endif /* _SDP_INET_H */ Index: stable/9/contrib/ofed/management/infiniband-diags/src/sminfo.c =================================================================== --- stable/9/contrib/ofed/management/infiniband-diags/src/sminfo.c (revision 273863) +++ stable/9/contrib/ofed/management/infiniband-diags/src/sminfo.c (revision 273864) @@ -1,206 +1,206 @@ /* * Copyright (c) 2004-2008 Voltaire Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ #if HAVE_CONFIG_H # include #endif /* HAVE_CONFIG_H */ #include #include #include #include #include #include #include #include #include #include "ibdiag_common.h" static uint8_t sminfo[1024]; char *argv0 = "sminfo"; static void usage(void) { fprintf(stderr, "Usage: %s [-d(ebug) -e(rr_show) -s state -p prio -a activity -D(irect) -G(uid) -V(ersion) -C ca_name -P ca_port " "-t(imeout) timeout_ms] [modifier]\n", argv0); exit(-1); } int strdata, xdata=1, bindata; enum { SMINFO_NOTACT, SMINFO_DISCOVER, SMINFO_STANDBY, SMINFO_MASTER, SMINFO_STATE_LAST, }; char *statestr[] = { - [SMINFO_NOTACT] "SMINFO_NOTACT", - [SMINFO_DISCOVER] "SMINFO_DISCOVER", - [SMINFO_STANDBY] "SMINFO_STANDBY", - [SMINFO_MASTER] "SMINFO_MASTER", + [SMINFO_NOTACT] = "SMINFO_NOTACT", + [SMINFO_DISCOVER] = "SMINFO_DISCOVER", + [SMINFO_STANDBY] = "SMINFO_STANDBY", + [SMINFO_MASTER] = "SMINFO_MASTER", }; #define STATESTR(s) (((unsigned)(s)) < SMINFO_STATE_LAST ? statestr[s] : "???") int main(int argc, char **argv) { int mgmt_classes[3] = {IB_SMI_CLASS, IB_SMI_DIRECT_CLASS, IB_SA_CLASS}; int mod = 0; ib_portid_t portid = {0}; int timeout = 0; /* use default */ uint8_t *p; unsigned act = 0; int prio = 0, state = SMINFO_STANDBY; uint64_t guid = 0, key = 0; extern int ibdebug; int dest_type = IB_DEST_LID; int udebug = 0; char *ca = 0; int ca_port = 0; static char const str_opts[] = "C:P:t:s:p:a:deDGVhu"; static const struct option long_opts[] = { { "C", 1, 0, 'C'}, { "P", 1, 0, 'P'}, { "debug", 0, 0, 'd'}, { "err_show", 0, 0, 'e'}, { "s", 1, 0, 's'}, { "p", 1, 0, 'p'}, { "a", 1, 0, 'a'}, { "Direct", 0, 0, 'D'}, { "Guid", 0, 0, 'G'}, { "Version", 0, 0, 'V'}, { "timeout", 1, 0, 't'}, { "help", 0, 0, 'h'}, { "usage", 0, 0, 'u'}, { } }; argv0 = argv[0]; while (1) { int ch = getopt_long(argc, argv, str_opts, long_opts, NULL); if ( ch == -1 ) break; switch(ch) { case 'C': ca = optarg; break; case 'P': ca_port = strtoul(optarg, 0, 0); break; case 'd': ibdebug++; madrpc_show_errors(1); umad_debug(udebug); udebug++; break; case 'e': madrpc_show_errors(1); break; case 'D': dest_type = IB_DEST_DRPATH; break; case 'G': dest_type = IB_DEST_GUID; break; case 't': timeout = strtoul(optarg, 0, 0); madrpc_set_timeout(timeout); break; case 'a': act = strtoul(optarg, 0, 0); break; case 's': state = strtoul(optarg, 0, 0); break; case 'p': prio = strtoul(optarg, 0, 0); break; case 'V': fprintf(stderr, "%s %s\n", argv0, get_build_version() ); exit(-1); default: usage(); break; } } argc -= optind; argv += optind; if (argc > 1) mod = atoi(argv[1]); madrpc_init(ca, ca_port, mgmt_classes, 3); if (argc) { if (ib_resolve_portid_str(&portid, argv[0], dest_type, 0) < 0) IBERROR("can't resolve destination port %s", argv[0]); } else { if (ib_resolve_smlid(&portid, timeout) < 0) IBERROR("can't resolve sm port %s", argv[0]); } mad_encode_field(sminfo, IB_SMINFO_GUID_F, &guid); mad_encode_field(sminfo, IB_SMINFO_ACT_F, &act); mad_encode_field(sminfo, IB_SMINFO_KEY_F, &key); mad_encode_field(sminfo, IB_SMINFO_PRIO_F, &prio); mad_encode_field(sminfo, IB_SMINFO_STATE_F, &state); if (mod) { if (!(p = smp_set(sminfo, &portid, IB_ATTR_SMINFO, mod, timeout))) IBERROR("query"); } else if (!(p = smp_query(sminfo, &portid, IB_ATTR_SMINFO, 0, timeout))) IBERROR("query"); mad_decode_field(sminfo, IB_SMINFO_GUID_F, &guid); mad_decode_field(sminfo, IB_SMINFO_ACT_F, &act); mad_decode_field(sminfo, IB_SMINFO_KEY_F, &key); mad_decode_field(sminfo, IB_SMINFO_PRIO_F, &prio); mad_decode_field(sminfo, IB_SMINFO_STATE_F, &state); printf("sminfo: sm lid %d sm guid 0x%" PRIx64 ", activity count %u priority %d state %d %s\n", portid.lid, guid, act, prio, state, STATESTR(state)); exit(0); } Index: stable/9/contrib/ofed/management/opensm/opensm/osm_console.c =================================================================== --- stable/9/contrib/ofed/management/opensm/opensm/osm_console.c (revision 273863) +++ stable/9/contrib/ofed/management/opensm/opensm/osm_console.c (revision 273864) @@ -1,1327 +1,1330 @@ /* * Copyright (c) 2005-2008 Voltaire, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ #if HAVE_CONFIG_H # include #endif /* HAVE_CONFIG_H */ #define _GNU_SOURCE /* for getline */ #include #include #include #include #include #include #ifdef ENABLE_OSM_CONSOLE_SOCKET #include #endif #include #include #include #include #include #include #include #include struct command { char *name; void (*help_function) (FILE * out, int detail); void (*parse_function) (char **p_last, osm_opensm_t * p_osm, FILE * out); }; static struct { int on; int delay_s; time_t previous; void (*loop_function) (osm_opensm_t * p_osm, FILE * out); } loop_command = { -on: 0, delay_s: 2, loop_function:NULL}; + .on = 0, + .delay_s = 2, + .loop_function = NULL, +}; static const struct command console_cmds[]; static inline char *next_token(char **p_last) { return strtok_r(NULL, " \t\n\r", p_last); } static void help_command(FILE * out, int detail) { int i; fprintf(out, "Supported commands and syntax:\n"); fprintf(out, "help []\n"); /* skip help command */ for (i = 1; console_cmds[i].name; i++) console_cmds[i].help_function(out, 0); } static void help_quit(FILE * out, int detail) { fprintf(out, "quit (not valid in local mode; use ctl-c)\n"); } static void help_loglevel(FILE * out, int detail) { fprintf(out, "loglevel []\n"); if (detail) { fprintf(out, " log-level is OR'ed from the following\n"); fprintf(out, " OSM_LOG_NONE 0x%02X\n", OSM_LOG_NONE); fprintf(out, " OSM_LOG_ERROR 0x%02X\n", OSM_LOG_ERROR); fprintf(out, " OSM_LOG_INFO 0x%02X\n", OSM_LOG_INFO); fprintf(out, " OSM_LOG_VERBOSE 0x%02X\n", OSM_LOG_VERBOSE); fprintf(out, " OSM_LOG_DEBUG 0x%02X\n", OSM_LOG_DEBUG); fprintf(out, " OSM_LOG_FUNCS 0x%02X\n", OSM_LOG_FUNCS); fprintf(out, " OSM_LOG_FRAMES 0x%02X\n", OSM_LOG_FRAMES); fprintf(out, " OSM_LOG_ROUTING 0x%02X\n", OSM_LOG_ROUTING); fprintf(out, " OSM_LOG_SYS 0x%02X\n", OSM_LOG_SYS); fprintf(out, "\n"); fprintf(out, " OSM_LOG_DEFAULT_LEVEL 0x%02X\n", OSM_LOG_DEFAULT_LEVEL); } } static void help_priority(FILE * out, int detail) { fprintf(out, "priority []\n"); } static void help_resweep(FILE * out, int detail) { fprintf(out, "resweep [heavy|light]\n"); } static void help_reroute(FILE * out, int detail) { fprintf(out, "reroute\n"); if (detail) { fprintf(out, "reroute the fabric\n"); } } static void help_status(FILE * out, int detail) { fprintf(out, "status [loop]\n"); if (detail) { fprintf(out, " loop -- type \"q\" to quit\n"); } } static void help_logflush(FILE * out, int detail) { fprintf(out, "logflush -- flush the opensm.log file\n"); } static void help_querylid(FILE * out, int detail) { fprintf(out, "querylid lid -- print internal information about the lid specified\n"); } static void help_portstatus(FILE * out, int detail) { fprintf(out, "portstatus [ca|switch|router]\n"); if (detail) { fprintf(out, "summarize port status\n"); fprintf(out, " [ca|switch|router] -- limit the results to the node type specified\n"); } } static void help_switchbalance(FILE * out, int detail) { fprintf(out, "switchbalance [verbose] [guid]\n"); if (detail) { fprintf(out, "output switch balancing information\n"); fprintf(out, " [verbose] -- verbose output\n" " [guid] -- limit results to specified guid\n"); } } static void help_lidbalance(FILE * out, int detail) { fprintf(out, "lidbalance [switchguid]\n"); if (detail) { fprintf(out, "output lid balanced forwarding information\n"); fprintf(out, " [switchguid] -- limit results to specified switch guid\n"); } } static void help_dump_conf(FILE *out, int detail) { fprintf(out, "dump_conf\n"); if (detail) { fprintf(out, "dump current opensm configuration\n"); } } #ifdef ENABLE_OSM_PERF_MGR static void help_perfmgr(FILE * out, int detail) { fprintf(out, "perfmgr [enable|disable|clear_counters|dump_counters|sweep_time[seconds]]\n"); if (detail) { fprintf(out, "perfmgr -- print the performance manager state\n"); fprintf(out, " [enable|disable] -- change the perfmgr state\n"); fprintf(out, " [sweep_time] -- change the perfmgr sweep time (requires [seconds] option)\n"); fprintf(out, " [clear_counters] -- clear the counters stored\n"); fprintf(out, " [dump_counters [mach]] -- dump the counters (optionally in [mach]ine readable format)\n"); fprintf(out, " [print_counters ] -- print the counters for the specified node\n"); } } #endif /* ENABLE_OSM_PERF_MGR */ /* more help routines go here */ static void help_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { char *p_cmd; int i, found = 0; p_cmd = next_token(p_last); if (!p_cmd) help_command(out, 0); else { for (i = 1; console_cmds[i].name; i++) { if (!strcmp(p_cmd, console_cmds[i].name)) { found = 1; console_cmds[i].help_function(out, 1); break; } } if (!found) { fprintf(out, "%s : Command not found\n\n", p_cmd); help_command(out, 0); } } } static void loglevel_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { char *p_cmd; int level; p_cmd = next_token(p_last); if (!p_cmd) fprintf(out, "Current log level is 0x%x\n", osm_log_get_level(&p_osm->log)); else { /* Handle x, 0x, and decimal specification of log level */ if (!strncmp(p_cmd, "x", 1)) { p_cmd++; level = strtoul(p_cmd, NULL, 16); } else { if (!strncmp(p_cmd, "0x", 2)) { p_cmd += 2; level = strtoul(p_cmd, NULL, 16); } else level = strtol(p_cmd, NULL, 10); } if ((level >= 0) && (level < 256)) { fprintf(out, "Setting log level to 0x%x\n", level); osm_log_set_level(&p_osm->log, level); } else fprintf(out, "Invalid log level 0x%x\n", level); } } static void priority_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { char *p_cmd; int priority; p_cmd = next_token(p_last); if (!p_cmd) fprintf(out, "Current sm-priority is %d\n", p_osm->subn.opt.sm_priority); else { priority = strtol(p_cmd, NULL, 0); if (0 > priority || 15 < priority) fprintf(out, "Invalid sm-priority %d; must be between 0 and 15\n", priority); else { fprintf(out, "Setting sm-priority to %d\n", priority); osm_set_sm_priority(&p_osm->sm, (uint8_t)priority); } } } static char *sm_state_str(int state) { switch (state) { case IB_SMINFO_STATE_DISCOVERING: return ("Discovering"); case IB_SMINFO_STATE_STANDBY: return ("Standby"); case IB_SMINFO_STATE_NOTACTIVE: return ("Not Active"); case IB_SMINFO_STATE_MASTER: return ("Master"); } return ("UNKNOWN"); } static char *sa_state_str(osm_sa_state_t state) { switch (state) { case OSM_SA_STATE_INIT: return ("Init"); case OSM_SA_STATE_READY: return ("Ready"); } return ("UNKNOWN"); } static void print_status(osm_opensm_t * p_osm, FILE * out) { cl_list_item_t *item; if (out) { cl_plock_acquire(&p_osm->lock); fprintf(out, " OpenSM Version : %s\n", p_osm->osm_version); fprintf(out, " SM State : %s\n", sm_state_str(p_osm->subn.sm_state)); fprintf(out, " SA State : %s\n", sa_state_str(p_osm->sa.state)); fprintf(out, " Routing Engine : %s\n", osm_routing_engine_type_str(p_osm-> routing_engine_used)); fprintf(out, " Loaded event plugins :"); if (cl_qlist_head(&p_osm->plugin_list) == cl_qlist_end(&p_osm->plugin_list)) { fprintf(out, " "); } for (item = cl_qlist_head(&p_osm->plugin_list); item != cl_qlist_end(&p_osm->plugin_list); item = cl_qlist_next(item)) fprintf(out, " %s", ((osm_epi_plugin_t *)item)->plugin_name); fprintf(out, "\n"); #ifdef ENABLE_OSM_PERF_MGR fprintf(out, "\n PerfMgr state/sweep state : %s/%s\n", osm_perfmgr_get_state_str(&(p_osm->perfmgr)), osm_perfmgr_get_sweep_state_str(&(p_osm->perfmgr))); #endif fprintf(out, "\n MAD stats\n" " ---------\n" " QP0 MADs outstanding : %d\n" " QP0 MADs outstanding (on wire) : %d\n" " QP0 MADs rcvd : %d\n" " QP0 MADs sent : %d\n" " QP0 unicasts sent : %d\n" " QP0 unknown MADs rcvd : %d\n" " SA MADs outstanding : %d\n" " SA MADs rcvd : %d\n" " SA MADs sent : %d\n" " SA unknown MADs rcvd : %d\n" " SA MADs ignored : %d\n", p_osm->stats.qp0_mads_outstanding, p_osm->stats.qp0_mads_outstanding_on_wire, p_osm->stats.qp0_mads_rcvd, p_osm->stats.qp0_mads_sent, p_osm->stats.qp0_unicasts_sent, p_osm->stats.qp0_mads_rcvd_unknown, p_osm->stats.sa_mads_outstanding, p_osm->stats.sa_mads_rcvd, p_osm->stats.sa_mads_sent, p_osm->stats.sa_mads_rcvd_unknown, p_osm->stats.sa_mads_ignored); fprintf(out, "\n Subnet flags\n" " ------------\n" " Ignore existing lfts : %d\n" " Subnet Init errors : %d\n" " In sweep hop 0 : %d\n" " First time master sweep : %d\n" " Coming out of standby : %d\n", p_osm->subn.ignore_existing_lfts, p_osm->subn.subnet_initialization_error, p_osm->subn.in_sweep_hop_0, p_osm->subn.first_time_master_sweep, p_osm->subn.coming_out_of_standby); fprintf(out, "\n"); cl_plock_release(&p_osm->lock); } } static int loop_command_check_time(void) { time_t cur = time(NULL); if ((loop_command.previous + loop_command.delay_s) < cur) { loop_command.previous = cur; return (1); } return (0); } static void status_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { char *p_cmd; p_cmd = next_token(p_last); if (p_cmd) { if (strcmp(p_cmd, "loop") == 0) { fprintf(out, "Looping on status command...\n"); fflush(out); loop_command.on = 1; loop_command.previous = time(NULL); loop_command.loop_function = print_status; } else { help_status(out, 1); return; } } print_status(p_osm, out); } static void resweep_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { char *p_cmd; p_cmd = next_token(p_last); if (!p_cmd || (strcmp(p_cmd, "heavy") != 0 && strcmp(p_cmd, "light") != 0)) { fprintf(out, "Invalid resweep command\n"); help_resweep(out, 1); } else { if (strcmp(p_cmd, "heavy") == 0) p_osm->subn.force_heavy_sweep = TRUE; osm_opensm_sweep(p_osm); } } static void reroute_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { p_osm->subn.force_reroute = TRUE; osm_opensm_sweep(p_osm); } static void logflush_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { fflush(p_osm->log.out_port); } static void querylid_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { int p = 0; uint16_t lid = 0; osm_port_t *p_port = NULL; char *p_cmd = next_token(p_last); if (!p_cmd) { fprintf(out, "no LID specified\n"); help_querylid(out, 1); return; } lid = (uint16_t) strtoul(p_cmd, NULL, 0); cl_plock_acquire(&p_osm->lock); if (lid > cl_ptr_vector_get_capacity(&(p_osm->subn.port_lid_tbl))) goto invalid_lid; p_port = cl_ptr_vector_get(&(p_osm->subn.port_lid_tbl), lid); if (!p_port) goto invalid_lid; fprintf(out, "Query results for LID %u\n", lid); fprintf(out, " GUID : 0x%016" PRIx64 "\n" " Node Desc : %s\n" " Node Type : %s\n" " Num Ports : %d\n", cl_ntoh64(p_port->guid), p_port->p_node->print_desc, ib_get_node_type_str(osm_node_get_type(p_port->p_node)), p_port->p_node->node_info.num_ports); if (p_port->p_node->sw) p = 0; else p = 1; for ( /* see above */ ; p < p_port->p_node->physp_tbl_size; p++) { fprintf(out, " Port %d health : %s\n", p, p_port->p_node->physp_table[p]. healthy ? "OK" : "ERROR"); } cl_plock_release(&p_osm->lock); return; invalid_lid: cl_plock_release(&p_osm->lock); fprintf(out, "Invalid lid %d\n", lid); return; } /** * Data structures for the portstatus command */ typedef struct _port_report { struct _port_report *next; uint64_t node_guid; uint8_t port_num; char print_desc[IB_NODE_DESCRIPTION_SIZE + 1]; } port_report_t; static void __tag_port_report(port_report_t ** head, uint64_t node_guid, uint8_t port_num, char *print_desc) { port_report_t *rep = malloc(sizeof(*rep)); if (!rep) return; rep->node_guid = node_guid; rep->port_num = port_num; memcpy(rep->print_desc, print_desc, IB_NODE_DESCRIPTION_SIZE + 1); rep->next = NULL; if (*head) { rep->next = *head; *head = rep; } else *head = rep; } static void __print_port_report(FILE * out, port_report_t * head) { port_report_t *item = head; while (item != NULL) { fprintf(out, " 0x%016" PRIx64 " %d (%s)\n", item->node_guid, item->port_num, item->print_desc); port_report_t *next = item->next; free(item); item = next; } } typedef struct { uint8_t node_type_lim; /* limit the results; 0 == ALL */ uint64_t total_nodes; uint64_t total_ports; uint64_t ports_down; uint64_t ports_active; uint64_t ports_disabled; port_report_t *disabled_ports; uint64_t ports_1X; uint64_t ports_4X; uint64_t ports_8X; uint64_t ports_12X; uint64_t ports_unknown_width; uint64_t ports_reduced_width; port_report_t *reduced_width_ports; uint64_t ports_sdr; uint64_t ports_ddr; uint64_t ports_qdr; uint64_t ports_unknown_speed; uint64_t ports_reduced_speed; port_report_t *reduced_speed_ports; } fabric_stats_t; /** * iterator function to get portstatus on each node */ static void __get_stats(cl_map_item_t * const p_map_item, void *context) { fabric_stats_t *fs = (fabric_stats_t *) context; osm_node_t *node = (osm_node_t *) p_map_item; uint8_t num_ports = osm_node_get_num_physp(node); uint8_t port = 0; /* Skip nodes we are not interested in */ if (fs->node_type_lim != 0 && fs->node_type_lim != node->node_info.node_type) return; fs->total_nodes++; for (port = 1; port < num_ports; port++) { osm_physp_t *phys = osm_node_get_physp_ptr(node, port); ib_port_info_t *pi = NULL; uint8_t active_speed = 0; uint8_t enabled_speed = 0; uint8_t active_width = 0; uint8_t enabled_width = 0; uint8_t port_state = 0; uint8_t port_phys_state = 0; if (!phys) continue; pi = &(phys->port_info); active_speed = ib_port_info_get_link_speed_active(pi); enabled_speed = ib_port_info_get_link_speed_enabled(pi); active_width = pi->link_width_active; enabled_width = pi->link_width_enabled; port_state = ib_port_info_get_port_state(pi); port_phys_state = ib_port_info_get_port_phys_state(pi); if ((enabled_width ^ active_width) > active_width) { __tag_port_report(&(fs->reduced_width_ports), cl_ntoh64(node->node_info.node_guid), port, node->print_desc); fs->ports_reduced_width++; } if ((enabled_speed ^ active_speed) > active_speed) { __tag_port_report(&(fs->reduced_speed_ports), cl_ntoh64(node->node_info.node_guid), port, node->print_desc); fs->ports_reduced_speed++; } switch (active_speed) { case IB_LINK_SPEED_ACTIVE_2_5: fs->ports_sdr++; break; case IB_LINK_SPEED_ACTIVE_5: fs->ports_ddr++; break; case IB_LINK_SPEED_ACTIVE_10: fs->ports_qdr++; break; default: fs->ports_unknown_speed++; break; } switch (active_width) { case IB_LINK_WIDTH_ACTIVE_1X: fs->ports_1X++; break; case IB_LINK_WIDTH_ACTIVE_4X: fs->ports_4X++; break; case IB_LINK_WIDTH_ACTIVE_8X: fs->ports_8X++; break; case IB_LINK_WIDTH_ACTIVE_12X: fs->ports_12X++; break; default: fs->ports_unknown_width++; break; } if (port_state == IB_LINK_DOWN) fs->ports_down++; else if (port_state == IB_LINK_ACTIVE) fs->ports_active++; if (port_phys_state == IB_PORT_PHYS_STATE_DISABLED) { __tag_port_report(&(fs->disabled_ports), cl_ntoh64(node->node_info.node_guid), port, node->print_desc); fs->ports_disabled++; } fs->total_ports++; } } static void portstatus_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { fabric_stats_t fs; struct timeval before, after; char *p_cmd; memset(&fs, 0, sizeof(fs)); p_cmd = next_token(p_last); if (p_cmd) { if (strcmp(p_cmd, "ca") == 0) { fs.node_type_lim = IB_NODE_TYPE_CA; } else if (strcmp(p_cmd, "switch") == 0) { fs.node_type_lim = IB_NODE_TYPE_SWITCH; } else if (strcmp(p_cmd, "router") == 0) { fs.node_type_lim = IB_NODE_TYPE_ROUTER; } else { fprintf(out, "Node type not understood\n"); help_portstatus(out, 1); return; } } gettimeofday(&before, NULL); /* for each node in the system gather the stats */ cl_plock_acquire(&p_osm->lock); cl_qmap_apply_func(&(p_osm->subn.node_guid_tbl), __get_stats, (void *)&fs); cl_plock_release(&p_osm->lock); gettimeofday(&after, NULL); /* report the stats */ fprintf(out, "\"%s\" port status:\n", fs.node_type_lim ? ib_get_node_type_str(fs. node_type_lim) : "ALL"); fprintf(out, " %" PRIu64 " port(s) scanned on %" PRIu64 " nodes in %lu us\n", fs.total_ports, fs.total_nodes, after.tv_usec - before.tv_usec); if (fs.ports_down) fprintf(out, " %" PRIu64 " down\n", fs.ports_down); if (fs.ports_active) fprintf(out, " %" PRIu64 " active\n", fs.ports_active); if (fs.ports_1X) fprintf(out, " %" PRIu64 " at 1X\n", fs.ports_1X); if (fs.ports_4X) fprintf(out, " %" PRIu64 " at 4X\n", fs.ports_4X); if (fs.ports_8X) fprintf(out, " %" PRIu64 " at 8X\n", fs.ports_8X); if (fs.ports_12X) fprintf(out, " %" PRIu64 " at 12X\n", fs.ports_12X); if (fs.ports_sdr) fprintf(out, " %" PRIu64 " at 2.5 Gbps\n", fs.ports_sdr); if (fs.ports_ddr) fprintf(out, " %" PRIu64 " at 5.0 Gbps\n", fs.ports_ddr); if (fs.ports_qdr) fprintf(out, " %" PRIu64 " at 10.0 Gbps\n", fs.ports_qdr); if (fs.ports_disabled + fs.ports_reduced_speed + fs.ports_reduced_width > 0) { fprintf(out, "\nPossible issues:\n"); } if (fs.ports_disabled) { fprintf(out, " %" PRIu64 " disabled\n", fs.ports_disabled); __print_port_report(out, fs.disabled_ports); } if (fs.ports_reduced_speed) { fprintf(out, " %" PRIu64 " with reduced speed\n", fs.ports_reduced_speed); __print_port_report(out, fs.reduced_speed_ports); } if (fs.ports_reduced_width) { fprintf(out, " %" PRIu64 " with reduced width\n", fs.ports_reduced_width); __print_port_report(out, fs.reduced_width_ports); } fprintf(out, "\n"); } static void switchbalance_check(osm_opensm_t * p_osm, osm_switch_t * p_sw, FILE * out, int verbose) { uint8_t port_num; uint8_t num_ports; const cl_qmap_t *p_port_tbl; osm_port_t *p_port; osm_physp_t *p_physp; osm_physp_t *p_rem_physp; osm_node_t *p_rem_node; uint32_t count[255]; /* max ports is a uint8_t */ uint8_t output_ports[255]; uint8_t output_ports_count = 0; uint32_t min_count = 0xFFFFFFFF; uint32_t max_count = 0; unsigned int i; memset(count, '\0', sizeof(uint32_t) * 255); /* Count port usage */ p_port_tbl = &p_osm->subn.port_guid_tbl; for (p_port = (osm_port_t *) cl_qmap_head(p_port_tbl); p_port != (osm_port_t *) cl_qmap_end(p_port_tbl); p_port = (osm_port_t *) cl_qmap_next(&p_port->map_item)) { uint16_t min_lid_ho; uint16_t max_lid_ho; uint16_t lid_ho; /* Don't count switches in port usage */ if (osm_node_get_type(p_port->p_node) == IB_NODE_TYPE_SWITCH) continue; osm_port_get_lid_range_ho(p_port, &min_lid_ho, &max_lid_ho); if (min_lid_ho == 0 || max_lid_ho == 0) continue; for (lid_ho = min_lid_ho; lid_ho <= max_lid_ho; lid_ho++) { port_num = osm_switch_get_port_by_lid(p_sw, lid_ho); if (port_num == OSM_NO_PATH) continue; count[port_num]++; } } num_ports = p_sw->num_ports; for (port_num = 1; port_num < num_ports; port_num++) { p_physp = osm_node_get_physp_ptr(p_sw->p_node, port_num); /* if port is down/unhealthy, don't consider it in * min/max calculations */ if (!p_physp || !osm_physp_is_healthy(p_physp) || !osm_physp_get_remote(p_physp)) continue; p_rem_physp = osm_physp_get_remote(p_physp); p_rem_node = osm_physp_get_node_ptr(p_rem_physp); /* If we are directly connected to a CA/router, its not really * up for balancing consideration. */ if (osm_node_get_type(p_rem_node) != IB_NODE_TYPE_SWITCH) continue; output_ports[output_ports_count] = port_num; output_ports_count++; if (count[port_num] < min_count) min_count = count[port_num]; if (count[port_num] > max_count) max_count = count[port_num]; } if (verbose || ((max_count - min_count) > 1)) { if ((max_count - min_count) > 1) fprintf(out, "Unbalanced Switch: 0x%016" PRIx64 " (%s)\n", cl_ntoh64(p_sw->p_node->node_info.node_guid), p_sw->p_node->print_desc); else fprintf(out, "Switch: 0x%016" PRIx64 " (%s)\n", cl_ntoh64(p_sw->p_node->node_info.node_guid), p_sw->p_node->print_desc); for (i = 0; i < output_ports_count; i++) { fprintf(out, "Port %d: %d\n", output_ports[i], count[output_ports[i]]); } } } static void switchbalance_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { char *p_cmd; uint64_t guid = 0; osm_switch_t *p_sw; int verbose = 0; p_cmd = next_token(p_last); if (p_cmd) { char *p_end; if (strcmp(p_cmd, "verbose") == 0) { verbose++; p_cmd = next_token(p_last); } if (p_cmd) { guid = strtoull(p_cmd, &p_end, 0); if (!guid || *p_end != '\0') { fprintf(out, "Invalid guid specified\n"); help_switchbalance(out, 1); return; } } } cl_plock_acquire(&p_osm->lock); if (guid) { p_sw = osm_get_switch_by_guid(&p_osm->subn, cl_hton64(guid)); if (!p_sw) { fprintf(out, "guid not found\n"); goto lock_exit; } switchbalance_check(p_osm, p_sw, out, verbose); } else { cl_qmap_t *p_sw_guid_tbl = &p_osm->subn.sw_guid_tbl; for (p_sw = (osm_switch_t *) cl_qmap_head(p_sw_guid_tbl); p_sw != (osm_switch_t *) cl_qmap_end(p_sw_guid_tbl); p_sw = (osm_switch_t *) cl_qmap_next(&p_sw->map_item)) switchbalance_check(p_osm, p_sw, out, verbose); } lock_exit: cl_plock_release(&p_osm->lock); return; } static void lidbalance_check(osm_opensm_t * p_osm, osm_switch_t * p_sw, FILE * out) { uint8_t port_num; const cl_qmap_t *p_port_tbl; osm_port_t *p_port; p_port_tbl = &p_osm->subn.port_guid_tbl; for (p_port = (osm_port_t *) cl_qmap_head(p_port_tbl); p_port != (osm_port_t *) cl_qmap_end(p_port_tbl); p_port = (osm_port_t *) cl_qmap_next(&p_port->map_item)) { uint32_t port_count[255]; /* max ports is a uint8_t */ osm_node_t *rem_node[255]; uint32_t rem_node_count; uint32_t rem_count[255]; osm_physp_t *p_physp; osm_physp_t *p_rem_physp; osm_node_t *p_rem_node; uint32_t port_min_count = 0xFFFFFFFF; uint32_t port_max_count = 0; uint32_t rem_min_count = 0xFFFFFFFF; uint32_t rem_max_count = 0; uint16_t min_lid_ho; uint16_t max_lid_ho; uint16_t lid_ho; uint8_t num_ports; unsigned int i; /* we only care about non-switches */ if (osm_node_get_type(p_port->p_node) == IB_NODE_TYPE_SWITCH) continue; osm_port_get_lid_range_ho(p_port, &min_lid_ho, &max_lid_ho); if (min_lid_ho == 0 || max_lid_ho == 0) continue; memset(port_count, '\0', sizeof(uint32_t) * 255); memset(rem_node, '\0', sizeof(osm_node_t *) * 255); rem_node_count = 0; memset(rem_count, '\0', sizeof(uint32_t) * 255); for (lid_ho = min_lid_ho; lid_ho <= max_lid_ho; lid_ho++) { boolean_t rem_node_found = FALSE; unsigned int indx = 0; port_num = osm_switch_get_port_by_lid(p_sw, lid_ho); if (port_num == OSM_NO_PATH) continue; p_physp = osm_node_get_physp_ptr(p_sw->p_node, port_num); /* if port is down/unhealthy, can't calculate */ if (!p_physp || !osm_physp_is_healthy(p_physp) || !osm_physp_get_remote(p_physp)) continue; p_rem_physp = osm_physp_get_remote(p_physp); p_rem_node = osm_physp_get_node_ptr(p_rem_physp); /* determine if we've seen this remote node before. * If not, store it. If yes, update the counter */ for (i = 0; i < rem_node_count; i++) { if (rem_node[i] == p_rem_node) { rem_node_found = TRUE; indx = i; break; } } if (!rem_node_found) { rem_node[rem_node_count] = p_rem_node; rem_count[rem_node_count]++; indx = rem_node_count; rem_node_count++; } else rem_count[indx]++; port_count[port_num]++; } if (!rem_node_count) continue; for (i = 0; i < rem_node_count; i++) { if (rem_count[i] < rem_min_count) rem_min_count = rem_count[i]; if (rem_count[i] > rem_max_count) rem_max_count = rem_count[i]; } num_ports = p_sw->num_ports; for (i = 0; i < num_ports; i++) { if (!port_count[i]) continue; if (port_count[i] < port_min_count) port_min_count = port_count[i]; if (port_count[i] > port_max_count) port_max_count = port_count[i]; } /* Output if this CA/router is being forwarded an unbalanced number of * times to a destination. */ if ((rem_max_count - rem_min_count) > 1) { fprintf(out, "Unbalanced Remote Forwarding: Switch 0x%016" PRIx64 " (%s): ", cl_ntoh64(p_sw->p_node->node_info.node_guid), p_sw->p_node->print_desc); if (osm_node_get_type(p_port->p_node) == IB_NODE_TYPE_CA) fprintf(out, "CA"); else if (osm_node_get_type(p_port->p_node) == IB_NODE_TYPE_ROUTER) fprintf(out, "Router"); fprintf(out, " 0x%016" PRIx64 " (%s): ", cl_ntoh64(p_port->p_node->node_info.node_guid), p_port->p_node->print_desc); for (i = 0; i < rem_node_count; i++) { fprintf(out, "Dest 0x%016" PRIx64 "(%s) - %u ", cl_ntoh64(rem_node[i]->node_info. node_guid), rem_node[i]->print_desc, rem_count[i]); } fprintf(out, "\n"); } /* Output if this CA/router is being forwarded through a port * an unbalanced number of times. */ if ((port_max_count - port_min_count) > 1) { fprintf(out, "Unbalanced Port Forwarding: Switch 0x%016" PRIx64 " (%s): ", cl_ntoh64(p_sw->p_node->node_info.node_guid), p_sw->p_node->print_desc); if (osm_node_get_type(p_port->p_node) == IB_NODE_TYPE_CA) fprintf(out, "CA"); else if (osm_node_get_type(p_port->p_node) == IB_NODE_TYPE_ROUTER) fprintf(out, "Router"); fprintf(out, " 0x%016" PRIx64 " (%s): ", cl_ntoh64(p_port->p_node->node_info.node_guid), p_port->p_node->print_desc); for (i = 0; i < num_ports; i++) { if (!port_count[i]) continue; fprintf(out, "Port %u - %u: ", i, port_count[i]); } fprintf(out, "\n"); } } } static void lidbalance_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { char *p_cmd; uint64_t guid = 0; osm_switch_t *p_sw; p_cmd = next_token(p_last); if (p_cmd) { char *p_end; guid = strtoull(p_cmd, &p_end, 0); if (!guid || *p_end != '\0') { fprintf(out, "Invalid switchguid specified\n"); help_lidbalance(out, 1); return; } } cl_plock_acquire(&p_osm->lock); if (guid) { p_sw = osm_get_switch_by_guid(&p_osm->subn, cl_hton64(guid)); if (!p_sw) { fprintf(out, "switchguid not found\n"); goto lock_exit; } lidbalance_check(p_osm, p_sw, out); } else { cl_qmap_t *p_sw_guid_tbl = &p_osm->subn.sw_guid_tbl; for (p_sw = (osm_switch_t *) cl_qmap_head(p_sw_guid_tbl); p_sw != (osm_switch_t *) cl_qmap_end(p_sw_guid_tbl); p_sw = (osm_switch_t *) cl_qmap_next(&p_sw->map_item)) lidbalance_check(p_osm, p_sw, out); } lock_exit: cl_plock_release(&p_osm->lock); return; } static void dump_conf_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { osm_subn_output_conf(out, &p_osm->subn.opt); } #ifdef ENABLE_OSM_PERF_MGR static void perfmgr_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { char *p_cmd; p_cmd = next_token(p_last); if (p_cmd) { if (strcmp(p_cmd, "enable") == 0) { osm_perfmgr_set_state(&(p_osm->perfmgr), PERFMGR_STATE_ENABLED); } else if (strcmp(p_cmd, "disable") == 0) { osm_perfmgr_set_state(&(p_osm->perfmgr), PERFMGR_STATE_DISABLE); } else if (strcmp(p_cmd, "clear_counters") == 0) { osm_perfmgr_clear_counters(&(p_osm->perfmgr)); } else if (strcmp(p_cmd, "dump_counters") == 0) { p_cmd = next_token(p_last); if (p_cmd && (strcmp(p_cmd, "mach") == 0)) { osm_perfmgr_dump_counters(&(p_osm->perfmgr), PERFMGR_EVENT_DB_DUMP_MR); } else { osm_perfmgr_dump_counters(&(p_osm->perfmgr), PERFMGR_EVENT_DB_DUMP_HR); } } else if (strcmp(p_cmd, "print_counters") == 0) { p_cmd = next_token(p_last); if (p_cmd) { osm_perfmgr_print_counters(&(p_osm->perfmgr), p_cmd, out); } else { fprintf(out, "print_counters requires a node name to be specified\n"); } } else if (strcmp(p_cmd, "sweep_time") == 0) { p_cmd = next_token(p_last); if (p_cmd) { uint16_t time_s = atoi(p_cmd); osm_perfmgr_set_sweep_time_s(&(p_osm->perfmgr), time_s); } else { fprintf(out, "sweep_time requires a time period (in seconds) to be specified\n"); } } else { fprintf(out, "\"%s\" option not found\n", p_cmd); } } else { fprintf(out, "Performance Manager status:\n" "state : %s\n" "sweep state : %s\n" "sweep time : %us\n" "outstanding queries/max : %d/%u\n", osm_perfmgr_get_state_str(&(p_osm->perfmgr)), osm_perfmgr_get_sweep_state_str(&(p_osm->perfmgr)), osm_perfmgr_get_sweep_time_s(&(p_osm->perfmgr)), p_osm->perfmgr.outstanding_queries, p_osm->perfmgr.max_outstanding_queries); } } #endif /* ENABLE_OSM_PERF_MGR */ static void quit_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { osm_console_exit(&p_osm->console, &p_osm->log); } static void help_version(FILE * out, int detail) { fprintf(out, "version -- print the OSM version\n"); } static void version_parse(char **p_last, osm_opensm_t * p_osm, FILE * out) { fprintf(out, "%s build %s %s\n", p_osm->osm_version, __DATE__, __TIME__); } /* more parse routines go here */ static const struct command console_cmds[] = { {"help", &help_command, &help_parse}, {"quit", &help_quit, &quit_parse}, {"loglevel", &help_loglevel, &loglevel_parse}, {"priority", &help_priority, &priority_parse}, {"resweep", &help_resweep, &resweep_parse}, {"reroute", &help_reroute, &reroute_parse}, {"status", &help_status, &status_parse}, {"logflush", &help_logflush, &logflush_parse}, {"querylid", &help_querylid, &querylid_parse}, {"portstatus", &help_portstatus, &portstatus_parse}, {"switchbalance", &help_switchbalance, &switchbalance_parse}, {"lidbalance", &help_lidbalance, &lidbalance_parse}, {"dump_conf", &help_dump_conf, &dump_conf_parse}, {"version", &help_version, &version_parse}, #ifdef ENABLE_OSM_PERF_MGR {"perfmgr", &help_perfmgr, &perfmgr_parse}, #endif /* ENABLE_OSM_PERF_MGR */ {NULL, NULL, NULL} /* end of array */ }; static void parse_cmd_line(char *line, osm_opensm_t * p_osm) { char *p_cmd, *p_last; int i, found = 0; FILE *out = p_osm->console.out; while (isspace(*line)) line++; if (!*line) return; /* find first token which is the command */ p_cmd = strtok_r(line, " \t\n\r", &p_last); if (p_cmd) { for (i = 0; console_cmds[i].name; i++) { if (loop_command.on) { if (!strcmp(p_cmd, "q")) { loop_command.on = 0; } found = 1; break; } if (!strcmp(p_cmd, console_cmds[i].name)) { found = 1; console_cmds[i].parse_function(&p_last, p_osm, out); break; } } if (!found) { fprintf(out, "%s : Command not found\n\n", p_cmd); help_command(out, 0); } } else { fprintf(out, "Error parsing command line: `%s'\n", line); } if (loop_command.on) { fprintf(out, "use \"q\" to quit loop\n"); fflush(out); } } void osm_console(osm_opensm_t * p_osm) { struct pollfd pollfd[2]; char *p_line; size_t len; ssize_t n; struct pollfd *fds; nfds_t nfds; osm_console_t *p_oct = &p_osm->console; osm_log_t *p_log = &p_osm->log; pollfd[0].fd = p_oct->socket; pollfd[0].events = POLLIN; pollfd[0].revents = 0; pollfd[1].fd = p_oct->in_fd; pollfd[1].events = POLLIN; pollfd[1].revents = 0; fds = p_oct->socket < 0 ? &pollfd[1] : pollfd; nfds = p_oct->socket < 0 || pollfd[1].fd < 0 ? 1 : 2; if (loop_command.on && loop_command_check_time() && loop_command.loop_function) { if (p_oct->out) { loop_command.loop_function(p_osm, p_oct->out); fflush(p_oct->out); } else { loop_command.on = 0; } } if (poll(fds, nfds, 1000) <= 0) return; #ifdef ENABLE_OSM_CONSOLE_SOCKET if (pollfd[0].revents & POLLIN) { int new_fd = 0; struct sockaddr_in sin; socklen_t len = sizeof(sin); struct hostent *hent; if ((new_fd = accept(p_oct->socket, &sin, &len)) < 0) { OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4B04: Failed to accept console socket: %s\n", strerror(errno)); p_oct->in_fd = -1; return; } if (inet_ntop (AF_INET, &sin.sin_addr, p_oct->client_ip, sizeof(p_oct->client_ip)) == NULL) { snprintf(p_oct->client_ip, 64, "STRING_UNKNOWN"); } if ((hent = gethostbyaddr((const char *)&sin.sin_addr, sizeof(struct in_addr), AF_INET)) == NULL) { snprintf(p_oct->client_hn, 128, "STRING_UNKNOWN"); } else { snprintf(p_oct->client_hn, 128, "%s", hent->h_name); } if (is_authorized(p_oct)) { cio_open(p_oct, new_fd, p_log); } else { OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4B05: Console connection denied: %s (%s)\n", p_oct->client_hn, p_oct->client_ip); close(new_fd); } return; } #endif if (pollfd[1].revents & POLLIN) { p_line = NULL; /* Get input line */ n = getline(&p_line, &len, p_oct->in); if (n > 0) { /* Parse and act on input */ parse_cmd_line(p_line, p_osm); if (!loop_command.on) { osm_console_prompt(p_oct->out); } } else osm_console_exit(p_oct, p_log); if (p_line) free(p_line); } } Index: stable/9/contrib/ofed/management/opensm/opensm/osm_subnet.c =================================================================== --- stable/9/contrib/ofed/management/opensm/opensm/osm_subnet.c (revision 273863) +++ stable/9/contrib/ofed/management/opensm/opensm/osm_subnet.c (revision 273864) @@ -1,1719 +1,1719 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ /* * Abstract: * Implementation of osm_subn_t. * This object represents an IBA subnet. * This object is part of the opensm family of objects. */ #if HAVE_CONFIG_H # include #endif /* HAVE_CONFIG_H */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static const char null_str[] = "(null)"; /********************************************************************** **********************************************************************/ void osm_subn_construct(IN osm_subn_t * const p_subn) { memset(p_subn, 0, sizeof(*p_subn)); cl_ptr_vector_construct(&p_subn->port_lid_tbl); cl_qmap_init(&p_subn->sw_guid_tbl); cl_qmap_init(&p_subn->node_guid_tbl); cl_qmap_init(&p_subn->port_guid_tbl); cl_qmap_init(&p_subn->sm_guid_tbl); cl_qlist_init(&p_subn->sa_sr_list); cl_qlist_init(&p_subn->sa_infr_list); cl_qlist_init(&p_subn->prefix_routes_list); cl_qmap_init(&p_subn->rtr_guid_tbl); cl_qmap_init(&p_subn->prtn_pkey_tbl); } /********************************************************************** **********************************************************************/ void osm_subn_destroy(IN osm_subn_t * const p_subn) { int i; osm_node_t *p_node, *p_next_node; osm_port_t *p_port, *p_next_port; osm_switch_t *p_sw, *p_next_sw; osm_remote_sm_t *p_rsm, *p_next_rsm; osm_prtn_t *p_prtn, *p_next_prtn; osm_mgrp_t *p_mgrp; osm_infr_t *p_infr, *p_next_infr; /* it might be a good idea to de-allocate all known objects */ p_next_node = (osm_node_t *) cl_qmap_head(&p_subn->node_guid_tbl); while (p_next_node != (osm_node_t *) cl_qmap_end(&p_subn->node_guid_tbl)) { p_node = p_next_node; p_next_node = (osm_node_t *) cl_qmap_next(&p_node->map_item); osm_node_delete(&p_node); } p_next_port = (osm_port_t *) cl_qmap_head(&p_subn->port_guid_tbl); while (p_next_port != (osm_port_t *) cl_qmap_end(&p_subn->port_guid_tbl)) { p_port = p_next_port; p_next_port = (osm_port_t *) cl_qmap_next(&p_port->map_item); osm_port_delete(&p_port); } p_next_sw = (osm_switch_t *) cl_qmap_head(&p_subn->sw_guid_tbl); while (p_next_sw != (osm_switch_t *) cl_qmap_end(&p_subn->sw_guid_tbl)) { p_sw = p_next_sw; p_next_sw = (osm_switch_t *) cl_qmap_next(&p_sw->map_item); osm_switch_delete(&p_sw); } p_next_rsm = (osm_remote_sm_t *) cl_qmap_head(&p_subn->sm_guid_tbl); while (p_next_rsm != (osm_remote_sm_t *) cl_qmap_end(&p_subn->sm_guid_tbl)) { p_rsm = p_next_rsm; p_next_rsm = (osm_remote_sm_t *) cl_qmap_next(&p_rsm->map_item); free(p_rsm); } p_next_prtn = (osm_prtn_t *) cl_qmap_head(&p_subn->prtn_pkey_tbl); while (p_next_prtn != (osm_prtn_t *) cl_qmap_end(&p_subn->prtn_pkey_tbl)) { p_prtn = p_next_prtn; p_next_prtn = (osm_prtn_t *) cl_qmap_next(&p_prtn->map_item); osm_prtn_delete(&p_prtn); } for (i = 0; i <= p_subn->max_mcast_lid_ho - IB_LID_MCAST_START_HO; i++) { p_mgrp = p_subn->mgroups[i]; p_subn->mgroups[i] = NULL; if (p_mgrp) osm_mgrp_delete(p_mgrp); } p_next_infr = (osm_infr_t *) cl_qlist_head(&p_subn->sa_infr_list); while (p_next_infr != (osm_infr_t *) cl_qlist_end(&p_subn->sa_infr_list)) { p_infr = p_next_infr; p_next_infr = (osm_infr_t *) cl_qlist_next(&p_infr->list_item); osm_infr_delete(p_infr); } cl_ptr_vector_destroy(&p_subn->port_lid_tbl); osm_qos_policy_destroy(p_subn->p_qos_policy); while (!cl_is_qlist_empty(&p_subn->prefix_routes_list)) { cl_list_item_t *item = cl_qlist_remove_head(&p_subn->prefix_routes_list); free(item); } } /********************************************************************** **********************************************************************/ ib_api_status_t osm_subn_init(IN osm_subn_t * const p_subn, IN osm_opensm_t * const p_osm, IN const osm_subn_opt_t * const p_opt) { cl_status_t status; p_subn->p_osm = p_osm; status = cl_ptr_vector_init(&p_subn->port_lid_tbl, OSM_SUBNET_VECTOR_MIN_SIZE, OSM_SUBNET_VECTOR_GROW_SIZE); if (status != CL_SUCCESS) return (status); status = cl_ptr_vector_set_capacity(&p_subn->port_lid_tbl, OSM_SUBNET_VECTOR_CAPACITY); if (status != CL_SUCCESS) return (status); /* LID zero is not valid. NULL out this entry for the convenience of other code. */ cl_ptr_vector_set(&p_subn->port_lid_tbl, 0, NULL); p_subn->opt = *p_opt; p_subn->max_ucast_lid_ho = IB_LID_UCAST_END_HO; p_subn->max_mcast_lid_ho = IB_LID_MCAST_END_HO; p_subn->min_ca_mtu = IB_MAX_MTU; p_subn->min_ca_rate = IB_MAX_RATE; p_subn->ignore_existing_lfts = TRUE; /* we assume master by default - so we only need to set it true if STANDBY */ p_subn->coming_out_of_standby = FALSE; return (IB_SUCCESS); } /********************************************************************** **********************************************************************/ ib_api_status_t osm_get_gid_by_mad_addr(IN osm_log_t * p_log, IN const osm_subn_t * p_subn, IN const osm_mad_addr_t * p_mad_addr, OUT ib_gid_t * p_gid) { const cl_ptr_vector_t *p_tbl; const osm_port_t *p_port = NULL; if (p_gid == NULL) { OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 7505: " "Provided output GID is NULL\n"); return (IB_INVALID_PARAMETER); } /* Find the port gid of the request in the subnet */ p_tbl = &p_subn->port_lid_tbl; CL_ASSERT(cl_ptr_vector_get_size(p_tbl) < 0x10000); if ((uint16_t) cl_ptr_vector_get_size(p_tbl) > cl_ntoh16(p_mad_addr->dest_lid)) { p_port = cl_ptr_vector_get(p_tbl, cl_ntoh16(p_mad_addr->dest_lid)); if (p_port == NULL) { OSM_LOG(p_log, OSM_LOG_DEBUG, "Did not find any port with LID: %u\n", cl_ntoh16(p_mad_addr->dest_lid)); return (IB_INVALID_PARAMETER); } p_gid->unicast.interface_id = p_port->p_physp->port_guid; p_gid->unicast.prefix = p_subn->opt.subnet_prefix; } else { /* The dest_lid is not in the subnet table - this is an error */ OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 7501: " "LID is out of range: %u\n", cl_ntoh16(p_mad_addr->dest_lid)); return (IB_INVALID_PARAMETER); } return (IB_SUCCESS); } /********************************************************************** **********************************************************************/ osm_physp_t *osm_get_physp_by_mad_addr(IN osm_log_t * p_log, IN const osm_subn_t * p_subn, IN osm_mad_addr_t * p_mad_addr) { const cl_ptr_vector_t *p_port_lid_tbl; osm_port_t *p_port = NULL; osm_physp_t *p_physp = NULL; /* Find the port gid of the request in the subnet */ p_port_lid_tbl = &p_subn->port_lid_tbl; CL_ASSERT(cl_ptr_vector_get_size(p_port_lid_tbl) < 0x10000); if ((uint16_t) cl_ptr_vector_get_size(p_port_lid_tbl) > cl_ntoh16(p_mad_addr->dest_lid)) { p_port = cl_ptr_vector_get(p_port_lid_tbl, cl_ntoh16(p_mad_addr->dest_lid)); if (p_port == NULL) { /* The port is not in the port_lid table - this is an error */ OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 7502: " "Cannot locate port object by lid: %u\n", cl_ntoh16(p_mad_addr->dest_lid)); goto Exit; } p_physp = p_port->p_physp; } else { /* The dest_lid is not in the subnet table - this is an error */ OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 7503: " "Lid is out of range: %u\n", cl_ntoh16(p_mad_addr->dest_lid)); } Exit: return p_physp; } /********************************************************************** **********************************************************************/ osm_port_t *osm_get_port_by_mad_addr(IN osm_log_t * p_log, IN const osm_subn_t * p_subn, IN osm_mad_addr_t * p_mad_addr) { const cl_ptr_vector_t *p_port_lid_tbl; osm_port_t *p_port = NULL; /* Find the port gid of the request in the subnet */ p_port_lid_tbl = &p_subn->port_lid_tbl; CL_ASSERT(cl_ptr_vector_get_size(p_port_lid_tbl) < 0x10000); if ((uint16_t) cl_ptr_vector_get_size(p_port_lid_tbl) > cl_ntoh16(p_mad_addr->dest_lid)) { p_port = cl_ptr_vector_get(p_port_lid_tbl, cl_ntoh16(p_mad_addr->dest_lid)); } else { /* The dest_lid is not in the subnet table - this is an error */ OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 7504: " "Lid is out of range: %u\n", cl_ntoh16(p_mad_addr->dest_lid)); } return p_port; } /********************************************************************** **********************************************************************/ osm_switch_t *osm_get_switch_by_guid(IN const osm_subn_t * p_subn, IN uint64_t guid) { osm_switch_t *p_switch; p_switch = (osm_switch_t *) cl_qmap_get(&(p_subn->sw_guid_tbl), guid); if (p_switch == (osm_switch_t *) cl_qmap_end(&(p_subn->sw_guid_tbl))) p_switch = NULL; return p_switch; } /********************************************************************** **********************************************************************/ osm_node_t *osm_get_node_by_guid(IN osm_subn_t const *p_subn, IN uint64_t guid) { osm_node_t *p_node; p_node = (osm_node_t *) cl_qmap_get(&(p_subn->node_guid_tbl), guid); if (p_node == (osm_node_t *) cl_qmap_end(&(p_subn->node_guid_tbl))) p_node = NULL; return p_node; } /********************************************************************** **********************************************************************/ osm_port_t *osm_get_port_by_guid(IN osm_subn_t const *p_subn, IN ib_net64_t guid) { osm_port_t *p_port; p_port = (osm_port_t *) cl_qmap_get(&(p_subn->port_guid_tbl), guid); if (p_port == (osm_port_t *) cl_qmap_end(&(p_subn->port_guid_tbl))) p_port = NULL; return p_port; } /********************************************************************** **********************************************************************/ static void subn_set_default_qos_options(IN osm_qos_options_t * opt) { opt->max_vls = OSM_DEFAULT_QOS_MAX_VLS; opt->high_limit = OSM_DEFAULT_QOS_HIGH_LIMIT; opt->vlarb_high = OSM_DEFAULT_QOS_VLARB_HIGH; opt->vlarb_low = OSM_DEFAULT_QOS_VLARB_LOW; opt->sl2vl = OSM_DEFAULT_QOS_SL2VL; } static void subn_init_qos_options(IN osm_qos_options_t * opt) { opt->max_vls = 0; opt->high_limit = -1; opt->vlarb_high = NULL; opt->vlarb_low = NULL; opt->sl2vl = NULL; } /********************************************************************** **********************************************************************/ void osm_subn_set_default_opt(IN osm_subn_opt_t * const p_opt) { memset(p_opt, 0, sizeof(osm_subn_opt_t)); p_opt->guid = 0; p_opt->m_key = OSM_DEFAULT_M_KEY; p_opt->sm_key = OSM_DEFAULT_SM_KEY; p_opt->sa_key = OSM_DEFAULT_SA_KEY; p_opt->subnet_prefix = IB_DEFAULT_SUBNET_PREFIX; p_opt->m_key_lease_period = 0; p_opt->sweep_interval = OSM_DEFAULT_SWEEP_INTERVAL_SECS; p_opt->max_wire_smps = OSM_DEFAULT_SMP_MAX_ON_WIRE; p_opt->console = OSM_DEFAULT_CONSOLE; p_opt->console_port = OSM_DEFAULT_CONSOLE_PORT; p_opt->transaction_timeout = OSM_DEFAULT_TRANS_TIMEOUT_MILLISEC; /* by default we will consider waiting for 50x transaction timeout normal */ p_opt->max_msg_fifo_timeout = 50 * OSM_DEFAULT_TRANS_TIMEOUT_MILLISEC; p_opt->sm_priority = OSM_DEFAULT_SM_PRIORITY; p_opt->lmc = OSM_DEFAULT_LMC; p_opt->lmc_esp0 = FALSE; p_opt->max_op_vls = OSM_DEFAULT_MAX_OP_VLS; p_opt->force_link_speed = 15; p_opt->reassign_lids = FALSE; p_opt->ignore_other_sm = FALSE; p_opt->single_thread = FALSE; p_opt->disable_multicast = FALSE; p_opt->force_log_flush = FALSE; p_opt->subnet_timeout = OSM_DEFAULT_SUBNET_TIMEOUT; p_opt->packet_life_time = OSM_DEFAULT_SWITCH_PACKET_LIFE; p_opt->vl_stall_count = OSM_DEFAULT_VL_STALL_COUNT; p_opt->leaf_vl_stall_count = OSM_DEFAULT_LEAF_VL_STALL_COUNT; p_opt->head_of_queue_lifetime = OSM_DEFAULT_HEAD_OF_QUEUE_LIFE; p_opt->leaf_head_of_queue_lifetime = OSM_DEFAULT_LEAF_HEAD_OF_QUEUE_LIFE; p_opt->local_phy_errors_threshold = OSM_DEFAULT_ERROR_THRESHOLD; p_opt->overrun_errors_threshold = OSM_DEFAULT_ERROR_THRESHOLD; p_opt->sminfo_polling_timeout = OSM_SM_DEFAULT_POLLING_TIMEOUT_MILLISECS; p_opt->polling_retry_number = OSM_SM_DEFAULT_POLLING_RETRY_NUMBER; p_opt->force_heavy_sweep = FALSE; p_opt->log_flags = OSM_LOG_DEFAULT_LEVEL; p_opt->honor_guid2lid_file = FALSE; p_opt->daemon = FALSE; p_opt->sm_inactive = FALSE; p_opt->babbling_port_policy = FALSE; #ifdef ENABLE_OSM_PERF_MGR p_opt->perfmgr = FALSE; p_opt->perfmgr_redir = TRUE; p_opt->perfmgr_sweep_time_s = OSM_PERFMGR_DEFAULT_SWEEP_TIME_S; p_opt->perfmgr_max_outstanding_queries = OSM_PERFMGR_DEFAULT_MAX_OUTSTANDING_QUERIES; p_opt->event_db_dump_file = NULL; /* use default */ #endif /* ENABLE_OSM_PERF_MGR */ p_opt->event_plugin_name = NULL; p_opt->node_name_map_name = NULL; p_opt->dump_files_dir = getenv("OSM_TMP_DIR"); if (!p_opt->dump_files_dir || !(*p_opt->dump_files_dir)) p_opt->dump_files_dir = OSM_DEFAULT_TMP_DIR; p_opt->log_file = OSM_DEFAULT_LOG_FILE; p_opt->log_max_size = 0; p_opt->partition_config_file = OSM_DEFAULT_PARTITION_CONFIG_FILE; p_opt->no_partition_enforcement = FALSE; p_opt->qos = FALSE; p_opt->qos_policy_file = OSM_DEFAULT_QOS_POLICY_FILE; p_opt->accum_log_file = TRUE; p_opt->port_prof_ignore_file = NULL; p_opt->port_profile_switch_nodes = FALSE; p_opt->sweep_on_trap = TRUE; p_opt->use_ucast_cache = FALSE; p_opt->routing_engine_names = NULL; p_opt->connect_roots = FALSE; p_opt->lid_matrix_dump_file = NULL; p_opt->lfts_file = NULL; p_opt->root_guid_file = NULL; p_opt->cn_guid_file = NULL; p_opt->ids_guid_file = NULL; p_opt->guid_routing_order_file = NULL; p_opt->sa_db_file = NULL; p_opt->exit_on_fatal = TRUE; p_opt->enable_quirks = FALSE; p_opt->no_clients_rereg = FALSE; p_opt->prefix_routes_file = OSM_DEFAULT_PREFIX_ROUTES_FILE; p_opt->consolidate_ipv6_snm_req = FALSE; subn_init_qos_options(&p_opt->qos_options); subn_init_qos_options(&p_opt->qos_ca_options); subn_init_qos_options(&p_opt->qos_sw0_options); subn_init_qos_options(&p_opt->qos_swe_options); subn_init_qos_options(&p_opt->qos_rtr_options); } /********************************************************************** **********************************************************************/ static void log_report(const char *fmt, ...) { char buf[128]; va_list args; va_start(args, fmt); vsnprintf(buf, sizeof(buf), fmt, args); va_end(args); - printf(buf); + printf("%s", buf); cl_log_event("OpenSM", CL_LOG_INFO, buf, NULL, 0); } static void log_config_value(char *name, const char *fmt, ...) { char buf[128]; va_list args; unsigned n; va_start(args, fmt); n = snprintf(buf, sizeof(buf), " Loading Cached Option:%s = ", name); if (n > sizeof(buf)) n = sizeof(buf); n += vsnprintf(buf + n, sizeof(buf) - n, fmt, args); if (n > sizeof(buf)) n = sizeof(buf); snprintf(buf + n, sizeof(buf) - n, "\n"); va_end(args); - printf(buf); + printf("%s", buf); cl_log_event("OpenSM", CL_LOG_INFO, buf, NULL, 0); } static void opts_unpack_net64(IN char *p_req_key, IN char *p_key, IN char *p_val_str, IN uint64_t * p_val) { if (!strcmp(p_req_key, p_key)) { uint64_t val = strtoull(p_val_str, NULL, 0); if (cl_hton64(val) != *p_val) { log_config_value(p_key, "0x%016" PRIx64, val); *p_val = cl_ntoh64(val); } } } /********************************************************************** **********************************************************************/ static void opts_unpack_uint32(IN char *p_req_key, IN char *p_key, IN char *p_val_str, IN uint32_t * p_val) { if (!strcmp(p_req_key, p_key)) { uint32_t val = strtoul(p_val_str, NULL, 0); if (val != *p_val) { log_config_value(p_key, "%u", val); *p_val = val; } } } /********************************************************************** **********************************************************************/ static void opts_unpack_int32(IN char *p_req_key, IN char *p_key, IN char *p_val_str, IN int32_t * p_val) { if (!strcmp(p_req_key, p_key)) { int32_t val = strtol(p_val_str, NULL, 0); if (val != *p_val) { log_config_value(p_key, "%d", val); *p_val = val; } } } /********************************************************************** **********************************************************************/ static void opts_unpack_uint16(IN char *p_req_key, IN char *p_key, IN char *p_val_str, IN uint16_t * p_val) { if (!strcmp(p_req_key, p_key)) { uint16_t val = (uint16_t) strtoul(p_val_str, NULL, 0); if (val != *p_val) { log_config_value(p_key, "%u", val); *p_val = val; } } } /********************************************************************** **********************************************************************/ static void opts_unpack_net16(IN char *p_req_key, IN char *p_key, IN char *p_val_str, IN uint16_t * p_val) { if (!strcmp(p_req_key, p_key)) { uint32_t val; val = strtoul(p_val_str, NULL, 0); CL_ASSERT(val < 0x10000); if (cl_hton32(val) != *p_val) { log_config_value(p_key, "0x%04x", val); *p_val = cl_hton16((uint16_t) val); } } } /********************************************************************** **********************************************************************/ static void opts_unpack_uint8(IN char *p_req_key, IN char *p_key, IN char *p_val_str, IN uint8_t * p_val) { if (!strcmp(p_req_key, p_key)) { uint32_t val; val = strtoul(p_val_str, NULL, 0); CL_ASSERT(val < 0x100); if (val != *p_val) { log_config_value(p_key, "%u", val); *p_val = (uint8_t) val; } } } /********************************************************************** **********************************************************************/ static void opts_unpack_boolean(IN char *p_req_key, IN char *p_key, IN char *p_val_str, IN boolean_t * p_val) { if (!strcmp(p_req_key, p_key) && p_val_str) { boolean_t val; if (strcmp("TRUE", p_val_str)) val = FALSE; else val = TRUE; if (val != *p_val) { log_config_value(p_key, "%s", p_val_str); *p_val = val; } } } /********************************************************************** **********************************************************************/ static void opts_unpack_charp(IN char *p_req_key, IN char *p_key, IN char *p_val_str, IN char **p_val) { if (!strcmp(p_req_key, p_key) && p_val_str) { const char *current_str = *p_val ? *p_val : null_str ; if (strcmp(p_val_str, current_str)) { log_config_value(p_key, "%s", p_val_str); /* special case the "(null)" string */ if (strcmp(null_str, p_val_str) == 0) { *p_val = NULL; } else { /* Ignore the possible memory leak here; the pointer may be to a static default. */ *p_val = strdup(p_val_str); } } } } /********************************************************************** **********************************************************************/ static char *clean_val(char *val) { char *p = val; /* clean leading spaces */ while (isspace(*p)) p++; val = p; if (!*val) return val; /* clean trailing spaces */ p = val + strlen(val) - 1; while (p > val && isspace(*p)) p--; p[1] = '\0'; /* clean quotas */ if ((*val == '\"' && *p == '\"') || (*val == '\'' && *p == '\'')) { val++; p--; } return val; } /********************************************************************** **********************************************************************/ static void subn_parse_qos_options(IN const char *prefix, IN char *p_key, IN char *p_val_str, IN osm_qos_options_t * opt) { char name[256]; snprintf(name, sizeof(name), "%s_max_vls", prefix); opts_unpack_uint32(name, p_key, p_val_str, &opt->max_vls); snprintf(name, sizeof(name), "%s_high_limit", prefix); opts_unpack_int32(name, p_key, p_val_str, &opt->high_limit); snprintf(name, sizeof(name), "%s_vlarb_high", prefix); opts_unpack_charp(name, p_key, p_val_str, &opt->vlarb_high); snprintf(name, sizeof(name), "%s_vlarb_low", prefix); opts_unpack_charp(name, p_key, p_val_str, &opt->vlarb_low); snprintf(name, sizeof(name), "%s_sl2vl", prefix); opts_unpack_charp(name, p_key, p_val_str, &opt->sl2vl); } static int subn_dump_qos_options(FILE * file, const char *set_name, const char *prefix, osm_qos_options_t * opt) { return fprintf(file, "# %s\n" "%s_max_vls %u\n" "%s_high_limit %d\n" "%s_vlarb_high %s\n" "%s_vlarb_low %s\n" "%s_sl2vl %s\n", set_name, prefix, opt->max_vls, prefix, opt->high_limit, prefix, opt->vlarb_high, prefix, opt->vlarb_low, prefix, opt->sl2vl); } /********************************************************************** **********************************************************************/ static ib_api_status_t append_prefix_route(IN osm_subn_t * const p_subn, uint64_t prefix, uint64_t guid) { osm_prefix_route_t *route; route = malloc(sizeof *route); if (! route) { OSM_LOG(&p_subn->p_osm->log, OSM_LOG_ERROR, "out of memory"); return IB_ERROR; } route->prefix = cl_hton64(prefix); route->guid = cl_hton64(guid); cl_qlist_insert_tail(&p_subn->prefix_routes_list, &route->list_item); return IB_SUCCESS; } static ib_api_status_t osm_parse_prefix_routes_file(IN osm_subn_t * const p_subn) { osm_log_t *log = &p_subn->p_osm->log; FILE *fp; char buf[1024]; int line = 0; int errors = 0; while (!cl_is_qlist_empty(&p_subn->prefix_routes_list)) { cl_list_item_t *item = cl_qlist_remove_head(&p_subn->prefix_routes_list); free(item); } fp = fopen(p_subn->opt.prefix_routes_file, "r"); if (! fp) { if (errno == ENOENT) return IB_SUCCESS; OSM_LOG(log, OSM_LOG_ERROR, "fopen(%s) failed: %s", p_subn->opt.prefix_routes_file, strerror(errno)); return IB_ERROR; } while (fgets(buf, sizeof buf, fp) != NULL) { char *p_prefix, *p_guid, *p_extra, *p_last, *p_end; uint64_t prefix, guid; line++; if (errors > 10) break; p_prefix = strtok_r(buf, " \t\n", &p_last); if (! p_prefix) continue; /* ignore blank lines */ if (*p_prefix == '#') continue; /* ignore comment lines */ p_guid = strtok_r(NULL, " \t\n", &p_last); if (! p_guid) { OSM_LOG(log, OSM_LOG_ERROR, "%s:%d: missing GUID\n", p_subn->opt.prefix_routes_file, line); errors++; continue; } p_extra = strtok_r(NULL, " \t\n", &p_last); if (p_extra && *p_extra != '#') { OSM_LOG(log, OSM_LOG_INFO, "%s:%d: extra tokens ignored\n", p_subn->opt.prefix_routes_file, line); } if (strcmp(p_prefix, "*") == 0) prefix = 0; else { prefix = strtoull(p_prefix, &p_end, 16); if (*p_end != '\0') { OSM_LOG(log, OSM_LOG_ERROR, "%s:%d: illegal prefix: %s\n", p_subn->opt.prefix_routes_file, line, p_prefix); errors++; continue; } } if (strcmp(p_guid, "*") == 0) guid = 0; else { guid = strtoull(p_guid, &p_end, 16); if (*p_end != '\0' && *p_end != '#') { OSM_LOG(log, OSM_LOG_ERROR, "%s:%d: illegal GUID: %s\n", p_subn->opt.prefix_routes_file, line, p_guid); errors++; continue; } } if (append_prefix_route(p_subn, prefix, guid) != IB_SUCCESS) { errors++; break; } } fclose(fp); return (errors == 0) ? IB_SUCCESS : IB_ERROR; } /********************************************************************** **********************************************************************/ static void subn_verify_max_vls(unsigned *max_vls, const char *prefix, unsigned dflt) { if (!(*max_vls) || *max_vls > 15) { log_report(" Invalid Cached Option: %s_max_vls=%u: " "Using Default = %u\n", prefix, *max_vls, dflt); *max_vls = dflt; } } static void subn_verify_high_limit(int *high_limit, const char *prefix, int dflt) { if (*high_limit < 0 || *high_limit > 255) { log_report(" Invalid Cached Option: %s_high_limit=%d: " "Using Default: %d\n", prefix, *high_limit, dflt); *high_limit = dflt; } } static void subn_verify_vlarb(char **vlarb, const char *prefix, const char *suffix, char *dflt) { char *str, *tok, *end, *ptr; int count = 0; if (*vlarb == NULL) { log_report(" Invalid Cached Option: %s_vlarb_%s: " "Using Default\n", prefix, suffix); *vlarb = dflt; return; } str = strdup(*vlarb); tok = strtok_r(str, ",\n", &ptr); while (tok) { char *vl_str, *weight_str; vl_str = tok; weight_str = strchr(tok, ':'); if (weight_str) { long vl, weight; *weight_str = '\0'; weight_str++; vl = strtol(vl_str, &end, 0); if (*end) log_report(" Warning: Cached Option " "%s_vlarb_%s:vl=%s" " improperly formatted\n", prefix, suffix, vl_str); else if (vl < 0 || vl > 14) log_report(" Warning: Cached Option " "%s_vlarb_%s:vl=%ld out of range\n", prefix, suffix, vl); weight = strtol(weight_str, &end, 0); if (*end) log_report(" Warning: Cached Option " "%s_vlarb_%s:weight=%s " "improperly formatted\n", prefix, suffix, weight_str); else if (weight < 0 || weight > 255) log_report(" Warning: Cached Option " "%s_vlarb_%s:weight=%ld " "out of range\n", prefix, suffix, weight); } else log_report(" Warning: Cached Option " "%s_vlarb_%s:vl:weight=%s " "improperly formatted\n", prefix, suffix, tok); count++; tok = strtok_r(NULL, ",\n", &ptr); } if (count > 64) log_report(" Warning: Cached Option %s_vlarb_%s: > 64 listed:" " excess vl:weight pairs will be dropped\n", prefix, suffix); free(str); } static void subn_verify_sl2vl(char **sl2vl, const char *prefix, char *dflt) { char *str, *tok, *end, *ptr; int count = 0; if (*sl2vl == NULL) { log_report(" Invalid Cached Option: %s_sl2vl: Using Default\n", prefix); *sl2vl = dflt; return; } str = strdup(*sl2vl); tok = strtok_r(str, ",\n", &ptr); while (tok) { long vl = strtol(tok, &end, 0); if (*end) log_report(" Warning: Cached Option %s_sl2vl:vl=%s " "improperly formatted\n", prefix, tok); else if (vl < 0 || vl > 15) log_report(" Warning: Cached Option %s_sl2vl:vl=%ld " "out of range\n", prefix, vl); count++; tok = strtok_r(NULL, ",\n", &ptr); } if (count < 16) log_report(" Warning: Cached Option %s_sl2vl: < 16 VLs " "listed\n", prefix); if (count > 16) log_report(" Warning: Cached Option %s_sl2vl: > 16 listed: " "excess VLs will be dropped\n", prefix); free(str); } static void subn_verify_qos_set(osm_qos_options_t *set, const char *prefix, osm_qos_options_t *dflt) { subn_verify_max_vls(&set->max_vls, prefix, dflt->max_vls); subn_verify_high_limit(&set->high_limit, prefix, dflt->high_limit); subn_verify_vlarb(&set->vlarb_low, prefix, "low", dflt->vlarb_low); subn_verify_vlarb(&set->vlarb_high, prefix, "high", dflt->vlarb_high); subn_verify_sl2vl(&set->sl2vl, prefix, dflt->sl2vl); } int osm_subn_verify_config(IN osm_subn_opt_t * const p_opts) { if (p_opts->lmc > 7) { log_report(" Invalid Cached Option Value:lmc = %u:" "Using Default:%u\n", p_opts->lmc, OSM_DEFAULT_LMC); p_opts->lmc = OSM_DEFAULT_LMC; } if (15 < p_opts->sm_priority) { log_report(" Invalid Cached Option Value:sm_priority = %u:" "Using Default:%u\n", p_opts->sm_priority, OSM_DEFAULT_SM_PRIORITY); p_opts->sm_priority = OSM_DEFAULT_SM_PRIORITY; } if ((15 < p_opts->force_link_speed) || (p_opts->force_link_speed > 7 && p_opts->force_link_speed < 15)) { log_report(" Invalid Cached Option Value:force_link_speed = %u:" "Using Default:%u\n", p_opts->force_link_speed, IB_PORT_LINK_SPEED_ENABLED_MASK); p_opts->force_link_speed = IB_PORT_LINK_SPEED_ENABLED_MASK; } if (strcmp(p_opts->console, OSM_DISABLE_CONSOLE) && strcmp(p_opts->console, OSM_LOCAL_CONSOLE) #ifdef ENABLE_OSM_CONSOLE_SOCKET && strcmp(p_opts->console, OSM_LOOPBACK_CONSOLE) && strcmp(p_opts->console, OSM_REMOTE_CONSOLE) #endif ) { log_report(" Invalid Cached Option Value:console = %s" ", Using Default:%s\n", p_opts->console, OSM_DEFAULT_CONSOLE); p_opts->console = OSM_DEFAULT_CONSOLE; } if (p_opts->qos) { osm_qos_options_t dflt; /* the default options in qos_options must be correct. * every other one need not be, b/c those will default * back to whatever is in qos_options. */ subn_set_default_qos_options(&dflt); subn_verify_qos_set(&p_opts->qos_options, "qos", &dflt); subn_verify_qos_set(&p_opts->qos_ca_options, "qos_ca", &p_opts->qos_options); subn_verify_qos_set(&p_opts->qos_sw0_options, "qos_sw0", &p_opts->qos_options); subn_verify_qos_set(&p_opts->qos_swe_options, "qos_swe", &p_opts->qos_options); subn_verify_qos_set(&p_opts->qos_rtr_options, "qos_rtr", &p_opts->qos_options); } #ifdef ENABLE_OSM_PERF_MGR if (p_opts->perfmgr_sweep_time_s < 1) { log_report(" Invalid Cached Option Value:perfmgr_sweep_time_s " "= %u Using Default:%u\n", p_opts->perfmgr_sweep_time_s, OSM_PERFMGR_DEFAULT_SWEEP_TIME_S); p_opts->perfmgr_sweep_time_s = OSM_PERFMGR_DEFAULT_SWEEP_TIME_S; } if (p_opts->perfmgr_max_outstanding_queries < 1) { log_report(" Invalid Cached Option Value:" "perfmgr_max_outstanding_queries = %u" " Using Default:%u\n", p_opts->perfmgr_max_outstanding_queries, OSM_PERFMGR_DEFAULT_MAX_OUTSTANDING_QUERIES); p_opts->perfmgr_max_outstanding_queries = OSM_PERFMGR_DEFAULT_MAX_OUTSTANDING_QUERIES; } #endif return 0; } /********************************************************************** **********************************************************************/ int osm_subn_parse_conf_file(char *file_name, osm_subn_opt_t * const p_opts) { char line[1024]; FILE *opts_file; char *p_key, *p_val; opts_file = fopen(file_name, "r"); if (!opts_file) { if (errno == ENOENT) return 1; printf("cannot open file \'%s\': %s\n", file_name, strerror(errno)); return -1; } printf(" Reading Cached Option File: %s\n", file_name); cl_log_event("OpenSM", CL_LOG_INFO, line, NULL, 0); p_opts->config_file = file_name; while (fgets(line, 1023, opts_file) != NULL) { /* get the first token */ p_key = strtok_r(line, " \t\n", &p_val); if (!p_key) continue; p_val = clean_val(p_val); opts_unpack_net64("guid", p_key, p_val, &p_opts->guid); opts_unpack_net64("m_key", p_key, p_val, &p_opts->m_key); opts_unpack_net64("sm_key", p_key, p_val, &p_opts->sm_key); opts_unpack_net64("sa_key", p_key, p_val, &p_opts->sa_key); opts_unpack_net64("subnet_prefix", p_key, p_val, &p_opts->subnet_prefix); opts_unpack_net16("m_key_lease_period", p_key, p_val, &p_opts->m_key_lease_period); opts_unpack_uint32("sweep_interval", p_key, p_val, &p_opts->sweep_interval); opts_unpack_uint32("max_wire_smps", p_key, p_val, &p_opts->max_wire_smps); opts_unpack_charp("console", p_key, p_val, &p_opts->console); opts_unpack_uint16("console_port", p_key, p_val, &p_opts->console_port); opts_unpack_uint32("transaction_timeout", p_key, p_val, &p_opts->transaction_timeout); opts_unpack_uint32("max_msg_fifo_timeout", p_key, p_val, &p_opts->max_msg_fifo_timeout); opts_unpack_uint8("sm_priority", p_key, p_val, &p_opts->sm_priority); opts_unpack_uint8("lmc", p_key, p_val, &p_opts->lmc); opts_unpack_boolean("lmc_esp0", p_key, p_val, &p_opts->lmc_esp0); opts_unpack_uint8("max_op_vls", p_key, p_val, &p_opts->max_op_vls); opts_unpack_uint8("force_link_speed", p_key, p_val, &p_opts->force_link_speed); opts_unpack_boolean("reassign_lids", p_key, p_val, &p_opts->reassign_lids); opts_unpack_boolean("ignore_other_sm", p_key, p_val, &p_opts->ignore_other_sm); opts_unpack_boolean("single_thread", p_key, p_val, &p_opts->single_thread); opts_unpack_boolean("disable_multicast", p_key, p_val, &p_opts->disable_multicast); opts_unpack_boolean("force_log_flush", p_key, p_val, &p_opts->force_log_flush); opts_unpack_uint8("subnet_timeout", p_key, p_val, &p_opts->subnet_timeout); opts_unpack_uint8("packet_life_time", p_key, p_val, &p_opts->packet_life_time); opts_unpack_uint8("vl_stall_count", p_key, p_val, &p_opts->vl_stall_count); opts_unpack_uint8("leaf_vl_stall_count", p_key, p_val, &p_opts->leaf_vl_stall_count); opts_unpack_uint8("head_of_queue_lifetime", p_key, p_val, &p_opts->head_of_queue_lifetime); opts_unpack_uint8("leaf_head_of_queue_lifetime", p_key, p_val, &p_opts->leaf_head_of_queue_lifetime); opts_unpack_uint8("local_phy_errors_threshold", p_key, p_val, &p_opts->local_phy_errors_threshold); opts_unpack_uint8("overrun_errors_threshold", p_key, p_val, &p_opts->overrun_errors_threshold); opts_unpack_uint32("sminfo_polling_timeout", p_key, p_val, &p_opts->sminfo_polling_timeout); opts_unpack_uint32("polling_retry_number", p_key, p_val, &p_opts->polling_retry_number); opts_unpack_boolean("force_heavy_sweep", p_key, p_val, &p_opts->force_heavy_sweep); opts_unpack_uint8("log_flags", p_key, p_val, &p_opts->log_flags); opts_unpack_charp("port_prof_ignore_file", p_key, p_val, &p_opts->port_prof_ignore_file); opts_unpack_boolean("port_profile_switch_nodes", p_key, p_val, &p_opts->port_profile_switch_nodes); opts_unpack_boolean("sweep_on_trap", p_key, p_val, &p_opts->sweep_on_trap); opts_unpack_charp("routing_engine", p_key, p_val, &p_opts->routing_engine_names); opts_unpack_boolean("connect_roots", p_key, p_val, &p_opts->connect_roots); opts_unpack_boolean("use_ucast_cache", p_key, p_val, &p_opts->use_ucast_cache); opts_unpack_charp("log_file", p_key, p_val, &p_opts->log_file); opts_unpack_uint32("log_max_size", p_key, p_val, (void *) & p_opts->log_max_size); p_opts->log_max_size *= 1024 * 1024; /* convert to MB */ opts_unpack_charp("partition_config_file", p_key, p_val, &p_opts->partition_config_file); opts_unpack_boolean("no_partition_enforcement", p_key, p_val, &p_opts->no_partition_enforcement); opts_unpack_boolean("qos", p_key, p_val, &p_opts->qos); opts_unpack_charp("qos_policy_file", p_key, p_val, &p_opts->qos_policy_file); opts_unpack_boolean("accum_log_file", p_key, p_val, &p_opts->accum_log_file); opts_unpack_charp("dump_files_dir", p_key, p_val, &p_opts->dump_files_dir); opts_unpack_charp("lid_matrix_dump_file", p_key, p_val, &p_opts->lid_matrix_dump_file); opts_unpack_charp("lfts_file", p_key, p_val, &p_opts->lfts_file); opts_unpack_charp("root_guid_file", p_key, p_val, &p_opts->root_guid_file); opts_unpack_charp("cn_guid_file", p_key, p_val, &p_opts->cn_guid_file); opts_unpack_charp("ids_guid_file", p_key, p_val, &p_opts->ids_guid_file); opts_unpack_charp("guid_routing_order_file", p_key, p_val, &p_opts->guid_routing_order_file); opts_unpack_charp("sa_db_file", p_key, p_val, &p_opts->sa_db_file); opts_unpack_boolean("exit_on_fatal", p_key, p_val, &p_opts->exit_on_fatal); opts_unpack_boolean("honor_guid2lid_file", p_key, p_val, &p_opts->honor_guid2lid_file); opts_unpack_boolean("daemon", p_key, p_val, &p_opts->daemon); opts_unpack_boolean("sm_inactive", p_key, p_val, &p_opts->sm_inactive); opts_unpack_boolean("babbling_port_policy", p_key, p_val, &p_opts->babbling_port_policy); #ifdef ENABLE_OSM_PERF_MGR opts_unpack_boolean("perfmgr", p_key, p_val, &p_opts->perfmgr); opts_unpack_boolean("perfmgr_redir", p_key, p_val, &p_opts->perfmgr_redir); opts_unpack_uint16("perfmgr_sweep_time_s", p_key, p_val, &p_opts->perfmgr_sweep_time_s); opts_unpack_uint32("perfmgr_max_outstanding_queries", p_key, p_val, &p_opts->perfmgr_max_outstanding_queries); opts_unpack_charp("event_db_dump_file", p_key, p_val, &p_opts->event_db_dump_file); #endif /* ENABLE_OSM_PERF_MGR */ opts_unpack_charp("event_plugin_name", p_key, p_val, &p_opts->event_plugin_name); opts_unpack_charp("node_name_map_name", p_key, p_val, &p_opts->node_name_map_name); subn_parse_qos_options("qos", p_key, p_val, &p_opts->qos_options); subn_parse_qos_options("qos_ca", p_key, p_val, &p_opts->qos_ca_options); subn_parse_qos_options("qos_sw0", p_key, p_val, &p_opts->qos_sw0_options); subn_parse_qos_options("qos_swe", p_key, p_val, &p_opts->qos_swe_options); subn_parse_qos_options("qos_rtr", p_key, p_val, &p_opts->qos_rtr_options); opts_unpack_boolean("enable_quirks", p_key, p_val, &p_opts->enable_quirks); opts_unpack_boolean("no_clients_rereg", p_key, p_val, &p_opts->no_clients_rereg); opts_unpack_charp("prefix_routes_file", p_key, p_val, &p_opts->prefix_routes_file); opts_unpack_boolean("consolidate_ipv6_snm_req", p_key, p_val, &p_opts->consolidate_ipv6_snm_req); } fclose(opts_file); osm_subn_verify_config(p_opts); return 0; } int osm_subn_rescan_conf_files(IN osm_subn_t * const p_subn) { FILE *opts_file; char line[1024]; char *p_key, *p_val, *p_last; if (!p_subn->opt.config_file) return 0; opts_file = fopen(p_subn->opt.config_file, "r"); if (!opts_file) { if (errno == ENOENT) return 1; OSM_LOG(&p_subn->p_osm->log, OSM_LOG_ERROR, "cannot open file \'%s\': %s\n", p_subn->opt.config_file, strerror(errno)); return -1; } subn_init_qos_options(&p_subn->opt.qos_options); subn_init_qos_options(&p_subn->opt.qos_ca_options); subn_init_qos_options(&p_subn->opt.qos_sw0_options); subn_init_qos_options(&p_subn->opt.qos_swe_options); subn_init_qos_options(&p_subn->opt.qos_rtr_options); while (fgets(line, 1023, opts_file) != NULL) { /* get the first token */ p_key = strtok_r(line, " \t\n", &p_last); if (p_key) { p_val = strtok_r(NULL, " \t\n", &p_last); subn_parse_qos_options("qos", p_key, p_val, &p_subn->opt.qos_options); subn_parse_qos_options("qos_ca", p_key, p_val, &p_subn->opt.qos_ca_options); subn_parse_qos_options("qos_sw0", p_key, p_val, &p_subn->opt.qos_sw0_options); subn_parse_qos_options("qos_swe", p_key, p_val, &p_subn->opt.qos_swe_options); subn_parse_qos_options("qos_rtr", p_key, p_val, &p_subn->opt.qos_rtr_options); } } fclose(opts_file); osm_subn_verify_config(&p_subn->opt); osm_parse_prefix_routes_file(p_subn); return 0; } /********************************************************************** **********************************************************************/ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts) { fprintf(out, "#\n# DEVICE ATTRIBUTES OPTIONS\n#\n" "# The port GUID on which the OpenSM is running\n" "guid 0x%016" PRIx64 "\n\n" "# M_Key value sent to all ports qualifying all Set(PortInfo)\n" "m_key 0x%016" PRIx64 "\n\n" "# The lease period used for the M_Key on this subnet in [sec]\n" "m_key_lease_period %u\n\n" "# SM_Key value of the SM used for SM authentication\n" "sm_key 0x%016" PRIx64 "\n\n" "# SM_Key value to qualify rcv SA queries as 'trusted'\n" "sa_key 0x%016" PRIx64 "\n\n" "# Note that for both values above (sm_key and sa_key)\n" "# OpenSM version 3.2.1 and below used the default value '1'\n" "# in a host byte order, it is fixed now but you may need to\n" "# change the values to interoperate with old OpenSM running\n" "# on a little endian machine.\n\n" "# Subnet prefix used on this subnet\n" "subnet_prefix 0x%016" PRIx64 "\n\n" "# The LMC value used on this subnet\n" "lmc %u\n\n" "# lmc_esp0 determines whether LMC value used on subnet is used for\n" "# enhanced switch port 0. If TRUE, LMC value for subnet is used for\n" "# ESP0. Otherwise, LMC value for ESP0s is 0.\n" "lmc_esp0 %s\n\n" "# The code of maximal time a packet can live in a switch\n" "# The actual time is 4.096usec * 2^\n" "# The value 0x14 disables this mechanism\n" "packet_life_time 0x%02x\n\n" "# The number of sequential packets dropped that cause the port\n" "# to enter the VLStalled state. The result of setting this value to\n" "# zero is undefined.\n" "vl_stall_count 0x%02x\n\n" "# The number of sequential packets dropped that cause the port\n" "# to enter the VLStalled state. This value is for switch ports\n" "# driving a CA or router port. The result of setting this value\n" "# to zero is undefined.\n" "leaf_vl_stall_count 0x%02x\n\n" "# The code of maximal time a packet can wait at the head of\n" "# transmission queue.\n" "# The actual time is 4.096usec * 2^\n" "# The value 0x14 disables this mechanism\n" "head_of_queue_lifetime 0x%02x\n\n" "# The maximal time a packet can wait at the head of queue on\n" "# switch port connected to a CA or router port\n" "leaf_head_of_queue_lifetime 0x%02x\n\n" "# Limit the maximal operational VLs\n" "max_op_vls %u\n\n" "# Force PortInfo:LinkSpeedEnabled on switch ports\n" "# If 0, don't modify PortInfo:LinkSpeedEnabled on switch port\n" "# Otherwise, use value for PortInfo:LinkSpeedEnabled on switch port\n" "# Values are (IB Spec 1.2.1, 14.2.5.6 Table 146 \"PortInfo\")\n" "# 1: 2.5 Gbps\n" "# 3: 2.5 or 5.0 Gbps\n" "# 5: 2.5 or 10.0 Gbps\n" "# 7: 2.5 or 5.0 or 10.0 Gbps\n" "# 2,4,6,8-14 Reserved\n" "# Default 15: set to PortInfo:LinkSpeedSupported\n" "force_link_speed %u\n\n" "# The subnet_timeout code that will be set for all the ports\n" "# The actual timeout is 4.096usec * 2^\n" "subnet_timeout %u\n\n" "# Threshold of local phy errors for sending Trap 129\n" "local_phy_errors_threshold 0x%02x\n\n" "# Threshold of credit overrun errors for sending Trap 130\n" "overrun_errors_threshold 0x%02x\n\n", cl_ntoh64(p_opts->guid), cl_ntoh64(p_opts->m_key), cl_ntoh16(p_opts->m_key_lease_period), cl_ntoh64(p_opts->sm_key), cl_ntoh64(p_opts->sa_key), cl_ntoh64(p_opts->subnet_prefix), p_opts->lmc, p_opts->lmc_esp0 ? "TRUE" : "FALSE", p_opts->packet_life_time, p_opts->vl_stall_count, p_opts->leaf_vl_stall_count, p_opts->head_of_queue_lifetime, p_opts->leaf_head_of_queue_lifetime, p_opts->max_op_vls, p_opts->force_link_speed, p_opts->subnet_timeout, p_opts->local_phy_errors_threshold, p_opts->overrun_errors_threshold); fprintf(out, "#\n# PARTITIONING OPTIONS\n#\n" "# Partition configuration file to be used\n" "partition_config_file %s\n\n" "# Disable partition enforcement by switches\n" "no_partition_enforcement %s\n\n", p_opts->partition_config_file, p_opts->no_partition_enforcement ? "TRUE" : "FALSE"); fprintf(out, "#\n# SWEEP OPTIONS\n#\n" "# The number of seconds between subnet sweeps (0 disables it)\n" "sweep_interval %u\n\n" "# If TRUE cause all lids to be reassigned\n" "reassign_lids %s\n\n" "# If TRUE forces every sweep to be a heavy sweep\n" "force_heavy_sweep %s\n\n" "# If TRUE every trap will cause a heavy sweep.\n" "# NOTE: successive identical traps (>10) are suppressed\n" "sweep_on_trap %s\n\n", p_opts->sweep_interval, p_opts->reassign_lids ? "TRUE" : "FALSE", p_opts->force_heavy_sweep ? "TRUE" : "FALSE", p_opts->sweep_on_trap ? "TRUE" : "FALSE"); fprintf(out, "#\n# ROUTING OPTIONS\n#\n" "# If TRUE count switches as link subscriptions\n" "port_profile_switch_nodes %s\n\n", p_opts->port_profile_switch_nodes ? "TRUE" : "FALSE"); fprintf(out, "# Name of file with port guids to be ignored by port profiling\n" "port_prof_ignore_file %s\n\n", p_opts->port_prof_ignore_file ? p_opts->port_prof_ignore_file : null_str); fprintf(out, "# Routing engine\n" "# Multiple routing engines can be specified separated by\n" "# commas so that specific ordering of routing algorithms will\n" "# be tried if earlier routing engines fail.\n" "# Supported engines: minhop, updn, file, ftree, lash, dor\n" "routing_engine %s\n\n", p_opts->routing_engine_names ? p_opts->routing_engine_names : null_str); fprintf(out, "# Connect roots (use FALSE if unsure)\n" "connect_roots %s\n\n", p_opts->connect_roots ? "TRUE" : "FALSE"); fprintf(out, "# Use unicast routing cache (use FALSE if unsure)\n" "use_ucast_cache %s\n\n", p_opts->use_ucast_cache ? "TRUE" : "FALSE"); fprintf(out, "# Lid matrix dump file name\n" "lid_matrix_dump_file %s\n\n", p_opts->lid_matrix_dump_file ? p_opts->lid_matrix_dump_file : null_str); fprintf(out, "# LFTs file name\nlfts_file %s\n\n", p_opts->lfts_file ? p_opts->lfts_file : null_str); fprintf(out, "# The file holding the root node guids (for fat-tree or Up/Down)\n" "# One guid in each line\nroot_guid_file %s\n\n", p_opts->root_guid_file ? p_opts->root_guid_file : null_str); fprintf(out, "# The file holding the fat-tree compute node guids\n" "# One guid in each line\ncn_guid_file %s\n\n", p_opts->cn_guid_file ? p_opts->cn_guid_file : null_str); fprintf(out, "# The file holding the node ids which will be used by" " Up/Down algorithm instead\n# of GUIDs (one guid and" " id in each line)\nids_guid_file %s\n\n", p_opts->ids_guid_file ? p_opts->ids_guid_file : null_str); fprintf(out, "# The file holding guid routing order guids (for MinHop and Up/Down)\n" "guid_routing_order_file %s\n\n", p_opts->guid_routing_order_file ? p_opts->guid_routing_order_file : null_str); fprintf(out, "# SA database file name\nsa_db_file %s\n\n", p_opts->sa_db_file ? p_opts->sa_db_file : null_str); fprintf(out, "#\n# HANDOVER - MULTIPLE SMs OPTIONS\n#\n" "# SM priority used for deciding who is the master\n" "# Range goes from 0 (lowest priority) to 15 (highest).\n" "sm_priority %u\n\n" "# If TRUE other SMs on the subnet should be ignored\n" "ignore_other_sm %s\n\n" "# Timeout in [msec] between two polls of active master SM\n" "sminfo_polling_timeout %u\n\n" "# Number of failing polls of remote SM that declares it dead\n" "polling_retry_number %u\n\n" "# If TRUE honor the guid2lid file when coming out of standby\n" "# state, if such file exists and is valid\n" "honor_guid2lid_file %s\n\n", p_opts->sm_priority, p_opts->ignore_other_sm ? "TRUE" : "FALSE", p_opts->sminfo_polling_timeout, p_opts->polling_retry_number, p_opts->honor_guid2lid_file ? "TRUE" : "FALSE"); fprintf(out, "#\n# TIMING AND THREADING OPTIONS\n#\n" "# Maximum number of SMPs sent in parallel\n" "max_wire_smps %u\n\n" "# The maximum time in [msec] allowed for a transaction to complete\n" "transaction_timeout %u\n\n" "# Maximal time in [msec] a message can stay in the incoming message queue.\n" "# If there is more than one message in the queue and the last message\n" "# stayed in the queue more than this value, any SA request will be\n" "# immediately returned with a BUSY status.\n" "max_msg_fifo_timeout %u\n\n" "# Use a single thread for handling SA queries\n" "single_thread %s\n\n", p_opts->max_wire_smps, p_opts->transaction_timeout, p_opts->max_msg_fifo_timeout, p_opts->single_thread ? "TRUE" : "FALSE"); fprintf(out, "#\n# MISC OPTIONS\n#\n" "# Daemon mode\n" "daemon %s\n\n" "# SM Inactive\n" "sm_inactive %s\n\n" "# Babbling Port Policy\n" "babbling_port_policy %s\n\n", p_opts->daemon ? "TRUE" : "FALSE", p_opts->sm_inactive ? "TRUE" : "FALSE", p_opts->babbling_port_policy ? "TRUE" : "FALSE"); #ifdef ENABLE_OSM_PERF_MGR fprintf(out, "#\n# Performance Manager Options\n#\n" "# perfmgr enable\n" "perfmgr %s\n\n" "# perfmgr redirection enable\n" "perfmgr_redir %s\n\n" "# sweep time in seconds\n" "perfmgr_sweep_time_s %u\n\n" "# Max outstanding queries\n" "perfmgr_max_outstanding_queries %u\n\n", p_opts->perfmgr ? "TRUE" : "FALSE", p_opts->perfmgr_redir ? "TRUE" : "FALSE", p_opts->perfmgr_sweep_time_s, p_opts->perfmgr_max_outstanding_queries); fprintf(out, "#\n# Event DB Options\n#\n" "# Dump file to dump the events to\n" "event_db_dump_file %s\n\n", p_opts->event_db_dump_file ? p_opts->event_db_dump_file : null_str); #endif /* ENABLE_OSM_PERF_MGR */ fprintf(out, "#\n# Event Plugin Options\n#\n" "event_plugin_name %s\n\n", p_opts->event_plugin_name ? p_opts->event_plugin_name : null_str); fprintf(out, "#\n# Node name map for mapping node's to more descriptive node descriptions\n" "# (man ibnetdiscover for more information)\n#\n" "node_name_map_name %s\n\n", p_opts->node_name_map_name ? p_opts->node_name_map_name : null_str); fprintf(out, "#\n# DEBUG FEATURES\n#\n" "# The log flags used\n" "log_flags 0x%02x\n\n" "# Force flush of the log file after each log message\n" "force_log_flush %s\n\n" "# Log file to be used\n" "log_file %s\n\n" "# Limit the size of the log file in MB. If overrun, log is restarted\n" "log_max_size %lu\n\n" "# If TRUE will accumulate the log over multiple OpenSM sessions\n" "accum_log_file %s\n\n" "# The directory to hold the file OpenSM dumps\n" "dump_files_dir %s\n\n" "# If TRUE enables new high risk options and hardware specific quirks\n" "enable_quirks %s\n\n" "# If TRUE disables client reregistration\n" "no_clients_rereg %s\n\n" "# If TRUE OpenSM should disable multicast support and\n" "# no multicast routing is performed if TRUE\n" "disable_multicast %s\n\n" "# If TRUE opensm will exit on fatal initialization issues\n" "exit_on_fatal %s\n\n" "# console [off|local" #ifdef ENABLE_OSM_CONSOLE_SOCKET "|loopback|socket]\n" #else "]\n" #endif "console %s\n\n" "# Telnet port for console (default %d)\n" "console_port %d\n\n", p_opts->log_flags, p_opts->force_log_flush ? "TRUE" : "FALSE", p_opts->log_file, p_opts->log_max_size/1024/1024, p_opts->accum_log_file ? "TRUE" : "FALSE", p_opts->dump_files_dir, p_opts->enable_quirks ? "TRUE" : "FALSE", p_opts->no_clients_rereg ? "TRUE" : "FALSE", p_opts->disable_multicast ? "TRUE" : "FALSE", p_opts->exit_on_fatal ? "TRUE" : "FALSE", p_opts->console, OSM_DEFAULT_CONSOLE_PORT, p_opts->console_port); fprintf(out, "#\n# QoS OPTIONS\n#\n" "# Enable QoS setup\n" "qos %s\n\n" "# QoS policy file to be used\n" "qos_policy_file %s\n\n", p_opts->qos ? "TRUE" : "FALSE", p_opts->qos_policy_file); subn_dump_qos_options(out, "QoS default options", "qos", &p_opts->qos_options); fprintf(out, "\n"); subn_dump_qos_options(out, "QoS CA options", "qos_ca", &p_opts->qos_ca_options); fprintf(out, "\n"); subn_dump_qos_options(out, "QoS Switch Port 0 options", "qos_sw0", &p_opts->qos_sw0_options); fprintf(out, "\n"); subn_dump_qos_options(out, "QoS Switch external ports options", "qos_swe", &p_opts->qos_swe_options); fprintf(out, "\n"); subn_dump_qos_options(out, "QoS Router ports options", "qos_rtr", &p_opts->qos_rtr_options); fprintf(out, "\n"); fprintf(out, "# Prefix routes file name\n" "prefix_routes_file %s\n\n", p_opts->prefix_routes_file); fprintf(out, "#\n# IPv6 Solicited Node Multicast (SNM) Options\n#\n" "consolidate_ipv6_snm_req %s\n\n", p_opts->consolidate_ipv6_snm_req ? "TRUE" : "FALSE"); /* optional string attributes ... */ return 0; } int osm_subn_write_conf_file(char *file_name, IN osm_subn_opt_t *const p_opts) { FILE *opts_file; opts_file = fopen(file_name, "w"); if (!opts_file) { printf("cannot open file \'%s\' for writing: %s\n", file_name, strerror(errno)); return -1; } if (osm_subn_output_conf(opts_file, p_opts) < 0) return -1; fclose(opts_file); return 0; } Index: stable/9/contrib =================================================================== --- stable/9/contrib (revision 273863) +++ stable/9/contrib (revision 273864) Property changes on: stable/9/contrib ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/contrib:r230135,247671,247675,254121-254123,254734,255932,255969-255970,255972-255973,256116,256179,256269,256281,256470,256682,257864,257867,258276,259608,268314,269861-269862,271127,272407 Index: stable/9/sys/conf/files =================================================================== --- stable/9/sys/conf/files (revision 273863) +++ stable/9/sys/conf/files (revision 273864) @@ -1,3689 +1,3704 @@ # $FreeBSD$ # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # acpi_quirks.h optional acpi \ dependency "$S/tools/acpi_quirks2h.awk $S/dev/acpica/acpi_quirks" \ compile-with "${AWK} -f $S/tools/acpi_quirks2h.awk $S/dev/acpica/acpi_quirks" \ no-obj no-implicit-rule before-depend \ clean "acpi_quirks.h" aicasm optional ahc | ahd \ dependency "$S/dev/aic7xxx/aicasm/*.[chyl]" \ compile-with "CC='${CC}' ${MAKE} -f $S/dev/aic7xxx/aicasm/Makefile MAKESRCPATH=$S/dev/aic7xxx/aicasm" \ no-obj no-implicit-rule \ clean "aicasm* y.tab.h" aic7xxx_seq.h optional ahc \ compile-with "./aicasm ${INCLUDES} -I$S/cam/scsi -I$S/dev/aic7xxx -o aic7xxx_seq.h -r aic7xxx_reg.h -p aic7xxx_reg_print.c -i $S/dev/aic7xxx/aic7xxx_osm.h $S/dev/aic7xxx/aic7xxx.seq" \ no-obj no-implicit-rule before-depend local \ clean "aic7xxx_seq.h" \ dependency "$S/dev/aic7xxx/aic7xxx.{reg,seq} $S/cam/scsi/scsi_message.h aicasm" aic7xxx_reg.h optional ahc \ compile-with "./aicasm ${INCLUDES} -I$S/cam/scsi -I$S/dev/aic7xxx -o aic7xxx_seq.h -r aic7xxx_reg.h -p aic7xxx_reg_print.c -i $S/dev/aic7xxx/aic7xxx_osm.h $S/dev/aic7xxx/aic7xxx.seq" \ no-obj no-implicit-rule before-depend local \ clean "aic7xxx_reg.h" \ dependency "$S/dev/aic7xxx/aic7xxx.{reg,seq} $S/cam/scsi/scsi_message.h aicasm" aic7xxx_reg_print.c optional ahc \ compile-with "./aicasm ${INCLUDES} -I$S/cam/scsi -I$S/dev/aic7xxx -o aic7xxx_seq.h -r aic7xxx_reg.h -p aic7xxx_reg_print.c -i $S/dev/aic7xxx/aic7xxx_osm.h $S/dev/aic7xxx/aic7xxx.seq" \ no-obj no-implicit-rule local \ clean "aic7xxx_reg_print.c" \ dependency "$S/dev/aic7xxx/aic7xxx.{reg,seq} $S/cam/scsi/scsi_message.h aicasm" aic7xxx_reg_print.o optional ahc ahc_reg_pretty_print \ compile-with "${NORMAL_C}" \ no-implicit-rule local aic79xx_seq.h optional ahd pci \ compile-with "./aicasm ${INCLUDES} -I$S/cam/scsi -I$S/dev/aic7xxx -o aic79xx_seq.h -r aic79xx_reg.h -p aic79xx_reg_print.c -i $S/dev/aic7xxx/aic79xx_osm.h $S/dev/aic7xxx/aic79xx.seq" \ no-obj no-implicit-rule before-depend local \ clean "aic79xx_seq.h" \ dependency "$S/dev/aic7xxx/aic79xx.{reg,seq} $S/cam/scsi/scsi_message.h aicasm" aic79xx_reg.h optional ahd pci \ compile-with "./aicasm ${INCLUDES} -I$S/cam/scsi -I$S/dev/aic7xxx -o aic79xx_seq.h -r aic79xx_reg.h -p aic79xx_reg_print.c -i $S/dev/aic7xxx/aic79xx_osm.h $S/dev/aic7xxx/aic79xx.seq" \ no-obj no-implicit-rule before-depend local \ clean "aic79xx_reg.h" \ dependency "$S/dev/aic7xxx/aic79xx.{reg,seq} $S/cam/scsi/scsi_message.h aicasm" aic79xx_reg_print.c optional ahd pci \ compile-with "./aicasm ${INCLUDES} -I$S/cam/scsi -I$S/dev/aic7xxx -o aic79xx_seq.h -r aic79xx_reg.h -p aic79xx_reg_print.c -i $S/dev/aic7xxx/aic79xx_osm.h $S/dev/aic7xxx/aic79xx.seq" \ no-obj no-implicit-rule local \ clean "aic79xx_reg_print.c" \ dependency "$S/dev/aic7xxx/aic79xx.{reg,seq} $S/cam/scsi/scsi_message.h aicasm" aic79xx_reg_print.o optional ahd pci ahd_reg_pretty_print \ compile-with "${NORMAL_C}" \ no-implicit-rule local # # The 'fdt_dtb_file' target covers an actual DTB file name, which is derived # from the specified source (DTS) file: .dts -> .dtb # fdt_dtb_file optional fdt \ compile-with "if [ -f $S/boot/fdt/dts/${FDT_DTS_FILE} ]; then dtc -O dtb -o `echo ${FDT_DTS_FILE} | cut -d. -f1`.dtb -b 0 -p 1024 $S/boot/fdt/dts/${FDT_DTS_FILE}; fi" \ no-obj no-implicit-rule before-depend \ clean "`echo ${FDT_DTS_FILE} | cut -d. -f1`.dtb" fdt_static_dtb.h optional fdt fdt_dtb_static \ compile-with "sh $S/tools/fdt/make_dtbh.sh ${FDT_DTS_FILE} ." \ no-obj no-implicit-rule before-depend \ clean "fdt_static_dtb.h" feeder_eq_gen.h optional sound \ dependency "$S/tools/sound/feeder_eq_mkfilter.awk" \ compile-with "${AWK} -f $S/tools/sound/feeder_eq_mkfilter.awk -- ${FEEDER_EQ_PRESETS} > feeder_eq_gen.h" \ no-obj no-implicit-rule before-depend \ clean "feeder_eq_gen.h" feeder_rate_gen.h optional sound \ dependency "$S/tools/sound/feeder_rate_mkfilter.awk" \ compile-with "${AWK} -f $S/tools/sound/feeder_rate_mkfilter.awk -- ${FEEDER_RATE_PRESETS} > feeder_rate_gen.h" \ no-obj no-implicit-rule before-depend \ clean "feeder_rate_gen.h" snd_fxdiv_gen.h optional sound \ dependency "$S/tools/sound/snd_fxdiv_gen.awk" \ compile-with "${AWK} -f $S/tools/sound/snd_fxdiv_gen.awk -- > snd_fxdiv_gen.h" \ no-obj no-implicit-rule before-depend \ clean "snd_fxdiv_gen.h" miidevs.h optional miibus | mii \ dependency "$S/tools/miidevs2h.awk $S/dev/mii/miidevs" \ compile-with "${AWK} -f $S/tools/miidevs2h.awk $S/dev/mii/miidevs" \ no-obj no-implicit-rule before-depend \ clean "miidevs.h" pccarddevs.h standard \ dependency "$S/tools/pccarddevs2h.awk $S/dev/pccard/pccarddevs" \ compile-with "${AWK} -f $S/tools/pccarddevs2h.awk $S/dev/pccard/pccarddevs" \ no-obj no-implicit-rule before-depend \ clean "pccarddevs.h" teken_state.h optional sc | vt \ dependency "$S/teken/gensequences $S/teken/sequences" \ compile-with "${AWK} -f $S/teken/gensequences $S/teken/sequences > teken_state.h" \ no-obj no-implicit-rule before-depend \ clean "teken_state.h" usbdevs.h optional usb \ dependency "$S/tools/usbdevs2h.awk $S/dev/usb/usbdevs" \ compile-with "${AWK} -f $S/tools/usbdevs2h.awk $S/dev/usb/usbdevs -h" \ no-obj no-implicit-rule before-depend \ clean "usbdevs.h" usbdevs_data.h optional usb \ dependency "$S/tools/usbdevs2h.awk $S/dev/usb/usbdevs" \ compile-with "${AWK} -f $S/tools/usbdevs2h.awk $S/dev/usb/usbdevs -d" \ no-obj no-implicit-rule before-depend \ clean "usbdevs_data.h" cam/cam.c optional scbus cam/cam_compat.c optional scbus cam/cam_periph.c optional scbus cam/cam_queue.c optional scbus cam/cam_sim.c optional scbus cam/cam_xpt.c optional scbus cam/ata/ata_all.c optional scbus cam/ata/ata_xpt.c optional scbus cam/ata/ata_pmp.c optional scbus cam/scsi/scsi_xpt.c optional scbus cam/scsi/scsi_all.c optional scbus cam/scsi/scsi_cd.c optional cd cam/scsi/scsi_ch.c optional ch cam/ata/ata_da.c optional ada | da cam/ctl/ctl.c optional ctl cam/ctl/ctl_backend.c optional ctl cam/ctl/ctl_backend_block.c optional ctl cam/ctl/ctl_backend_ramdisk.c optional ctl cam/ctl/ctl_cmd_table.c optional ctl cam/ctl/ctl_frontend.c optional ctl cam/ctl/ctl_frontend_cam_sim.c optional ctl cam/ctl/ctl_frontend_internal.c optional ctl cam/ctl/ctl_scsi_all.c optional ctl cam/ctl/ctl_error.c optional ctl cam/ctl/ctl_util.c optional ctl cam/ctl/scsi_ctl.c optional ctl cam/scsi/scsi_da.c optional da cam/scsi/scsi_low.c optional ct | ncv | nsp | stg cam/scsi/scsi_low_pisa.c optional ct | ncv | nsp | stg cam/scsi/scsi_pass.c optional pass cam/scsi/scsi_pt.c optional pt cam/scsi/scsi_sa.c optional sa cam/scsi/scsi_enc.c optional ses cam/scsi/scsi_enc_ses.c optional ses cam/scsi/scsi_enc_safte.c optional ses cam/scsi/scsi_sg.c optional sg cam/scsi/scsi_targ_bh.c optional targbh cam/scsi/scsi_target.c optional targ cam/scsi/smp_all.c optional scbus contrib/altq/altq/altq_cbq.c optional altq \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/altq/altq/altq_cdnr.c optional altq contrib/altq/altq/altq_hfsc.c optional altq \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/altq/altq/altq_priq.c optional altq \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/altq/altq/altq_red.c optional altq \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/altq/altq/altq_rio.c optional altq \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/altq/altq/altq_rmclass.c optional altq contrib/altq/altq/altq_subr.c optional altq \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/dev/acpica/debugger/dbcmds.c optional acpi acpi_debug contrib/dev/acpica/debugger/dbdisply.c optional acpi acpi_debug contrib/dev/acpica/debugger/dbexec.c optional acpi acpi_debug contrib/dev/acpica/debugger/dbfileio.c optional acpi acpi_debug contrib/dev/acpica/debugger/dbhistry.c optional acpi acpi_debug contrib/dev/acpica/debugger/dbinput.c optional acpi acpi_debug contrib/dev/acpica/debugger/dbmethod.c optional acpi acpi_debug contrib/dev/acpica/debugger/dbnames.c optional acpi acpi_debug contrib/dev/acpica/debugger/dbstats.c optional acpi acpi_debug contrib/dev/acpica/debugger/dbutils.c optional acpi acpi_debug contrib/dev/acpica/debugger/dbxface.c optional acpi acpi_debug contrib/dev/acpica/disassembler/dmbuffer.c optional acpi acpi_debug contrib/dev/acpica/disassembler/dmnames.c optional acpi acpi_debug contrib/dev/acpica/disassembler/dmopcode.c optional acpi acpi_debug contrib/dev/acpica/disassembler/dmobject.c optional acpi acpi_debug contrib/dev/acpica/disassembler/dmresrc.c optional acpi acpi_debug contrib/dev/acpica/disassembler/dmresrcl.c optional acpi acpi_debug contrib/dev/acpica/disassembler/dmresrcs.c optional acpi acpi_debug contrib/dev/acpica/disassembler/dmutils.c optional acpi acpi_debug contrib/dev/acpica/disassembler/dmwalk.c optional acpi acpi_debug contrib/dev/acpica/dispatcher/dsargs.c optional acpi contrib/dev/acpica/dispatcher/dscontrol.c optional acpi contrib/dev/acpica/dispatcher/dsfield.c optional acpi contrib/dev/acpica/dispatcher/dsinit.c optional acpi contrib/dev/acpica/dispatcher/dsmethod.c optional acpi contrib/dev/acpica/dispatcher/dsmthdat.c optional acpi contrib/dev/acpica/dispatcher/dsobject.c optional acpi contrib/dev/acpica/dispatcher/dsopcode.c optional acpi contrib/dev/acpica/dispatcher/dsutils.c optional acpi contrib/dev/acpica/dispatcher/dswexec.c optional acpi contrib/dev/acpica/dispatcher/dswload.c optional acpi contrib/dev/acpica/dispatcher/dswload2.c optional acpi contrib/dev/acpica/dispatcher/dswscope.c optional acpi contrib/dev/acpica/dispatcher/dswstate.c optional acpi contrib/dev/acpica/events/evevent.c optional acpi contrib/dev/acpica/events/evglock.c optional acpi contrib/dev/acpica/events/evgpe.c optional acpi contrib/dev/acpica/events/evgpeblk.c optional acpi contrib/dev/acpica/events/evgpeinit.c optional acpi contrib/dev/acpica/events/evgpeutil.c optional acpi contrib/dev/acpica/events/evmisc.c optional acpi contrib/dev/acpica/events/evregion.c optional acpi contrib/dev/acpica/events/evrgnini.c optional acpi contrib/dev/acpica/events/evsci.c optional acpi contrib/dev/acpica/events/evxface.c optional acpi contrib/dev/acpica/events/evxfevnt.c optional acpi contrib/dev/acpica/events/evxfgpe.c optional acpi contrib/dev/acpica/events/evxfregn.c optional acpi contrib/dev/acpica/executer/exconfig.c optional acpi contrib/dev/acpica/executer/exconvrt.c optional acpi contrib/dev/acpica/executer/excreate.c optional acpi contrib/dev/acpica/executer/exdebug.c optional acpi contrib/dev/acpica/executer/exdump.c optional acpi contrib/dev/acpica/executer/exfield.c optional acpi contrib/dev/acpica/executer/exfldio.c optional acpi contrib/dev/acpica/executer/exmisc.c optional acpi contrib/dev/acpica/executer/exmutex.c optional acpi contrib/dev/acpica/executer/exnames.c optional acpi contrib/dev/acpica/executer/exoparg1.c optional acpi contrib/dev/acpica/executer/exoparg2.c optional acpi contrib/dev/acpica/executer/exoparg3.c optional acpi contrib/dev/acpica/executer/exoparg6.c optional acpi contrib/dev/acpica/executer/exprep.c optional acpi contrib/dev/acpica/executer/exregion.c optional acpi contrib/dev/acpica/executer/exresnte.c optional acpi contrib/dev/acpica/executer/exresolv.c optional acpi contrib/dev/acpica/executer/exresop.c optional acpi contrib/dev/acpica/executer/exstore.c optional acpi contrib/dev/acpica/executer/exstoren.c optional acpi contrib/dev/acpica/executer/exstorob.c optional acpi contrib/dev/acpica/executer/exsystem.c optional acpi contrib/dev/acpica/executer/exutils.c optional acpi contrib/dev/acpica/hardware/hwacpi.c optional acpi contrib/dev/acpica/hardware/hwgpe.c optional acpi contrib/dev/acpica/hardware/hwpci.c optional acpi contrib/dev/acpica/hardware/hwregs.c optional acpi contrib/dev/acpica/hardware/hwsleep.c optional acpi contrib/dev/acpica/hardware/hwtimer.c optional acpi contrib/dev/acpica/hardware/hwvalid.c optional acpi contrib/dev/acpica/hardware/hwxface.c optional acpi contrib/dev/acpica/namespace/nsaccess.c optional acpi contrib/dev/acpica/namespace/nsalloc.c optional acpi contrib/dev/acpica/namespace/nsdump.c optional acpi contrib/dev/acpica/namespace/nseval.c optional acpi contrib/dev/acpica/namespace/nsinit.c optional acpi contrib/dev/acpica/namespace/nsload.c optional acpi contrib/dev/acpica/namespace/nsnames.c optional acpi contrib/dev/acpica/namespace/nsobject.c optional acpi contrib/dev/acpica/namespace/nsparse.c optional acpi contrib/dev/acpica/namespace/nspredef.c optional acpi contrib/dev/acpica/namespace/nsrepair.c optional acpi contrib/dev/acpica/namespace/nsrepair2.c optional acpi contrib/dev/acpica/namespace/nssearch.c optional acpi contrib/dev/acpica/namespace/nsutils.c optional acpi contrib/dev/acpica/namespace/nswalk.c optional acpi contrib/dev/acpica/namespace/nsxfeval.c optional acpi contrib/dev/acpica/namespace/nsxfname.c optional acpi contrib/dev/acpica/namespace/nsxfobj.c optional acpi contrib/dev/acpica/parser/psargs.c optional acpi contrib/dev/acpica/parser/psloop.c optional acpi contrib/dev/acpica/parser/psopcode.c optional acpi contrib/dev/acpica/parser/psparse.c optional acpi contrib/dev/acpica/parser/psscope.c optional acpi contrib/dev/acpica/parser/pstree.c optional acpi contrib/dev/acpica/parser/psutils.c optional acpi contrib/dev/acpica/parser/pswalk.c optional acpi contrib/dev/acpica/parser/psxface.c optional acpi contrib/dev/acpica/resources/rsaddr.c optional acpi contrib/dev/acpica/resources/rscalc.c optional acpi contrib/dev/acpica/resources/rscreate.c optional acpi contrib/dev/acpica/resources/rsdump.c optional acpi contrib/dev/acpica/resources/rsinfo.c optional acpi contrib/dev/acpica/resources/rsio.c optional acpi contrib/dev/acpica/resources/rsirq.c optional acpi contrib/dev/acpica/resources/rslist.c optional acpi contrib/dev/acpica/resources/rsmemory.c optional acpi contrib/dev/acpica/resources/rsmisc.c optional acpi contrib/dev/acpica/resources/rsutils.c optional acpi contrib/dev/acpica/resources/rsxface.c optional acpi contrib/dev/acpica/tables/tbfadt.c optional acpi contrib/dev/acpica/tables/tbfind.c optional acpi contrib/dev/acpica/tables/tbinstal.c optional acpi contrib/dev/acpica/tables/tbutils.c optional acpi contrib/dev/acpica/tables/tbxface.c optional acpi contrib/dev/acpica/tables/tbxfroot.c optional acpi contrib/dev/acpica/utilities/utalloc.c optional acpi contrib/dev/acpica/utilities/utcache.c optional acpi contrib/dev/acpica/utilities/utcopy.c optional acpi contrib/dev/acpica/utilities/utdebug.c optional acpi contrib/dev/acpica/utilities/utdecode.c optional acpi contrib/dev/acpica/utilities/utdelete.c optional acpi contrib/dev/acpica/utilities/uteval.c optional acpi contrib/dev/acpica/utilities/utglobal.c optional acpi contrib/dev/acpica/utilities/utids.c optional acpi contrib/dev/acpica/utilities/utinit.c optional acpi contrib/dev/acpica/utilities/utlock.c optional acpi contrib/dev/acpica/utilities/utmath.c optional acpi contrib/dev/acpica/utilities/utmisc.c optional acpi contrib/dev/acpica/utilities/utmutex.c optional acpi contrib/dev/acpica/utilities/utobject.c optional acpi contrib/dev/acpica/utilities/utosi.c optional acpi contrib/dev/acpica/utilities/utresrc.c optional acpi contrib/dev/acpica/utilities/utstate.c optional acpi contrib/dev/acpica/utilities/utxface.c optional acpi contrib/dev/acpica/utilities/utxferror.c optional acpi contrib/ipfilter/netinet/fil.c optional ipfilter inet \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_auth.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_fil_freebsd.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_frag.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_log.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_nat.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_proxy.c optional ipfilter inet \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_state.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_lookup.c optional ipfilter inet \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -Wno-error -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_pool.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_htable.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/ip_sync.c optional ipfilter inet \ compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter" contrib/ipfilter/netinet/mlfk_ipl.c optional ipfilter inet \ compile-with "${NORMAL_C} -I$S/contrib/ipfilter" contrib/libfdt/fdt.c optional fdt contrib/libfdt/fdt_ro.c optional fdt contrib/libfdt/fdt_rw.c optional fdt contrib/libfdt/fdt_strerror.c optional fdt contrib/libfdt/fdt_sw.c optional fdt contrib/libfdt/fdt_wip.c optional fdt contrib/ngatm/netnatm/api/cc_conn.c optional ngatm_ccatm \ compile-with "${NORMAL_C_NOWERROR} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_data.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_dump.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_port.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_sig.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/cc_user.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/api/unisap.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/misc/straddr.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/misc/unimsg_common.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/msg/traffic.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/msg/uni_ie.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/msg/uni_msg.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/saal/saal_sscfu.c optional ngatm_sscfu \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/saal/saal_sscop.c optional ngatm_sscop \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_call.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_coord.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_party.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_print.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_reset.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_uni.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_unimsgcpy.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/ngatm/netnatm/sig/sig_verify.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" contrib/pf/net/if_pflog.c optional pflog pf inet \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/if_pfsync.c optional pfsync pf inet \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf.c optional pf inet \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf_if.c optional pf inet \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf_ioctl.c optional pf inet \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf_lb.c optional pf inet \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf_norm.c optional pf inet \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf_osfp.c optional pf inet \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf_ruleset.c optional pf inet \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/net/pf_table.c optional pf inet \ compile-with "${NORMAL_C} -I$S/contrib/pf" contrib/pf/netinet/in4_cksum.c optional pf inet crypto/blowfish/bf_ecb.c optional ipsec crypto/blowfish/bf_skey.c optional crypto | ipsec crypto/camellia/camellia.c optional crypto | ipsec crypto/camellia/camellia-api.c optional crypto | ipsec crypto/des/des_ecb.c optional crypto | ipsec | netsmb crypto/des/des_setkey.c optional crypto | ipsec | netsmb crypto/rc4/rc4.c optional netgraph_mppc_encryption | kgssapi crypto/rijndael/rijndael-alg-fst.c optional crypto | geom_bde | \ ipsec | random | wlan_ccmp crypto/rijndael/rijndael-api-fst.c optional geom_bde | random crypto/rijndael/rijndael-api.c optional crypto | ipsec | wlan_ccmp crypto/sha1.c optional carp | crypto | ipsec | \ netgraph_mppc_encryption | sctp crypto/sha2/sha2.c optional crypto | geom_bde | ipsec | random | \ sctp ddb/db_access.c optional ddb ddb/db_break.c optional ddb ddb/db_capture.c optional ddb ddb/db_command.c optional ddb ddb/db_examine.c optional ddb ddb/db_expr.c optional ddb ddb/db_input.c optional ddb ddb/db_lex.c optional ddb ddb/db_main.c optional ddb ddb/db_output.c optional ddb ddb/db_print.c optional ddb ddb/db_ps.c optional ddb ddb/db_run.c optional ddb ddb/db_script.c optional ddb ddb/db_sym.c optional ddb ddb/db_thread.c optional ddb ddb/db_textdump.c optional ddb ddb/db_variables.c optional ddb ddb/db_watch.c optional ddb ddb/db_write_cmd.c optional ddb #dev/dpt/dpt_control.c optional dpt dev/aac/aac.c optional aac dev/aac/aac_cam.c optional aacp aac dev/aac/aac_debug.c optional aac dev/aac/aac_disk.c optional aac dev/aac/aac_linux.c optional aac compat_linux dev/aac/aac_pci.c optional aac pci dev/aacraid/aacraid.c optional aacraid dev/aacraid/aacraid_cam.c optional aacraid scbus dev/aacraid/aacraid_debug.c optional aacraid dev/aacraid/aacraid_linux.c optional aacraid compat_linux dev/aacraid/aacraid_pci.c optional aacraid pci dev/acpi_support/acpi_wmi.c optional acpi_wmi acpi dev/acpi_support/acpi_asus.c optional acpi_asus acpi dev/acpi_support/acpi_fujitsu.c optional acpi_fujitsu acpi dev/acpi_support/acpi_hp.c optional acpi_hp acpi dev/acpi_support/acpi_ibm.c optional acpi_ibm acpi dev/acpi_support/acpi_panasonic.c optional acpi_panasonic acpi dev/acpi_support/acpi_sony.c optional acpi_sony acpi dev/acpi_support/acpi_toshiba.c optional acpi_toshiba acpi dev/acpi_support/atk0110.c optional aibs acpi dev/acpica/Osd/OsdDebug.c optional acpi dev/acpica/Osd/OsdHardware.c optional acpi dev/acpica/Osd/OsdInterrupt.c optional acpi dev/acpica/Osd/OsdMemory.c optional acpi dev/acpica/Osd/OsdSchedule.c optional acpi dev/acpica/Osd/OsdStream.c optional acpi dev/acpica/Osd/OsdSynch.c optional acpi dev/acpica/Osd/OsdTable.c optional acpi dev/acpica/acpi.c optional acpi dev/acpica/acpi_acad.c optional acpi dev/acpica/acpi_battery.c optional acpi dev/acpica/acpi_button.c optional acpi dev/acpica/acpi_cmbat.c optional acpi dev/acpica/acpi_cpu.c optional acpi dev/acpica/acpi_ec.c optional acpi dev/acpica/acpi_hpet.c optional acpi dev/acpica/acpi_isab.c optional acpi isa dev/acpica/acpi_lid.c optional acpi dev/acpica/acpi_package.c optional acpi dev/acpica/acpi_pci.c optional acpi pci dev/acpica/acpi_pci_link.c optional acpi pci dev/acpica/acpi_pcib.c optional acpi pci dev/acpica/acpi_pcib_acpi.c optional acpi pci dev/acpica/acpi_pcib_pci.c optional acpi pci dev/acpica/acpi_perf.c optional acpi dev/acpica/acpi_powerres.c optional acpi dev/acpica/acpi_quirk.c optional acpi dev/acpica/acpi_resource.c optional acpi dev/acpica/acpi_smbat.c optional acpi dev/acpica/acpi_thermal.c optional acpi dev/acpica/acpi_throttle.c optional acpi dev/acpica/acpi_timer.c optional acpi dev/acpica/acpi_video.c optional acpi_video acpi dev/acpica/acpi_dock.c optional acpi_dock acpi dev/adlink/adlink.c optional adlink dev/advansys/adv_eisa.c optional adv eisa dev/advansys/adv_pci.c optional adv pci dev/advansys/advansys.c optional adv dev/advansys/advlib.c optional adv dev/advansys/advmcode.c optional adv dev/advansys/adw_pci.c optional adw pci dev/advansys/adwcam.c optional adw dev/advansys/adwlib.c optional adw dev/advansys/adwmcode.c optional adw dev/ae/if_ae.c optional ae pci dev/age/if_age.c optional age pci dev/agp/agp.c optional agp pci dev/agp/agp_if.m optional agp pci dev/aha/aha.c optional aha dev/aha/aha_isa.c optional aha isa dev/aha/aha_mca.c optional aha mca dev/ahb/ahb.c optional ahb eisa dev/ahci/ahci.c optional ahci pci dev/aic/aic.c optional aic dev/aic/aic_pccard.c optional aic pccard dev/aic7xxx/ahc_eisa.c optional ahc eisa dev/aic7xxx/ahc_isa.c optional ahc isa dev/aic7xxx/ahc_pci.c optional ahc pci \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/aic7xxx/ahd_pci.c optional ahd pci \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/aic7xxx/aic7770.c optional ahc dev/aic7xxx/aic79xx.c optional ahd pci dev/aic7xxx/aic79xx_osm.c optional ahd pci dev/aic7xxx/aic79xx_pci.c optional ahd pci dev/aic7xxx/aic7xxx.c optional ahc dev/aic7xxx/aic7xxx_93cx6.c optional ahc dev/aic7xxx/aic7xxx_osm.c optional ahc dev/aic7xxx/aic7xxx_pci.c optional ahc pci dev/alc/if_alc.c optional alc pci dev/ale/if_ale.c optional ale pci dev/amd/amd.c optional amd dev/amr/amr.c optional amr dev/amr/amr_cam.c optional amrp amr dev/amr/amr_disk.c optional amr dev/amr/amr_linux.c optional amr compat_linux dev/amr/amr_pci.c optional amr pci dev/an/if_an.c optional an dev/an/if_an_isa.c optional an isa dev/an/if_an_pccard.c optional an pccard dev/an/if_an_pci.c optional an pci dev/asr/asr.c optional asr pci \ compile-with "${NORMAL_C} ${NO_WARRAY_BOUNDS}" # dev/ata/ata_if.m optional ata | atacore dev/ata/ata-all.c optional ata | atacore dev/ata/ata-dma.c optional ata | atacore dev/ata/ata-lowlevel.c optional ata | atacore dev/ata/ata-queue.c optional ata | atacore dev/ata/ata-sata.c optional ata | atacore dev/ata/ata-card.c optional ata pccard | atapccard dev/ata/ata-cbus.c optional ata pc98 | atapc98 dev/ata/ata-isa.c optional ata isa | ataisa dev/ata/ata-pci.c optional ata pci | atapci dev/ata/chipsets/ata-ahci.c optional ata pci | ataahci | ataacerlabs | \ ataati | ataintel | atajmicron | \ atavia | atanvidia dev/ata/chipsets/ata-acard.c optional ata pci | ataacard dev/ata/chipsets/ata-acerlabs.c optional ata pci | ataacerlabs dev/ata/chipsets/ata-adaptec.c optional ata pci | ataadaptec dev/ata/chipsets/ata-amd.c optional ata pci | ataamd dev/ata/chipsets/ata-ati.c optional ata pci | ataati dev/ata/chipsets/ata-cenatek.c optional ata pci | atacenatek dev/ata/chipsets/ata-cypress.c optional ata pci | atacypress dev/ata/chipsets/ata-cyrix.c optional ata pci | atacyrix dev/ata/chipsets/ata-highpoint.c optional ata pci | atahighpoint dev/ata/chipsets/ata-intel.c optional ata pci | ataintel dev/ata/chipsets/ata-ite.c optional ata pci | ataite dev/ata/chipsets/ata-jmicron.c optional ata pci | atajmicron dev/ata/chipsets/ata-marvell.c optional ata pci | atamarvell | ataadaptec dev/ata/chipsets/ata-micron.c optional ata pci | atamicron dev/ata/chipsets/ata-national.c optional ata pci | atanational dev/ata/chipsets/ata-netcell.c optional ata pci | atanetcell dev/ata/chipsets/ata-nvidia.c optional ata pci | atanvidia dev/ata/chipsets/ata-promise.c optional ata pci | atapromise dev/ata/chipsets/ata-serverworks.c optional ata pci | ataserverworks dev/ata/chipsets/ata-siliconimage.c optional ata pci | atasiliconimage | ataati dev/ata/chipsets/ata-sis.c optional ata pci | atasis dev/ata/chipsets/ata-via.c optional ata pci | atavia dev/ata/ata-disk.c optional atadisk dev/ata/ata-raid.c optional ataraid dev/ata/atapi-cd.c optional atapicd dev/ata/atapi-fd.c optional atapifd dev/ata/atapi-tape.c optional atapist dev/ata/atapi-cam.c optional atapicam # dev/ath/if_ath_pci.c optional ath_pci pci \ compile-with "${NORMAL_C} -I$S/dev/ath" # dev/ath/if_ath_ahb.c optional ath_ahb \ compile-with "${NORMAL_C} -I$S/dev/ath" # # XXX Work around clang warning, until maintainer approves fix. dev/ath/if_ath.c optional ath \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED} -I$S/dev/ath" dev/ath/if_ath_debug.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_keycache.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_tx.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_tx_ht.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/if_ath_sysctl.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ah_osdep.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" # dev/ath/ath_hal/ah.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_v1.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_v3.c optional ath_hal | ath_ar5211 | ath_ar5212 \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_v14.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_hal/ah_eeprom_v4k.c \ optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath" # XXX Work around clang warning, until maintainer approves fix. dev/ath/ath_hal/ah_eeprom_9287.c \ optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED} -I$S/dev/ath" dev/ath/ath_hal/ah_regdomain.c optional ath \ compile-with "${NORMAL_C} ${NO_WSHIFT_COUNT_NEGATIVE} ${NO_WSHIFT_COUNT_OVERFLOW} -I$S/dev/ath" # ar5210 dev/ath/ath_hal/ar5210/ar5210_attach.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_beacon.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_interrupts.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_keycache.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_misc.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_phy.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_power.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_recv.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_reset.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5210/ar5210_xmit.c optional ath_hal | ath_ar5210 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar5211 dev/ath/ath_hal/ar5211/ar5211_attach.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_beacon.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_interrupts.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_keycache.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_misc.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_phy.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_power.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_recv.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_reset.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5211/ar5211_xmit.c optional ath_hal | ath_ar5211 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar5212 dev/ath/ath_hal/ar5212/ar5212_ani.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_attach.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_beacon.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_eeprom.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_gpio.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_interrupts.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_keycache.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_misc.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_phy.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_power.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_recv.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_reset.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_rfgain.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5212_xmit.c \ optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 | \ ath_ar9285 ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar5416 (depends on ar5212) dev/ath/ath_hal/ar5416/ar5416_ani.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_attach.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_beacon.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_cal.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_cal_iq.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_cal_adcgain.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_cal_adcdc.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_eeprom.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_gpio.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_interrupts.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_keycache.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_misc.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_phy.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_power.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_recv.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_reset.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar5416_xmit.c \ optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 | ath_ar9285 | \ ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9130 (depends upon ar5416) - also requires AH_SUPPORT_AR9130 dev/ath/ath_hal/ar9001/ar9130_attach.c optional ath_hal | ath_ar9130 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9001/ar9130_phy.c optional ath_hal | ath_ar9130 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9001/ar9130_eeprom.c optional ath_hal | ath_ar9130 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9160 (depends on ar5416) dev/ath/ath_hal/ar9001/ar9160_attach.c optional ath_hal | ath_ar9160 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9280 (depends on ar5416) dev/ath/ath_hal/ar9002/ar9280_attach.c optional ath_hal | ath_ar9280 | \ ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9280_olc.c optional ath_hal | ath_ar9280 | \ ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9285 (depends on ar5416 and ar9280) dev/ath/ath_hal/ar9002/ar9285_attach.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_reset.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_cal.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_phy.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285_diversity.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ar9287 (depends on ar5416) dev/ath/ath_hal/ar9002/ar9287_attach.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9287_reset.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9287_cal.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9287_olc.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # rf backends dev/ath/ath_hal/ar5212/ar2316.c optional ath_rf2316 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar2317.c optional ath_rf2317 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar2413.c optional ath_hal | ath_rf2413 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar2425.c optional ath_hal | ath_rf2425 | ath_rf2417 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5111.c optional ath_hal | ath_rf5111 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5112.c optional ath_hal | ath_rf5112 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5212/ar5413.c optional ath_hal | ath_rf5413 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar5416/ar2133.c optional ath_hal | ath_ar5416 | ath_ar9160 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9280.c optional ath_hal | ath_ar9280 | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9285.c optional ath_hal | ath_ar9285 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" dev/ath/ath_hal/ar9002/ar9287.c optional ath_hal | ath_ar9287 \ compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal" # ath rate control algorithms dev/ath/ath_rate/amrr/amrr.c optional ath_rate_amrr \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_rate/onoe/onoe.c optional ath_rate_onoe \ compile-with "${NORMAL_C} -I$S/dev/ath" dev/ath/ath_rate/sample/sample.c optional ath_rate_sample \ compile-with "${NORMAL_C} -I$S/dev/ath" # ath DFS modules dev/ath/ath_dfs/null/dfs_null.c optional ath \ compile-with "${NORMAL_C} -I$S/dev/ath" # dev/bce/if_bce.c optional bce dev/bfe/if_bfe.c optional bfe dev/bge/if_bge.c optional bge dev/bktr/bktr_audio.c optional bktr pci dev/bktr/bktr_card.c optional bktr pci dev/bktr/bktr_core.c optional bktr pci dev/bktr/bktr_i2c.c optional bktr pci smbus dev/bktr/bktr_os.c optional bktr pci dev/bktr/bktr_tuner.c optional bktr pci dev/bktr/msp34xx.c optional bktr pci dev/buslogic/bt.c optional bt dev/buslogic/bt_eisa.c optional bt eisa dev/buslogic/bt_isa.c optional bt isa dev/buslogic/bt_mca.c optional bt mca dev/buslogic/bt_pci.c optional bt pci dev/bwi/bwimac.c optional bwi dev/bwi/bwiphy.c optional bwi dev/bwi/bwirf.c optional bwi dev/bwi/if_bwi.c optional bwi dev/bwi/if_bwi_pci.c optional bwi pci # XXX Work around clang warning, until maintainer approves fix. dev/bwn/if_bwn.c optional bwn siba_bwn \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}" dev/cardbus/cardbus.c optional cardbus dev/cardbus/cardbus_cis.c optional cardbus dev/cardbus/cardbus_device.c optional cardbus dev/cas/if_cas.c optional cas dev/cfi/cfi_core.c optional cfi dev/cfi/cfi_dev.c optional cfi dev/cfi/cfi_disk.c optional cfid dev/ciss/ciss.c optional ciss dev/cm/smc90cx6.c optional cm dev/cmx/cmx.c optional cmx dev/cmx/cmx_pccard.c optional cmx pccard dev/cpufreq/ichss.c optional cpufreq dev/cs/if_cs.c optional cs dev/cs/if_cs_isa.c optional cs isa dev/cs/if_cs_pccard.c optional cs pccard dev/cxgb/cxgb_main.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/cxgb_sge.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_mc5.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_vsc7323.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_vsc8211.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_ael1002.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_aq100x.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_mv88e1xxx.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_xgmac.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_t3_hw.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_tn1010.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/sys/uipc_mvec.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/cxgb_t3fw.c optional cxgb cxgb_t3fw \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgbe/t4_main.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_sge.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/t4_l2t.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" dev/cxgbe/common/t4_hw.c optional cxgbe pci \ compile-with "${NORMAL_C} -I$S/dev/cxgbe" t4fw_cfg.c optional cxgbe \ compile-with "${AWK} -f $S/tools/fw_stub.awk t4fw_cfg.fw:t4fw_cfg t4fw_cfg_uwire.fw:t4fw_cfg_uwire t4fw.fw:t4fw -mt4fw_cfg -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "t4fw_cfg.c" t4fw_cfg.fwo optional cxgbe \ dependency "t4fw_cfg.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t4fw_cfg.fwo" t4fw_cfg.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t4fw_cfg.txt" \ compile-with "${CP} ${.ALLSRC} ${.TARGET}" \ no-obj no-implicit-rule \ clean "t4fw_cfg.fw" t4fw_cfg_uwire.fwo optional cxgbe \ dependency "t4fw_cfg_uwire.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t4fw_cfg_uwire.fwo" t4fw_cfg_uwire.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t4fw_cfg_uwire.txt" \ compile-with "${CP} ${.ALLSRC} ${.TARGET}" \ no-obj no-implicit-rule \ clean "t4fw_cfg_uwire.fw" t4fw.fwo optional cxgbe \ dependency "t4fw.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t4fw.fwo" t4fw.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t4fw-1.11.27.0.bin.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "t4fw.fw" t5fw_cfg.c optional cxgbe \ compile-with "${AWK} -f $S/tools/fw_stub.awk t5fw_cfg.fw:t5fw_cfg t5fw.fw:t5fw -mt5fw_cfg -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "t5fw_cfg.c" t5fw_cfg.fwo optional cxgbe \ dependency "t5fw_cfg.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t5fw_cfg.fwo" t5fw_cfg.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t5fw_cfg.txt" \ compile-with "${CP} ${.ALLSRC} ${.TARGET}" \ no-obj no-implicit-rule \ clean "t5fw_cfg.fw" t5fw.fwo optional cxgbe \ dependency "t5fw.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "t5fw.fwo" t5fw.fw optional cxgbe \ dependency "$S/dev/cxgbe/firmware/t5fw-1.11.27.0.bin.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "t5fw.fw" dev/cy/cy.c optional cy dev/cy/cy_isa.c optional cy isa dev/cy/cy_pci.c optional cy pci dev/dc/if_dc.c optional dc pci dev/dc/dcphy.c optional dc pci dev/dc/pnphy.c optional dc pci dev/dcons/dcons.c optional dcons dev/dcons/dcons_crom.c optional dcons_crom dev/dcons/dcons_os.c optional dcons dev/de/if_de.c optional de pci dev/digi/CX.c optional digi_CX dev/digi/CX_PCI.c optional digi_CX_PCI dev/digi/EPCX.c optional digi_EPCX dev/digi/EPCX_PCI.c optional digi_EPCX_PCI dev/digi/Xe.c optional digi_Xe dev/digi/Xem.c optional digi_Xem dev/digi/Xr.c optional digi_Xr dev/digi/digi.c optional digi dev/digi/digi_isa.c optional digi isa dev/digi/digi_pci.c optional digi pci dev/dpt/dpt_eisa.c optional dpt eisa dev/dpt/dpt_pci.c optional dpt pci dev/dpt/dpt_scsi.c optional dpt dev/drm/ati_pcigart.c optional drm dev/drm/drm_agpsupport.c optional drm dev/drm/drm_auth.c optional drm dev/drm/drm_bufs.c optional drm dev/drm/drm_context.c optional drm dev/drm/drm_dma.c optional drm dev/drm/drm_drawable.c optional drm dev/drm/drm_drv.c optional drm dev/drm/drm_fops.c optional drm dev/drm/drm_hashtab.c optional drm dev/drm/drm_ioctl.c optional drm dev/drm/drm_irq.c optional drm dev/drm/drm_lock.c optional drm dev/drm/drm_memory.c optional drm dev/drm/drm_mm.c optional drm dev/drm/drm_pci.c optional drm dev/drm/drm_scatter.c optional drm dev/drm/drm_sman.c optional drm dev/drm/drm_sysctl.c optional drm dev/drm/drm_vm.c optional drm dev/drm/i915_dma.c optional i915drm dev/drm/i915_drv.c optional i915drm dev/drm/i915_irq.c optional i915drm dev/drm/i915_mem.c optional i915drm dev/drm/i915_suspend.c optional i915drm dev/drm/mach64_dma.c optional mach64drm dev/drm/mach64_drv.c optional mach64drm dev/drm/mach64_irq.c optional mach64drm dev/drm/mach64_state.c optional mach64drm dev/drm/mga_dma.c optional mgadrm dev/drm/mga_drv.c optional mgadrm dev/drm/mga_irq.c optional mgadrm dev/drm/mga_state.c optional mgadrm \ compile-with "${NORMAL_C} -finline-limit=13500" dev/drm/mga_warp.c optional mgadrm dev/drm/r128_cce.c optional r128drm \ compile-with "${NORMAL_C} ${NO_WUNUSED_VALUE} ${NO_WCONSTANT_CONVERSION}" dev/drm/r128_drv.c optional r128drm dev/drm/r128_irq.c optional r128drm dev/drm/r128_state.c optional r128drm \ compile-with "${NORMAL_C} ${NO_WUNUSED_VALUE} -finline-limit=13500" dev/drm/r300_cmdbuf.c optional radeondrm dev/drm/r600_blit.c optional radeondrm dev/drm/r600_cp.c optional radeondrm \ compile-with "${NORMAL_C} ${NO_WUNUSED_VALUE} ${NO_WCONSTANT_CONVERSION}" dev/drm/radeon_cp.c optional radeondrm \ compile-with "${NORMAL_C} ${NO_WUNUSED_VALUE} ${NO_WCONSTANT_CONVERSION}" dev/drm/radeon_cs.c optional radeondrm dev/drm/radeon_drv.c optional radeondrm dev/drm/radeon_irq.c optional radeondrm dev/drm/radeon_mem.c optional radeondrm dev/drm/radeon_state.c optional radeondrm dev/drm/savage_bci.c optional savagedrm dev/drm/savage_drv.c optional savagedrm dev/drm/savage_state.c optional savagedrm dev/drm/sis_drv.c optional sisdrm dev/drm/sis_ds.c optional sisdrm dev/drm/sis_mm.c optional sisdrm dev/drm/tdfx_drv.c optional tdfxdrm dev/drm/via_dma.c optional viadrm dev/drm/via_dmablit.c optional viadrm dev/drm/via_drv.c optional viadrm dev/drm/via_irq.c optional viadrm dev/drm/via_map.c optional viadrm dev/drm/via_mm.c optional viadrm dev/drm/via_verifier.c optional viadrm dev/drm/via_video.c optional viadrm dev/ed/if_ed.c optional ed dev/ed/if_ed_novell.c optional ed dev/ed/if_ed_rtl80x9.c optional ed dev/ed/if_ed_pccard.c optional ed pccard dev/ed/if_ed_pci.c optional ed pci dev/eisa/eisa_if.m standard dev/eisa/eisaconf.c optional eisa dev/e1000/if_em.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/if_lem.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/if_igb.c optional igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_80003es2lan.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82540.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82541.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82542.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82543.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82571.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_82575.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_ich8lan.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_i210.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_api.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_mac.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_manage.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_nvm.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_phy.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_vf.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_mbx.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/e1000/e1000_osdep.c optional em | igb \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/et/if_et.c optional et dev/en/if_en_pci.c optional en pci dev/en/midway.c optional en dev/ep/if_ep.c optional ep dev/ep/if_ep_eisa.c optional ep eisa dev/ep/if_ep_isa.c optional ep isa dev/ep/if_ep_mca.c optional ep mca dev/ep/if_ep_pccard.c optional ep pccard dev/esp/esp_pci.c optional esp pci dev/esp/ncr53c9x.c optional esp dev/ex/if_ex.c optional ex dev/ex/if_ex_isa.c optional ex isa dev/ex/if_ex_pccard.c optional ex pccard dev/exca/exca.c optional cbb dev/fatm/if_fatm.c optional fatm pci dev/fb/fbd.c optional fbd | vt dev/fb/fb_if.m standard dev/fb/splash.c optional sc splash dev/fdt/fdt_common.c optional fdt dev/fdt/fdt_pci.c optional fdt pci dev/fdt/fdt_static_dtb.S optional fdt fdt_dtb_static dev/fdt/fdtbus.c optional fdt dev/fdt/simplebus.c optional fdt dev/fe/if_fe.c optional fe dev/fe/if_fe_pccard.c optional fe pccard dev/firewire/firewire.c optional firewire dev/firewire/fwcrom.c optional firewire dev/firewire/fwdev.c optional firewire dev/firewire/fwdma.c optional firewire dev/firewire/fwmem.c optional firewire dev/firewire/fwohci.c optional firewire dev/firewire/fwohci_pci.c optional firewire pci dev/firewire/if_fwe.c optional fwe dev/firewire/if_fwip.c optional fwip dev/firewire/sbp.c optional sbp dev/firewire/sbp_targ.c optional sbp_targ dev/flash/at45d.c optional at45d dev/flash/mx25l.c optional mx25l dev/fxp/if_fxp.c optional fxp dev/fxp/inphy.c optional fxp dev/gem/if_gem.c optional gem dev/gem/if_gem_pci.c optional gem pci dev/gem/if_gem_sbus.c optional gem sbus dev/gpio/gpiobus.c optional gpio \ dependency "gpiobus_if.h" dev/gpio/gpioc.c optional gpio \ dependency "gpio_if.h" dev/gpio/gpioiic.c optional gpioiic dev/gpio/gpioled.c optional gpioled dev/gpio/gpio_if.m optional gpio dev/gpio/gpiobus_if.m optional gpio dev/hatm/if_hatm.c optional hatm pci dev/hatm/if_hatm_intr.c optional hatm pci dev/hatm/if_hatm_ioctl.c optional hatm pci dev/hatm/if_hatm_rx.c optional hatm pci dev/hatm/if_hatm_tx.c optional hatm pci dev/hifn/hifn7751.c optional hifn dev/hme/if_hme.c optional hme dev/hme/if_hme_pci.c optional hme pci dev/hme/if_hme_sbus.c optional hme sbus dev/hptiop/hptiop.c optional hptiop scbus dev/hwpmc/hwpmc_logging.c optional hwpmc dev/hwpmc/hwpmc_mod.c optional hwpmc dev/hwpmc/hwpmc_soft.c optional hwpmc dev/ichsmb/ichsmb.c optional ichsmb dev/ichsmb/ichsmb_pci.c optional ichsmb pci dev/ida/ida.c optional ida dev/ida/ida_disk.c optional ida dev/ida/ida_eisa.c optional ida eisa dev/ida/ida_pci.c optional ida pci dev/ie/if_ie.c optional ie isa nowerror dev/ie/if_ie_isa.c optional ie isa dev/ieee488/ibfoo.c optional pcii | tnt4882 dev/ieee488/pcii.c optional pcii dev/ieee488/tnt4882.c optional tnt4882 dev/ieee488/upd7210.c optional pcii | tnt4882 dev/iicbus/ad7418.c optional ad7418 dev/iicbus/ds133x.c optional ds133x dev/iicbus/ds1672.c optional ds1672 dev/iicbus/icee.c optional icee dev/iicbus/if_ic.c optional ic dev/iicbus/iic.c optional iic dev/iicbus/iicbb.c optional iicbb dev/iicbus/iicbb_if.m optional iicbb dev/iicbus/iicbus.c optional iicbus dev/iicbus/iicbus_if.m optional iicbus dev/iicbus/iiconf.c optional iicbus dev/iicbus/iicsmb.c optional iicsmb \ dependency "iicbus_if.h" dev/iicbus/pcf8563.c optional pcf8563 dev/iir/iir.c optional iir dev/iir/iir_ctrl.c optional iir dev/iir/iir_pci.c optional iir pci # XXX Work around clang warning, until maintainer approves fix. dev/ips/ips.c optional ips \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}" dev/ips/ips_commands.c optional ips dev/ips/ips_disk.c optional ips dev/ips/ips_ioctl.c optional ips dev/ips/ips_pci.c optional ips pci dev/ipw/if_ipw.c optional ipw ipwbssfw.c optional ipwbssfw | ipwfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk ipw_bss.fw:ipw_bss:130 -lintel_ipw -mipw_bss -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "ipwbssfw.c" ipw_bss.fwo optional ipwbssfw | ipwfw \ dependency "ipw_bss.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "ipw_bss.fwo" ipw_bss.fw optional ipwbssfw | ipwfw \ dependency "$S/contrib/dev/ipw/ipw2100-1.3.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "ipw_bss.fw" ipwibssfw.c optional ipwibssfw | ipwfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk ipw_ibss.fw:ipw_ibss:130 -lintel_ipw -mipw_ibss -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "ipwibssfw.c" ipw_ibss.fwo optional ipwibssfw | ipwfw \ dependency "ipw_ibss.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "ipw_ibss.fwo" ipw_ibss.fw optional ipwibssfw | ipwfw \ dependency "$S/contrib/dev/ipw/ipw2100-1.3-i.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "ipw_ibss.fw" ipwmonitorfw.c optional ipwmonitorfw | ipwfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk ipw_monitor.fw:ipw_monitor:130 -lintel_ipw -mipw_monitor -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "ipwmonitorfw.c" ipw_monitor.fwo optional ipwmonitorfw | ipwfw \ dependency "ipw_monitor.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "ipw_monitor.fwo" ipw_monitor.fw optional ipwmonitorfw | ipwfw \ dependency "$S/contrib/dev/ipw/ipw2100-1.3-p.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "ipw_monitor.fw" dev/iscsi/initiator/iscsi.c optional iscsi_initiator scbus dev/iscsi/initiator/iscsi_subr.c optional iscsi_initiator scbus dev/iscsi/initiator/isc_cam.c optional iscsi_initiator scbus dev/iscsi/initiator/isc_soc.c optional iscsi_initiator scbus dev/iscsi/initiator/isc_sm.c optional iscsi_initiator scbus dev/iscsi/initiator/isc_subr.c optional iscsi_initiator scbus dev/isp/isp.c optional isp dev/isp/isp_freebsd.c optional isp dev/isp/isp_library.c optional isp dev/isp/isp_pci.c optional isp pci dev/isp/isp_sbus.c optional isp sbus dev/isp/isp_target.c optional isp dev/ispfw/ispfw.c optional ispfw dev/iwi/if_iwi.c optional iwi iwibssfw.c optional iwibssfw | iwifw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwi_bss.fw:iwi_bss:300 -lintel_iwi -miwi_bss -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwibssfw.c" iwi_bss.fwo optional iwibssfw | iwifw \ dependency "iwi_bss.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwi_bss.fwo" iwi_bss.fw optional iwibssfw | iwifw \ dependency "$S/contrib/dev/iwi/ipw2200-bss.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwi_bss.fw" iwiibssfw.c optional iwiibssfw | iwifw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwi_ibss.fw:iwi_ibss:300 -lintel_iwi -miwi_ibss -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwiibssfw.c" iwi_ibss.fwo optional iwiibssfw | iwifw \ dependency "iwi_ibss.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwi_ibss.fwo" iwi_ibss.fw optional iwiibssfw | iwifw \ dependency "$S/contrib/dev/iwi/ipw2200-ibss.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwi_ibss.fw" iwimonitorfw.c optional iwimonitorfw | iwifw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwi_monitor.fw:iwi_monitor:300 -lintel_iwi -miwi_monitor -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwimonitorfw.c" iwi_monitor.fwo optional iwimonitorfw | iwifw \ dependency "iwi_monitor.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwi_monitor.fwo" iwi_monitor.fw optional iwimonitorfw | iwifw \ dependency "$S/contrib/dev/iwi/ipw2200-sniffer.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwi_monitor.fw" dev/iwn/if_iwn.c optional iwn iwn1000fw.c optional iwn1000fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn1000.fw:iwn1000fw -miwn1000fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn1000fw.c" iwn1000fw.fwo optional iwn1000fw | iwnfw \ dependency "iwn1000.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn1000fw.fwo" iwn1000.fw optional iwn1000fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-1000-39.31.5.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn1000.fw" iwn4965fw.c optional iwn4965fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn4965.fw:iwn4965fw -miwn4965fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn4965fw.c" iwn4965fw.fwo optional iwn4965fw | iwnfw \ dependency "iwn4965.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn4965fw.fwo" iwn4965.fw optional iwn4965fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-4965-228.61.2.24.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn4965.fw" iwn5000fw.c optional iwn5000fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn5000.fw:iwn5000fw -miwn5000fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn5000fw.c" iwn5000fw.fwo optional iwn5000fw | iwnfw \ dependency "iwn5000.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn5000fw.fwo" iwn5000.fw optional iwn5000fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-5000-8.83.5.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn5000.fw" iwn5150fw.c optional iwn5150fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn5150.fw:iwn5150fw -miwn5150fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn5150fw.c" iwn5150fw.fwo optional iwn5150fw | iwnfw \ dependency "iwn5150.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn5150fw.fwo" iwn5150.fw optional iwn5150fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-5150-8.24.2.2.fw.uu"\ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn5150.fw" iwn6000fw.c optional iwn6000fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6000.fw:iwn6000fw -miwn6000fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn6000fw.c" iwn6000fw.fwo optional iwn6000fw | iwnfw \ dependency "iwn6000.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn6000fw.fwo" iwn6000.fw optional iwn6000fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-6000-9.221.4.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn6000.fw" iwn6000g2afw.c optional iwn6000g2afw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6000g2a.fw:iwn6000g2afw -miwn6000g2afw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn6000g2afw.c" iwn6000g2afw.fwo optional iwn6000g2afw | iwnfw \ dependency "iwn6000g2a.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn6000g2afw.fwo" iwn6000g2a.fw optional iwn6000g2afw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-6000g2a-17.168.5.2.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn6000g2a.fw" iwn6000g2bfw.c optional iwn6000g2bfw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6000g2b.fw:iwn6000g2bfw -miwn6000g2bfw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn6000g2bfw.c" iwn6000g2bfw.fwo optional iwn6000g2bfw | iwnfw \ dependency "iwn6000g2b.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn6000g2bfw.fwo" iwn6000g2b.fw optional iwn6000g2bfw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-6000g2b-17.168.5.2.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn6000g2b.fw" iwn6050fw.c optional iwn6050fw | iwnfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6050.fw:iwn6050fw -miwn6050fw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "iwn6050fw.c" iwn6050fw.fwo optional iwn6050fw | iwnfw \ dependency "iwn6050.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "iwn6050fw.fwo" iwn6050.fw optional iwn6050fw | iwnfw \ dependency "$S/contrib/dev/iwn/iwlwifi-6050-41.28.5.1.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "iwn6050.fw" dev/ixgb/if_ixgb.c optional ixgb dev/ixgb/ixgb_ee.c optional ixgb dev/ixgb/ixgb_hw.c optional ixgb dev/ixgbe/ixgbe.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixv.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_phy.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_api.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_common.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_mbx.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_vf.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_82598.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_82599.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_x540.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_dcb.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_dcb_82598.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_dcb_82599.c optional ixgbe inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/jme/if_jme.c optional jme pci dev/joy/joy.c optional joy dev/joy/joy_isa.c optional joy isa dev/joy/joy_pccard.c optional joy pccard dev/kbdmux/kbdmux.c optional kbdmux dev/ksyms/ksyms.c optional ksyms dev/le/am7990.c optional le dev/le/am79900.c optional le dev/le/if_le_pci.c optional le pci dev/le/lance.c optional le dev/led/led.c standard dev/lge/if_lge.c optional lge dev/lmc/if_lmc.c optional lmc dev/malo/if_malo.c optional malo dev/malo/if_malohal.c optional malo dev/malo/if_malo_pci.c optional malo pci dev/mc146818/mc146818.c optional mc146818 dev/mca/mca_bus.c optional mca dev/mcd/mcd.c optional mcd isa nowerror dev/mcd/mcd_isa.c optional mcd isa nowerror dev/md/md.c optional md dev/mem/memdev.c optional mem dev/mem/memutil.c optional mem dev/mfi/mfi.c optional mfi dev/mfi/mfi_debug.c optional mfi dev/mfi/mfi_pci.c optional mfi pci dev/mfi/mfi_disk.c optional mfi dev/mfi/mfi_syspd.c optional mfi dev/mfi/mfi_tbolt.c optional mfi dev/mfi/mfi_linux.c optional mfi compat_linux dev/mfi/mfi_cam.c optional mfip scbus dev/mii/acphy.c optional miibus | acphy dev/mii/amphy.c optional miibus | amphy dev/mii/atphy.c optional miibus | atphy dev/mii/axphy.c optional miibus | axphy dev/mii/bmtphy.c optional miibus | bmtphy dev/mii/brgphy.c optional miibus | brgphy dev/mii/ciphy.c optional miibus | ciphy dev/mii/e1000phy.c optional miibus | e1000phy dev/mii/gentbi.c optional miibus | gentbi dev/mii/icsphy.c optional miibus | icsphy dev/mii/ip1000phy.c optional miibus | ip1000phy dev/mii/jmphy.c optional miibus | jmphy dev/mii/lxtphy.c optional miibus | lxtphy dev/mii/mii.c optional miibus | mii dev/mii/mii_bitbang.c optional miibus | mii_bitbang dev/mii/mii_physubr.c optional miibus | mii dev/mii/miibus_if.m optional miibus | mii dev/mii/mlphy.c optional miibus | mlphy dev/mii/nsgphy.c optional miibus | nsgphy dev/mii/nsphy.c optional miibus | nsphy dev/mii/nsphyter.c optional miibus | nsphyter dev/mii/pnaphy.c optional miibus | pnaphy dev/mii/qsphy.c optional miibus | qsphy dev/mii/rdcphy.c optional miibus | rdcphy dev/mii/rgephy.c optional miibus | rgephy dev/mii/rlphy.c optional miibus | rlphy dev/mii/rlswitch.c optional rlswitch dev/mii/smcphy.c optional miibus | smcphy dev/mii/tdkphy.c optional miibus | tdkphy dev/mii/tlphy.c optional miibus | tlphy dev/mii/truephy.c optional miibus | truephy dev/mii/ukphy.c optional miibus | mii dev/mii/ukphy_subr.c optional miibus | mii dev/mii/xmphy.c optional miibus | xmphy dev/mk48txx/mk48txx.c optional mk48txx dev/mlx/mlx.c optional mlx dev/mlx/mlx_disk.c optional mlx dev/mlx/mlx_pci.c optional mlx pci dev/mly/mly.c optional mly dev/mmc/mmc.c optional mmc dev/mmc/mmcbr_if.m standard dev/mmc/mmcbus_if.m standard dev/mmc/mmcsd.c optional mmcsd dev/mn/if_mn.c optional mn pci dev/mpr/mpr.c optional mpr dev/mpr/mpr_config.c optional mpr # XXX Work around clang warning, until maintainer approves fix. dev/mpr/mpr_mapping.c optional mpr \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}" dev/mpr/mpr_pci.c optional mpr pci dev/mpr/mpr_sas.c optional mpr \ compile-with "${NORMAL_C} ${NO_WUNNEEDED_INTERNAL_DECL}" dev/mpr/mpr_sas_lsi.c optional mpr dev/mpr/mpr_table.c optional mpr dev/mpr/mpr_user.c optional mpr dev/mps/mps.c optional mps dev/mps/mps_config.c optional mps # XXX Work around clang warning, until maintainer approves fix. dev/mps/mps_mapping.c optional mps \ compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}" dev/mps/mps_pci.c optional mps pci dev/mps/mps_sas.c optional mps \ compile-with "${NORMAL_C} ${NO_WUNNEEDED_INTERNAL_DECL}" dev/mps/mps_sas_lsi.c optional mps dev/mps/mps_table.c optional mps dev/mps/mps_user.c optional mps dev/mpt/mpt.c optional mpt dev/mpt/mpt_cam.c optional mpt dev/mpt/mpt_debug.c optional mpt dev/mpt/mpt_pci.c optional mpt pci dev/mpt/mpt_raid.c optional mpt dev/mpt/mpt_user.c optional mpt dev/mrsas/mrsas.c optional mrsas dev/mrsas/mrsas_cam.c optional mrsas dev/mrsas/mrsas_ioctl.c optional mrsas dev/mrsas/mrsas_fp.c optional mrsas dev/msk/if_msk.c optional msk dev/mvs/mvs.c optional mvs dev/mvs/mvs_if.m optional mvs dev/mvs/mvs_pci.c optional mvs pci dev/mwl/if_mwl.c optional mwl dev/mwl/if_mwl_pci.c optional mwl pci dev/mwl/mwlhal.c optional mwl mwlfw.c optional mwlfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk mw88W8363.fw:mw88W8363fw mwlboot.fw:mwlboot -mmwl -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "mwlfw.c" mw88W8363.fwo optional mwlfw \ dependency "mw88W8363.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "mw88W8363.fwo" mw88W8363.fw optional mwlfw \ dependency "$S/contrib/dev/mwl/mw88W8363.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "mw88W8363.fw" mwlboot.fwo optional mwlfw \ dependency "mwlboot.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "mwlboot.fwo" mwlboot.fw optional mwlfw \ dependency "$S/contrib/dev/mwl/mwlboot.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "mwlboot.fw" dev/mxge/if_mxge.c optional mxge pci dev/mxge/mxge_eth_z8e.c optional mxge pci dev/mxge/mxge_ethp_z8e.c optional mxge pci dev/mxge/mxge_rss_eth_z8e.c optional mxge pci dev/mxge/mxge_rss_ethp_z8e.c optional mxge pci dev/my/if_my.c optional my dev/ncv/ncr53c500.c optional ncv dev/ncv/ncr53c500_pccard.c optional ncv pccard dev/netmap/netmap.c optional netmap dev/netmap/netmap_freebsd.c optional netmap dev/netmap/netmap_generic.c optional netmap dev/netmap/netmap_mbq.c optional netmap dev/netmap/netmap_mem2.c optional netmap dev/netmap/netmap_offloadings.c optional netmap dev/netmap/netmap_pipe.c optional netmap dev/netmap/netmap_vale.c optional netmap dev/nge/if_nge.c optional nge dev/nxge/if_nxge.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-device.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-mm.c optional nxge dev/nxge/xgehal/xge-queue.c optional nxge dev/nxge/xgehal/xgehal-driver.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-ring.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-channel.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-fifo.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-stats.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nxge/xgehal/xgehal-config.c optional nxge dev/nxge/xgehal/xgehal-mgmt.c optional nxge \ compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN}" dev/nmdm/nmdm.c optional nmdm dev/nsp/nsp.c optional nsp dev/nsp/nsp_pccard.c optional nsp pccard dev/null/null.c standard dev/oce/oce_hw.c optional oce pci dev/oce/oce_if.c optional oce pci dev/oce/oce_mbox.c optional oce pci dev/oce/oce_queue.c optional oce pci dev/oce/oce_sysctl.c optional oce pci dev/oce/oce_util.c optional oce pci dev/patm/if_patm.c optional patm pci dev/patm/if_patm_attach.c optional patm pci dev/patm/if_patm_intr.c optional patm pci dev/patm/if_patm_ioctl.c optional patm pci dev/patm/if_patm_rtables.c optional patm pci dev/patm/if_patm_rx.c optional patm pci dev/patm/if_patm_tx.c optional patm pci dev/pbio/pbio.c optional pbio isa dev/pccard/card_if.m standard dev/pccard/pccard.c optional pccard dev/pccard/pccard_cis.c optional pccard dev/pccard/pccard_cis_quirks.c optional pccard dev/pccard/pccard_device.c optional pccard dev/pccard/power_if.m standard dev/pccbb/pccbb.c optional cbb dev/pccbb/pccbb_isa.c optional cbb isa dev/pccbb/pccbb_pci.c optional cbb pci dev/pcf/pcf.c optional pcf dev/pci/eisa_pci.c optional pci eisa dev/pci/fixup_pci.c optional pci dev/pci/hostb_pci.c optional pci dev/pci/ignore_pci.c optional pci dev/pci/isa_pci.c optional pci isa dev/pci/pci.c optional pci dev/pci/pci_if.m standard dev/pci/pci_pci.c optional pci dev/pci/pci_subr.c optional pci dev/pci/pci_user.c optional pci dev/pci/pcib_if.m standard dev/pci/vga_pci.c optional pci dev/pcn/if_pcn.c optional pcn pci dev/pdq/if_fea.c optional fea eisa dev/pdq/if_fpa.c optional fpa pci dev/pdq/pdq.c optional nowerror fea eisa | fpa pci dev/pdq/pdq_ifsubr.c optional nowerror fea eisa | fpa pci dev/ppbus/if_plip.c optional plip dev/ppbus/immio.c optional vpo dev/ppbus/lpbb.c optional lpbb dev/ppbus/lpt.c optional lpt dev/ppbus/pcfclock.c optional pcfclock dev/ppbus/ppb_1284.c optional ppbus dev/ppbus/ppb_base.c optional ppbus dev/ppbus/ppb_msq.c optional ppbus dev/ppbus/ppbconf.c optional ppbus dev/ppbus/ppbus_if.m optional ppbus dev/ppbus/ppi.c optional ppi dev/ppbus/pps.c optional pps dev/ppbus/vpo.c optional vpo dev/ppbus/vpoio.c optional vpo dev/ppc/ppc.c optional ppc dev/ppc/ppc_acpi.c optional ppc acpi dev/ppc/ppc_isa.c optional ppc isa dev/ppc/ppc_pci.c optional ppc pci dev/ppc/ppc_puc.c optional ppc puc dev/pst/pst-iop.c optional pst dev/pst/pst-pci.c optional pst pci dev/pst/pst-raid.c optional pst dev/pty/pty.c optional pty dev/puc/puc.c optional puc dev/puc/puc_cfg.c optional puc dev/puc/puc_pccard.c optional puc pccard dev/puc/puc_pci.c optional puc pci dev/puc/pucdata.c optional puc pci dev/quicc/quicc_core.c optional quicc dev/ral/rt2560.c optional ral dev/ral/rt2661.c optional ral dev/ral/rt2860.c optional ral dev/ral/if_ral_pci.c optional ral pci rt2561fw.c optional rt2561fw | ralfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rt2561.fw:rt2561fw -mrt2561 -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rt2561fw.c" rt2561fw.fwo optional rt2561fw | ralfw \ dependency "rt2561.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rt2561fw.fwo" rt2561.fw optional rt2561fw | ralfw \ dependency "$S/contrib/dev/ral/rt2561.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rt2561.fw" rt2561sfw.c optional rt2561sfw | ralfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rt2561s.fw:rt2561sfw -mrt2561s -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rt2561sfw.c" rt2561sfw.fwo optional rt2561sfw | ralfw \ dependency "rt2561s.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rt2561sfw.fwo" rt2561s.fw optional rt2561sfw | ralfw \ dependency "$S/contrib/dev/ral/rt2561s.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rt2561s.fw" rt2661fw.c optional rt2661fw | ralfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rt2661.fw:rt2661fw -mrt2661 -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rt2661fw.c" rt2661fw.fwo optional rt2661fw | ralfw \ dependency "rt2661.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rt2661fw.fwo" rt2661.fw optional rt2661fw | ralfw \ dependency "$S/contrib/dev/ral/rt2661.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rt2661.fw" rt2860fw.c optional rt2860fw | ralfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk rt2860.fw:rt2860fw -mrt2860 -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "rt2860fw.c" rt2860fw.fwo optional rt2860fw | ralfw \ dependency "rt2860.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "rt2860fw.fwo" rt2860.fw optional rt2860fw | ralfw \ dependency "$S/contrib/dev/ral/rt2860.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "rt2860.fw" dev/random/harvest.c standard dev/random/hash.c optional random dev/random/probe.c optional random dev/random/randomdev.c optional random dev/random/randomdev_soft.c optional random dev/random/yarrow.c optional random dev/rc/rc.c optional rc dev/re/if_re.c optional re dev/rndtest/rndtest.c optional rndtest dev/rp/rp.c optional rp dev/rp/rp_isa.c optional rp isa dev/rp/rp_pci.c optional rp pci dev/safe/safe.c optional safe dev/scc/scc_if.m optional scc dev/scc/scc_bfe_ebus.c optional scc ebus dev/scc/scc_bfe_quicc.c optional scc quicc dev/scc/scc_bfe_sbus.c optional scc fhc | scc sbus dev/scc/scc_core.c optional scc dev/scc/scc_dev_quicc.c optional scc quicc dev/scc/scc_dev_sab82532.c optional scc dev/scc/scc_dev_z8530.c optional scc dev/scd/scd.c optional scd isa dev/scd/scd_isa.c optional scd isa dev/sdhci/sdhci.c optional sdhci pci dev/sf/if_sf.c optional sf pci dev/sge/if_sge.c optional sge pci dev/si/si.c optional si dev/si/si2_z280.c optional si dev/si/si3_t225.c optional si dev/si/si_eisa.c optional si eisa dev/si/si_isa.c optional si isa dev/si/si_pci.c optional si pci dev/siba/siba_bwn.c optional siba_bwn pci dev/siba/siba_core.c optional siba_bwn pci dev/siis/siis.c optional siis pci dev/sis/if_sis.c optional sis pci dev/sk/if_sk.c optional sk pci dev/smbus/smb.c optional smb dev/smbus/smbconf.c optional smbus dev/smbus/smbus.c optional smbus dev/smbus/smbus_if.m optional smbus dev/smc/if_smc.c optional smc dev/sn/if_sn.c optional sn dev/sn/if_sn_isa.c optional sn isa dev/sn/if_sn_pccard.c optional sn pccard dev/snp/snp.c optional snp dev/sound/clone.c optional sound dev/sound/unit.c optional sound dev/sound/isa/ad1816.c optional snd_ad1816 isa dev/sound/isa/ess.c optional snd_ess isa dev/sound/isa/gusc.c optional snd_gusc isa dev/sound/isa/mss.c optional snd_mss isa dev/sound/isa/sb16.c optional snd_sb16 isa dev/sound/isa/sb8.c optional snd_sb8 isa dev/sound/isa/sbc.c optional snd_sbc isa dev/sound/isa/sndbuf_dma.c optional sound isa dev/sound/pci/als4000.c optional snd_als4000 pci dev/sound/pci/atiixp.c optional snd_atiixp pci dev/sound/pci/cmi.c optional snd_cmi pci dev/sound/pci/cs4281.c optional snd_cs4281 pci dev/sound/pci/csa.c optional snd_csa pci dev/sound/pci/csapcm.c optional snd_csa pci dev/sound/pci/ds1.c optional snd_ds1 pci dev/sound/pci/emu10k1.c optional snd_emu10k1 pci dev/sound/pci/emu10kx.c optional snd_emu10kx pci dev/sound/pci/emu10kx-pcm.c optional snd_emu10kx pci dev/sound/pci/emu10kx-midi.c optional snd_emu10kx pci dev/sound/pci/envy24.c optional snd_envy24 pci dev/sound/pci/envy24ht.c optional snd_envy24ht pci dev/sound/pci/es137x.c optional snd_es137x pci dev/sound/pci/fm801.c optional snd_fm801 pci dev/sound/pci/ich.c optional snd_ich pci dev/sound/pci/maestro.c optional snd_maestro pci dev/sound/pci/maestro3.c optional snd_maestro3 pci dev/sound/pci/neomagic.c optional snd_neomagic pci dev/sound/pci/solo.c optional snd_solo pci dev/sound/pci/spicds.c optional snd_spicds pci dev/sound/pci/t4dwave.c optional snd_t4dwave pci dev/sound/pci/via8233.c optional snd_via8233 pci dev/sound/pci/via82c686.c optional snd_via82c686 pci dev/sound/pci/vibes.c optional snd_vibes pci dev/sound/pci/hda/hdaa.c optional snd_hda pci dev/sound/pci/hda/hdaa_patches.c optional snd_hda pci dev/sound/pci/hda/hdac.c optional snd_hda pci dev/sound/pci/hda/hdac_if.m optional snd_hda pci dev/sound/pci/hda/hdacc.c optional snd_hda pci dev/sound/pci/hdspe.c optional snd_hdspe pci dev/sound/pci/hdspe-pcm.c optional snd_hdspe pci dev/sound/pcm/ac97.c optional sound dev/sound/pcm/ac97_if.m optional sound dev/sound/pcm/ac97_patch.c optional sound dev/sound/pcm/buffer.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/channel.c optional sound dev/sound/pcm/channel_if.m optional sound dev/sound/pcm/dsp.c optional sound dev/sound/pcm/feeder.c optional sound dev/sound/pcm/feeder_chain.c optional sound dev/sound/pcm/feeder_eq.c optional sound \ dependency "feeder_eq_gen.h" \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_if.m optional sound dev/sound/pcm/feeder_format.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_matrix.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_mixer.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_rate.c optional sound \ dependency "feeder_rate_gen.h" \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/feeder_volume.c optional sound \ dependency "snd_fxdiv_gen.h" dev/sound/pcm/mixer.c optional sound dev/sound/pcm/mixer_if.m optional sound dev/sound/pcm/sndstat.c optional sound dev/sound/pcm/sound.c optional sound dev/sound/pcm/vchan.c optional sound dev/sound/usb/uaudio.c optional snd_uaudio usb dev/sound/usb/uaudio_pcm.c optional snd_uaudio usb dev/sound/midi/midi.c optional sound dev/sound/midi/mpu401.c optional sound dev/sound/midi/mpu_if.m optional sound dev/sound/midi/mpufoi_if.m optional sound dev/sound/midi/sequencer.c optional sound dev/sound/midi/synth_if.m optional sound dev/spibus/spibus.c optional spibus \ dependency "spibus_if.h" dev/spibus/spibus_if.m optional spibus dev/ste/if_ste.c optional ste pci dev/stg/tmc18c30.c optional stg dev/stg/tmc18c30_isa.c optional stg isa dev/stg/tmc18c30_pccard.c optional stg pccard dev/stg/tmc18c30_pci.c optional stg pci dev/stg/tmc18c30_subr.c optional stg dev/stge/if_stge.c optional stge dev/streams/streams.c optional streams dev/sym/sym_hipd.c optional sym \ dependency "$S/dev/sym/sym_{conf,defs}.h" dev/syscons/blank/blank_saver.c optional blank_saver dev/syscons/daemon/daemon_saver.c optional daemon_saver dev/syscons/dragon/dragon_saver.c optional dragon_saver dev/syscons/fade/fade_saver.c optional fade_saver dev/syscons/fire/fire_saver.c optional fire_saver dev/syscons/green/green_saver.c optional green_saver dev/syscons/logo/logo.c optional logo_saver dev/syscons/logo/logo_saver.c optional logo_saver dev/syscons/rain/rain_saver.c optional rain_saver dev/syscons/schistory.c optional sc dev/syscons/scmouse.c optional sc dev/syscons/scterm.c optional sc dev/syscons/scvidctl.c optional sc dev/syscons/snake/snake_saver.c optional snake_saver dev/syscons/star/star_saver.c optional star_saver dev/syscons/syscons.c optional sc dev/syscons/sysmouse.c optional sc dev/syscons/warp/warp_saver.c optional warp_saver dev/tdfx/tdfx_linux.c optional tdfx_linux tdfx compat_linux dev/tdfx/tdfx_pci.c optional tdfx pci dev/ti/if_ti.c optional ti pci dev/tl/if_tl.c optional tl pci dev/trm/trm.c optional trm dev/twa/tw_cl_init.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_cl_intr.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_cl_io.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_cl_misc.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_osl_cam.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twa/tw_osl_freebsd.c optional twa \ compile-with "${NORMAL_C} -I$S/dev/twa" dev/twe/twe.c optional twe dev/twe/twe_freebsd.c optional twe dev/tws/tws.c optional tws dev/tws/tws_cam.c optional tws dev/tws/tws_hdm.c optional tws dev/tws/tws_services.c optional tws dev/tws/tws_user.c optional tws dev/tx/if_tx.c optional tx dev/txp/if_txp.c optional txp dev/uart/uart_bus_acpi.c optional uart acpi #dev/uart/uart_bus_cbus.c optional uart cbus dev/uart/uart_bus_ebus.c optional uart ebus dev/uart/uart_bus_fdt.c optional uart fdt dev/uart/uart_bus_isa.c optional uart isa dev/uart/uart_bus_pccard.c optional uart pccard dev/uart/uart_bus_pci.c optional uart pci dev/uart/uart_bus_puc.c optional uart puc dev/uart/uart_bus_scc.c optional uart scc dev/uart/uart_core.c optional uart dev/uart/uart_dbg.c optional uart gdb dev/uart/uart_dev_ns8250.c optional uart uart_ns8250 dev/uart/uart_dev_quicc.c optional uart quicc dev/uart/uart_dev_sab82532.c optional uart uart_sab82532 dev/uart/uart_dev_sab82532.c optional uart scc dev/uart/uart_dev_z8530.c optional uart uart_z8530 dev/uart/uart_dev_z8530.c optional uart scc dev/uart/uart_if.m optional uart dev/uart/uart_subr.c optional uart dev/uart/uart_tty.c optional uart dev/ubsec/ubsec.c optional ubsec # # USB controller drivers # dev/usb/controller/at91dci.c optional at91dci dev/usb/controller/at91dci_atmelarm.c optional at91dci at91rm9200 dev/usb/controller/musb_otg.c optional musb dev/usb/controller/musb_otg_atmelarm.c optional musb at91rm9200 dev/usb/controller/ehci.c optional ehci dev/usb/controller/ehci_pci.c optional ehci pci dev/usb/controller/ohci.c optional ohci dev/usb/controller/ohci_atmelarm.c optional ohci at91rm9200 dev/usb/controller/ohci_pci.c optional ohci pci dev/usb/controller/uhci.c optional uhci dev/usb/controller/uhci_pci.c optional uhci pci dev/usb/controller/xhci.c optional xhci dev/usb/controller/xhci_pci.c optional xhci pci dev/usb/controller/uss820dci.c optional uss820dci dev/usb/controller/uss820dci_atmelarm.c optional uss820dci at91rm9200 dev/usb/controller/usb_controller.c optional usb # # USB storage drivers # dev/usb/storage/umass.c optional umass dev/usb/storage/urio.c optional urio dev/usb/storage/ustorage_fs.c optional usfs # # USB core # dev/usb/usb_busdma.c optional usb dev/usb/usb_compat_linux.c optional usb dev/usb/usb_core.c optional usb dev/usb/usb_debug.c optional usb dev/usb/usb_dev.c optional usb dev/usb/usb_device.c optional usb dev/usb/usb_dynamic.c optional usb dev/usb/usb_error.c optional usb dev/usb/usb_generic.c optional usb dev/usb/usb_handle_request.c optional usb dev/usb/usb_hid.c optional usb dev/usb/usb_hub.c optional usb dev/usb/usb_if.m optional usb dev/usb/usb_lookup.c optional usb dev/usb/usb_mbuf.c optional usb dev/usb/usb_msctest.c optional usb dev/usb/usb_parse.c optional usb dev/usb/usb_pf.c optional usb dev/usb/usb_process.c optional usb dev/usb/usb_request.c optional usb dev/usb/usb_transfer.c optional usb dev/usb/usb_util.c optional usb # # USB network drivers # dev/usb/net/if_aue.c optional aue dev/usb/net/if_axe.c optional axe dev/usb/net/if_axge.c optional axge dev/usb/net/if_cdce.c optional cdce dev/usb/net/if_cue.c optional cue dev/usb/net/if_ipheth.c optional ipheth dev/usb/net/if_kue.c optional kue dev/usb/net/if_mos.c optional mos dev/usb/net/if_rue.c optional rue dev/usb/net/if_udav.c optional udav dev/usb/net/if_usie.c optional usie dev/usb/net/if_urndis.c optional urndis dev/usb/net/ruephy.c optional rue dev/usb/net/usb_ethernet.c optional aue | axe | axge | cdce | cue | kue | \ mos | rue | udav | ipheth | urndis dev/usb/net/uhso.c optional uhso # # USB WLAN drivers # dev/usb/wlan/if_rum.c optional rum dev/usb/wlan/if_run.c optional run runfw.c optional runfw \ compile-with "${AWK} -f $S/tools/fw_stub.awk run.fw:runfw -mrunfw -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "runfw.c" runfw.fwo optional runfw \ dependency "run.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "runfw.fwo" run.fw optional runfw \ dependency "$S/contrib/dev/run/rt2870.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "run.fw" dev/usb/wlan/if_uath.c optional uath dev/usb/wlan/if_upgt.c optional upgt dev/usb/wlan/if_ural.c optional ural dev/usb/wlan/if_urtw.c optional urtw dev/usb/wlan/if_zyd.c optional zyd # # USB serial and parallel port drivers # dev/usb/serial/u3g.c optional u3g dev/usb/serial/uark.c optional uark dev/usb/serial/ubsa.c optional ubsa dev/usb/serial/ubser.c optional ubser dev/usb/serial/uchcom.c optional uchcom dev/usb/serial/ucycom.c optional ucycom dev/usb/serial/ufoma.c optional ufoma dev/usb/serial/uftdi.c optional uftdi dev/usb/serial/ugensa.c optional ugensa dev/usb/serial/uipaq.c optional uipaq dev/usb/serial/ulpt.c optional ulpt dev/usb/serial/umcs.c optional umcs dev/usb/serial/umct.c optional umct dev/usb/serial/umodem.c optional umodem dev/usb/serial/umoscom.c optional umoscom dev/usb/serial/uplcom.c optional uplcom dev/usb/serial/uslcom.c optional uslcom dev/usb/serial/uvisor.c optional uvisor dev/usb/serial/uvscom.c optional uvscom dev/usb/serial/usb_serial.c optional ucom | u3g | uark | ubsa | ubser | \ uchcom | ucycom | ufoma | uftdi | \ ugensa | uipaq | umcs | umct | \ umodem | umoscom | uplcom | usie | \ uslcom | uvisor | uvscom # # USB misc drivers # dev/usb/misc/ufm.c optional ufm dev/usb/misc/udbp.c optional udbp # # USB input drivers # dev/usb/input/atp.c optional atp dev/usb/input/uep.c optional uep dev/usb/input/uhid.c optional uhid dev/usb/input/ukbd.c optional ukbd dev/usb/input/ums.c optional ums dev/usb/input/wsp.c optional wsp # # USB quirks # dev/usb/quirk/usb_quirk.c optional usb # # USB templates # dev/usb/template/usb_template.c optional usb_template dev/usb/template/usb_template_audio.c optional usb_template dev/usb/template/usb_template_cdce.c optional usb_template dev/usb/template/usb_template_kbd.c optional usb_template dev/usb/template/usb_template_modem.c optional usb_template dev/usb/template/usb_template_mouse.c optional usb_template dev/usb/template/usb_template_msc.c optional usb_template dev/usb/template/usb_template_mtp.c optional usb_template # # USB END # dev/utopia/idtphy.c optional utopia dev/utopia/suni.c optional utopia dev/utopia/utopia.c optional utopia dev/vge/if_vge.c optional vge dev/vkbd/vkbd.c optional vkbd dev/vr/if_vr.c optional vr pci dev/vt/colors/vt_termcolors.c optional vt dev/vt/font/vt_font_default.c optional vt dev/vt/font/vt_mouse_cursor.c optional vt dev/vt/hw/fb/vt_fb.c optional vt dev/vt/hw/vga/vga.c optional vt vt_vga dev/vt/logo/logo_freebsd.c optional vt splash dev/vt/vt_buf.c optional vt dev/vt/vt_consolectl.c optional vt dev/vt/vt_core.c optional vt dev/vt/vt_font.c optional vt dev/vt/vt_sysmouse.c optional vt dev/vte/if_vte.c optional vte pci dev/vx/if_vx.c optional vx dev/vx/if_vx_eisa.c optional vx eisa dev/vx/if_vx_pci.c optional vx pci dev/vxge/vxge.c optional vxge dev/vxge/vxgehal/vxgehal-ifmsg.c optional vxge dev/vxge/vxgehal/vxgehal-mrpcim.c optional vxge dev/vxge/vxgehal/vxge-queue.c optional vxge dev/vxge/vxgehal/vxgehal-ring.c optional vxge dev/vxge/vxgehal/vxgehal-swapper.c optional vxge dev/vxge/vxgehal/vxgehal-mgmt.c optional vxge dev/vxge/vxgehal/vxgehal-srpcim.c optional vxge dev/vxge/vxgehal/vxgehal-config.c optional vxge dev/vxge/vxgehal/vxgehal-blockpool.c optional vxge dev/vxge/vxgehal/vxgehal-doorbells.c optional vxge dev/vxge/vxgehal/vxgehal-mgmtaux.c optional vxge dev/vxge/vxgehal/vxgehal-device.c optional vxge dev/vxge/vxgehal/vxgehal-mm.c optional vxge dev/vxge/vxgehal/vxgehal-driver.c optional vxge dev/vxge/vxgehal/vxgehal-virtualpath.c optional vxge dev/vxge/vxgehal/vxgehal-channel.c optional vxge dev/vxge/vxgehal/vxgehal-fifo.c optional vxge dev/watchdog/watchdog.c standard dev/wb/if_wb.c optional wb pci dev/wds/wd7000.c optional wds isa dev/wi/if_wi.c optional wi dev/wi/if_wi_pccard.c optional wi pccard dev/wi/if_wi_pci.c optional wi pci dev/wl/if_wl.c optional wl isa dev/wpi/if_wpi.c optional wpi pci wpifw.c optional wpifw \ compile-with "${AWK} -f $S/tools/fw_stub.awk wpi.fw:wpifw:153229 -mwpi -c${.TARGET}" \ no-implicit-rule before-depend local \ clean "wpifw.c" wpifw.fwo optional wpifw \ dependency "wpi.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "wpifw.fwo" wpi.fw optional wpifw \ dependency "$S/contrib/dev/wpi/iwlwifi-3945-15.32.2.9.fw.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "wpi.fw" dev/xe/if_xe.c optional xe dev/xe/if_xe_pccard.c optional xe pccard dev/xl/if_xl.c optional xl pci dev/xl/xlphy.c optional xl pci fs/coda/coda_fbsd.c optional vcoda fs/coda/coda_psdev.c optional vcoda fs/coda/coda_subr.c optional vcoda fs/coda/coda_venus.c optional vcoda fs/coda/coda_vfsops.c optional vcoda fs/coda/coda_vnops.c optional vcoda fs/deadfs/dead_vnops.c standard fs/devfs/devfs_devs.c standard fs/devfs/devfs_dir.c standard fs/devfs/devfs_rule.c standard fs/devfs/devfs_vfsops.c standard fs/devfs/devfs_vnops.c standard fs/fdescfs/fdesc_vfsops.c optional fdescfs fs/fdescfs/fdesc_vnops.c optional fdescfs fs/fifofs/fifo_vnops.c standard fs/hpfs/hpfs_alsubr.c optional hpfs fs/hpfs/hpfs_lookup.c optional hpfs fs/hpfs/hpfs_subr.c optional hpfs fs/hpfs/hpfs_vfsops.c optional hpfs fs/hpfs/hpfs_vnops.c optional hpfs fs/msdosfs/msdosfs_conv.c optional msdosfs fs/msdosfs/msdosfs_denode.c optional msdosfs fs/msdosfs/msdosfs_fat.c optional msdosfs fs/msdosfs/msdosfs_fileno.c optional msdosfs fs/msdosfs/msdosfs_iconv.c optional msdosfs_iconv fs/msdosfs/msdosfs_lookup.c optional msdosfs fs/msdosfs/msdosfs_vfsops.c optional msdosfs fs/msdosfs/msdosfs_vnops.c optional msdosfs fs/nfs/nfs_commonkrpc.c optional nfscl | nfsd fs/nfs/nfs_commonsubs.c optional nfscl | nfsd fs/nfs/nfs_commonport.c optional nfscl | nfsd fs/nfs/nfs_commonacl.c optional nfscl | nfsd fs/nfsclient/nfs_clcomsubs.c optional nfscl fs/nfsclient/nfs_clsubs.c optional nfscl fs/nfsclient/nfs_clstate.c optional nfscl fs/nfsclient/nfs_clkrpc.c optional nfscl fs/nfsclient/nfs_clrpcops.c optional nfscl fs/nfsclient/nfs_clvnops.c optional nfscl fs/nfsclient/nfs_clnode.c optional nfscl fs/nfsclient/nfs_clvfsops.c optional nfscl fs/nfsclient/nfs_clport.c optional nfscl fs/nfsclient/nfs_clbio.c optional nfscl fs/nfsclient/nfs_clnfsiod.c optional nfscl fs/nfsserver/nfs_fha_new.c optional nfsd inet fs/nfsserver/nfs_nfsdsocket.c optional nfsd inet fs/nfsserver/nfs_nfsdsubs.c optional nfsd inet fs/nfsserver/nfs_nfsdstate.c optional nfsd inet fs/nfsserver/nfs_nfsdkrpc.c optional nfsd inet fs/nfsserver/nfs_nfsdserv.c optional nfsd inet fs/nfsserver/nfs_nfsdport.c optional nfsd inet fs/nfsserver/nfs_nfsdcache.c optional nfsd inet fs/ntfs/ntfs_compr.c optional ntfs fs/ntfs/ntfs_iconv.c optional ntfs_iconv fs/ntfs/ntfs_ihash.c optional ntfs fs/ntfs/ntfs_subr.c optional ntfs fs/ntfs/ntfs_vfsops.c optional ntfs fs/ntfs/ntfs_vnops.c optional ntfs fs/nullfs/null_subr.c optional nullfs fs/nullfs/null_vfsops.c optional nullfs fs/nullfs/null_vnops.c optional nullfs fs/nwfs/nwfs_io.c optional nwfs fs/nwfs/nwfs_ioctl.c optional nwfs fs/nwfs/nwfs_node.c optional nwfs fs/nwfs/nwfs_subr.c optional nwfs fs/nwfs/nwfs_vfsops.c optional nwfs fs/nwfs/nwfs_vnops.c optional nwfs fs/portalfs/portal_vfsops.c optional portalfs fs/portalfs/portal_vnops.c optional portalfs fs/procfs/procfs.c optional procfs fs/procfs/procfs_ctl.c optional procfs fs/procfs/procfs_dbregs.c optional procfs fs/procfs/procfs_fpregs.c optional procfs fs/procfs/procfs_ioctl.c optional procfs fs/procfs/procfs_map.c optional procfs fs/procfs/procfs_mem.c optional procfs fs/procfs/procfs_note.c optional procfs fs/procfs/procfs_osrel.c optional procfs fs/procfs/procfs_regs.c optional procfs fs/procfs/procfs_rlimit.c optional procfs fs/procfs/procfs_status.c optional procfs fs/procfs/procfs_type.c optional procfs fs/pseudofs/pseudofs.c optional pseudofs fs/pseudofs/pseudofs_fileno.c optional pseudofs fs/pseudofs/pseudofs_vncache.c optional pseudofs fs/pseudofs/pseudofs_vnops.c optional pseudofs fs/smbfs/smbfs_io.c optional smbfs fs/smbfs/smbfs_node.c optional smbfs fs/smbfs/smbfs_smb.c optional smbfs fs/smbfs/smbfs_subr.c optional smbfs fs/smbfs/smbfs_vfsops.c optional smbfs fs/smbfs/smbfs_vnops.c optional smbfs fs/udf/osta.c optional udf fs/udf/udf_iconv.c optional udf_iconv fs/udf/udf_vfsops.c optional udf fs/udf/udf_vnops.c optional udf fs/unionfs/union_subr.c optional unionfs fs/unionfs/union_vfsops.c optional unionfs fs/unionfs/union_vnops.c optional unionfs fs/tmpfs/tmpfs_vnops.c optional tmpfs fs/tmpfs/tmpfs_fifoops.c optional tmpfs fs/tmpfs/tmpfs_vfsops.c optional tmpfs fs/tmpfs/tmpfs_subr.c optional tmpfs gdb/gdb_cons.c optional gdb gdb/gdb_main.c optional gdb gdb/gdb_packet.c optional gdb geom/bde/g_bde.c optional geom_bde geom/bde/g_bde_crypt.c optional geom_bde geom/bde/g_bde_lock.c optional geom_bde geom/bde/g_bde_work.c optional geom_bde geom/cache/g_cache.c optional geom_cache geom/concat/g_concat.c optional geom_concat geom/eli/g_eli.c optional geom_eli geom/eli/g_eli_crypto.c optional geom_eli geom/eli/g_eli_ctl.c optional geom_eli geom/eli/g_eli_integrity.c optional geom_eli geom/eli/g_eli_key.c optional geom_eli geom/eli/g_eli_key_cache.c optional geom_eli geom/eli/g_eli_privacy.c optional geom_eli geom/eli/pkcs5v2.c optional geom_eli geom/gate/g_gate.c optional geom_gate geom/geom_aes.c optional geom_aes geom/geom_bsd.c optional geom_bsd geom/geom_bsd_enc.c optional geom_bsd geom/geom_ccd.c optional ccd | geom_ccd geom/geom_ctl.c standard geom/geom_dev.c standard geom/geom_disk.c standard geom/geom_dump.c standard geom/geom_event.c standard geom/geom_fox.c optional geom_fox geom/geom_io.c standard geom/geom_kern.c standard geom/geom_map.c optional geom_map geom/geom_mbr.c optional geom_mbr geom/geom_mbr_enc.c optional geom_mbr geom/geom_pc98.c optional geom_pc98 geom/geom_pc98_enc.c optional geom_pc98 geom/geom_redboot.c optional geom_redboot geom/geom_slice.c standard geom/geom_subr.c standard geom/geom_sunlabel.c optional geom_sunlabel geom/geom_sunlabel_enc.c optional geom_sunlabel geom/geom_vfs.c standard geom/geom_vol_ffs.c optional geom_vol geom/journal/g_journal.c optional geom_journal geom/journal/g_journal_ufs.c optional geom_journal geom/label/g_label.c optional geom_label geom/label/g_label_ext2fs.c optional geom_label geom/label/g_label_iso9660.c optional geom_label geom/label/g_label_msdosfs.c optional geom_label geom/label/g_label_ntfs.c optional geom_label geom/label/g_label_reiserfs.c optional geom_label geom/label/g_label_ufs.c optional geom_label geom/label/g_label_gpt.c optional geom_label geom/linux_lvm/g_linux_lvm.c optional geom_linux_lvm geom/mirror/g_mirror.c optional geom_mirror geom/mirror/g_mirror_ctl.c optional geom_mirror geom/mountver/g_mountver.c optional geom_mountver geom/multipath/g_multipath.c optional geom_multipath geom/nop/g_nop.c optional geom_nop geom/part/g_part.c standard geom/part/g_part_if.m standard geom/part/g_part_apm.c optional geom_part_apm geom/part/g_part_bsd.c optional geom_part_bsd geom/part/g_part_ebr.c optional geom_part_ebr geom/part/g_part_gpt.c optional geom_part_gpt geom/part/g_part_ldm.c optional geom_part_ldm geom/part/g_part_mbr.c optional geom_part_mbr geom/part/g_part_pc98.c optional geom_part_pc98 geom/part/g_part_vtoc8.c optional geom_part_vtoc8 geom/raid/g_raid.c optional geom_raid geom/raid/g_raid_ctl.c optional geom_raid geom/raid/g_raid_md_if.m optional geom_raid geom/raid/g_raid_tr_if.m optional geom_raid geom/raid/md_ddf.c optional geom_raid geom/raid/md_intel.c optional geom_raid geom/raid/md_jmicron.c optional geom_raid geom/raid/md_nvidia.c optional geom_raid geom/raid/md_promise.c optional geom_raid geom/raid/md_sii.c optional geom_raid geom/raid/tr_concat.c optional geom_raid geom/raid/tr_raid0.c optional geom_raid geom/raid/tr_raid1.c optional geom_raid geom/raid/tr_raid1e.c optional geom_raid geom/raid/tr_raid5.c optional geom_raid geom/raid3/g_raid3.c optional geom_raid3 geom/raid3/g_raid3_ctl.c optional geom_raid3 geom/shsec/g_shsec.c optional geom_shsec geom/stripe/g_stripe.c optional geom_stripe geom/uncompress/g_uncompress.c optional geom_uncompress contrib/xz-embedded/freebsd/xz_malloc.c \ optional xz_embedded | geom_uncompress \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_crc32.c \ optional xz_embedded | geom_uncompress \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_bcj.c \ optional xz_embedded | geom_uncompress \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_lzma2.c \ optional xz_embedded | geom_uncompress \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_stream.c \ optional xz_embedded | geom_uncompress \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" geom/uzip/g_uzip.c optional geom_uzip geom/vinum/geom_vinum.c optional geom_vinum geom/vinum/geom_vinum_create.c optional geom_vinum geom/vinum/geom_vinum_drive.c optional geom_vinum geom/vinum/geom_vinum_plex.c optional geom_vinum geom/vinum/geom_vinum_volume.c optional geom_vinum geom/vinum/geom_vinum_subr.c optional geom_vinum geom/vinum/geom_vinum_raid5.c optional geom_vinum geom/vinum/geom_vinum_share.c optional geom_vinum geom/vinum/geom_vinum_list.c optional geom_vinum geom/vinum/geom_vinum_rm.c optional geom_vinum geom/vinum/geom_vinum_init.c optional geom_vinum geom/vinum/geom_vinum_state.c optional geom_vinum geom/vinum/geom_vinum_rename.c optional geom_vinum geom/vinum/geom_vinum_move.c optional geom_vinum geom/vinum/geom_vinum_events.c optional geom_vinum geom/virstor/binstream.c optional geom_virstor geom/virstor/g_virstor.c optional geom_virstor geom/virstor/g_virstor_md.c optional geom_virstor geom/zero/g_zero.c optional geom_zero fs/ext2fs/ext2_alloc.c optional ext2fs fs/ext2fs/ext2_balloc.c optional ext2fs fs/ext2fs/ext2_bmap.c optional ext2fs fs/ext2fs/ext2_extents.c optional ext2fs fs/ext2fs/ext2_inode.c optional ext2fs fs/ext2fs/ext2_inode_cnv.c optional ext2fs fs/ext2fs/ext2_hash.c optional ext2fs fs/ext2fs/ext2_htree.c optional ext2fs fs/ext2fs/ext2_lookup.c optional ext2fs fs/ext2fs/ext2_subr.c optional ext2fs fs/ext2fs/ext2_vfsops.c optional ext2fs fs/ext2fs/ext2_vnops.c optional ext2fs gnu/fs/reiserfs/reiserfs_hashes.c optional reiserfs \ warning "kernel contains GPL contaminated ReiserFS filesystem" gnu/fs/reiserfs/reiserfs_inode.c optional reiserfs gnu/fs/reiserfs/reiserfs_item_ops.c optional reiserfs gnu/fs/reiserfs/reiserfs_namei.c optional reiserfs gnu/fs/reiserfs/reiserfs_prints.c optional reiserfs gnu/fs/reiserfs/reiserfs_stree.c optional reiserfs gnu/fs/reiserfs/reiserfs_vfsops.c optional reiserfs gnu/fs/reiserfs/reiserfs_vnops.c optional reiserfs # isa/isa_if.m standard isa/isa_common.c optional isa isa/isahint.c optional isa isa/pnp.c optional isa isapnp isa/pnpparse.c optional isa isapnp fs/cd9660/cd9660_bmap.c optional cd9660 fs/cd9660/cd9660_lookup.c optional cd9660 fs/cd9660/cd9660_node.c optional cd9660 fs/cd9660/cd9660_rrip.c optional cd9660 fs/cd9660/cd9660_util.c optional cd9660 fs/cd9660/cd9660_vfsops.c optional cd9660 fs/cd9660/cd9660_vnops.c optional cd9660 fs/cd9660/cd9660_iconv.c optional cd9660_iconv kern/bus_if.m standard kern/clock_if.m standard kern/cpufreq_if.m standard kern/device_if.m standard kern/imgact_elf.c standard kern/imgact_shell.c standard kern/inflate.c optional gzip kern/init_main.c standard kern/init_sysent.c standard kern/ksched.c optional _kposix_priority_scheduling kern/kern_acct.c standard kern/kern_alq.c optional alq kern/kern_clock.c standard kern/kern_condvar.c standard kern/kern_conf.c standard kern/kern_cons.c standard kern/kern_cpu.c standard kern/kern_cpuset.c standard kern/kern_context.c standard kern/kern_descrip.c standard kern/kern_dtrace.c optional kdtrace_hooks kern/kern_environment.c standard kern/kern_et.c standard kern/kern_event.c standard kern/kern_exec.c standard kern/kern_exit.c standard kern/kern_fail.c standard kern/kern_fork.c standard kern/kern_gzio.c optional gzio kern/kern_hhook.c standard kern/kern_idle.c standard kern/kern_intr.c standard kern/kern_jail.c standard kern/kern_khelp.c standard kern/kern_kthread.c standard kern/kern_ktr.c optional ktr kern/kern_ktrace.c standard kern/kern_linker.c standard kern/kern_lock.c standard kern/kern_lockf.c standard kern/kern_lockstat.c optional kdtrace_hooks kern/kern_loginclass.c standard kern/kern_malloc.c standard kern/kern_mbuf.c standard kern/kern_mib.c standard kern/kern_module.c standard kern/kern_mtxpool.c standard kern/kern_mutex.c standard kern/kern_ntptime.c standard kern/kern_osd.c standard kern/kern_physio.c standard kern/kern_pmc.c standard kern/kern_poll.c optional device_polling kern/kern_priv.c standard kern/kern_proc.c standard kern/kern_prot.c standard kern/kern_racct.c standard kern/kern_rangelock.c standard kern/kern_rctl.c standard kern/kern_resource.c standard kern/kern_rmlock.c standard kern/kern_rwlock.c standard kern/kern_sdt.c optional kdtrace_hooks kern/kern_sema.c standard kern/kern_sharedpage.c standard kern/kern_shutdown.c standard kern/kern_sig.c standard kern/kern_switch.c standard kern/kern_sx.c standard kern/kern_synch.c standard kern/kern_syscalls.c standard kern/kern_sysctl.c standard kern/kern_tc.c standard kern/kern_thr.c standard kern/kern_thread.c standard kern/kern_time.c standard kern/kern_timeout.c standard kern/kern_umtx.c standard kern/kern_uuid.c standard kern/kern_xxx.c standard kern/link_elf.c standard kern/linker_if.m standard kern/md4c.c optional netsmb kern/md5c.c standard kern/p1003_1b.c standard kern/posix4_mib.c standard kern/sched_4bsd.c optional sched_4bsd kern/sched_ule.c optional sched_ule kern/serdev_if.m standard kern/stack_protector.c standard \ compile-with "${NORMAL_C:N-fstack-protector*}" kern/subr_acl_nfs4.c standard kern/subr_acl_posix1e.c standard kern/subr_autoconf.c standard kern/subr_blist.c standard kern/subr_bus.c standard kern/subr_bus_dma.c standard kern/subr_bufring.c standard kern/subr_clock.c standard kern/subr_devstat.c standard kern/subr_disk.c standard kern/subr_eventhandler.c standard kern/subr_fattime.c standard kern/subr_firmware.c optional firmware kern/subr_hash.c standard kern/subr_hints.c standard kern/subr_kdb.c standard kern/subr_kobj.c standard kern/subr_lock.c standard kern/subr_log.c standard kern/subr_mbpool.c optional libmbpool kern/subr_mchain.c optional libmchain kern/subr_module.c standard kern/subr_msgbuf.c standard kern/subr_param.c standard kern/subr_pcpu.c standard kern/subr_power.c standard kern/subr_prf.c standard kern/subr_prof.c standard kern/subr_rman.c standard kern/subr_rtc.c standard kern/subr_sbuf.c standard kern/subr_scanf.c standard kern/subr_sglist.c standard kern/subr_sleepqueue.c standard kern/subr_smp.c standard kern/subr_stack.c optional ddb | stack | ktr kern/subr_taskqueue.c standard kern/subr_terminal.c optional vt kern/subr_trap.c standard kern/subr_turnstile.c standard kern/subr_uio.c standard kern/subr_unit.c standard kern/subr_witness.c optional witness kern/sys_capability.c standard kern/sys_generic.c standard kern/sys_pipe.c standard kern/sys_procdesc.c standard kern/sys_process.c standard kern/sys_socket.c standard kern/syscalls.c standard kern/sysv_ipc.c standard kern/sysv_msg.c optional sysvmsg kern/sysv_sem.c optional sysvsem kern/sysv_shm.c optional sysvshm kern/tty.c standard kern/tty_compat.c optional compat_43tty kern/tty_info.c standard kern/tty_inq.c standard kern/tty_outq.c standard kern/tty_pts.c standard kern/tty_tty.c standard kern/tty_ttydisc.c standard kern/uipc_accf.c optional inet kern/uipc_cow.c optional zero_copy_sockets kern/uipc_debug.c optional ddb kern/uipc_domain.c standard kern/uipc_mbuf.c standard kern/uipc_mbuf2.c standard kern/uipc_mqueue.c optional p1003_1b_mqueue kern/uipc_sem.c optional p1003_1b_semaphores kern/uipc_shm.c standard kern/uipc_sockbuf.c standard kern/uipc_socket.c standard kern/uipc_syscalls.c standard kern/uipc_usrreq.c standard kern/vfs_acl.c standard kern/vfs_aio.c optional vfs_aio kern/vfs_bio.c standard kern/vfs_cache.c standard kern/vfs_cluster.c standard kern/vfs_default.c standard kern/vfs_export.c standard kern/vfs_extattr.c standard kern/vfs_hash.c standard kern/vfs_init.c standard kern/vfs_lookup.c standard kern/vfs_mount.c standard kern/vfs_mountroot.c standard kern/vfs_subr.c standard kern/vfs_syscalls.c standard kern/vfs_vnops.c standard # # Kernel GSS-API # gssd.h optional kgssapi \ dependency "$S/kgssapi/gssd.x" \ compile-with "RPCGEN_CPP='${CPP}' rpcgen -hM $S/kgssapi/gssd.x | grep -v pthread.h > gssd.h" \ no-obj no-implicit-rule before-depend local \ clean "gssd.h" gssd_xdr.c optional kgssapi \ dependency "$S/kgssapi/gssd.x gssd.h" \ compile-with "RPCGEN_CPP='${CPP}' rpcgen -c $S/kgssapi/gssd.x -o gssd_xdr.c" \ no-implicit-rule before-depend local \ clean "gssd_xdr.c" gssd_clnt.c optional kgssapi \ dependency "$S/kgssapi/gssd.x gssd.h" \ compile-with "RPCGEN_CPP='${CPP}' rpcgen -lM $S/kgssapi/gssd.x | grep -v string.h > gssd_clnt.c" \ no-implicit-rule before-depend local \ clean "gssd_clnt.c" kgssapi/gss_accept_sec_context.c optional kgssapi kgssapi/gss_add_oid_set_member.c optional kgssapi kgssapi/gss_acquire_cred.c optional kgssapi kgssapi/gss_canonicalize_name.c optional kgssapi kgssapi/gss_create_empty_oid_set.c optional kgssapi kgssapi/gss_delete_sec_context.c optional kgssapi kgssapi/gss_display_status.c optional kgssapi kgssapi/gss_export_name.c optional kgssapi kgssapi/gss_get_mic.c optional kgssapi kgssapi/gss_init_sec_context.c optional kgssapi kgssapi/gss_impl.c optional kgssapi kgssapi/gss_import_name.c optional kgssapi kgssapi/gss_names.c optional kgssapi kgssapi/gss_pname_to_uid.c optional kgssapi kgssapi/gss_release_buffer.c optional kgssapi kgssapi/gss_release_cred.c optional kgssapi kgssapi/gss_release_name.c optional kgssapi kgssapi/gss_release_oid_set.c optional kgssapi kgssapi/gss_set_cred_option.c optional kgssapi kgssapi/gss_test_oid_set_member.c optional kgssapi kgssapi/gss_unwrap.c optional kgssapi kgssapi/gss_verify_mic.c optional kgssapi kgssapi/gss_wrap.c optional kgssapi kgssapi/gss_wrap_size_limit.c optional kgssapi kgssapi/gssd_prot.c optional kgssapi kgssapi/krb5/krb5_mech.c optional kgssapi kgssapi/krb5/kcrypto.c optional kgssapi kgssapi/krb5/kcrypto_aes.c optional kgssapi kgssapi/krb5/kcrypto_arcfour.c optional kgssapi kgssapi/krb5/kcrypto_des.c optional kgssapi kgssapi/krb5/kcrypto_des3.c optional kgssapi kgssapi/kgss_if.m optional kgssapi kgssapi/gsstest.c optional kgssapi_debug # These files in libkern/ are those needed by all architectures. Some # of the files in libkern/ are only needed on some architectures, e.g., # libkern/divdi3.c is needed by i386 but not alpha. Also, some of these # routines may be optimized for a particular platform. In either case, # the file should be moved to conf/files. from here. # libkern/arc4random.c standard libkern/bcd.c standard libkern/bsearch.c standard libkern/crc32.c standard libkern/fnmatch.c standard libkern/iconv.c optional libiconv libkern/iconv_converter_if.m optional libiconv libkern/iconv_ucs.c optional libiconv libkern/iconv_xlat.c optional libiconv libkern/iconv_xlat16.c optional libiconv libkern/index.c standard libkern/inet_aton.c standard libkern/inet_ntoa.c standard libkern/inet_ntop.c standard libkern/inet_pton.c standard libkern/mcount.c optional profiling-routine libkern/memcmp.c standard libkern/qsort.c standard libkern/qsort_r.c standard libkern/random.c standard libkern/rindex.c standard libkern/scanc.c standard libkern/skpc.c standard libkern/strcasecmp.c standard libkern/strcat.c standard libkern/strcmp.c standard libkern/strcpy.c standard libkern/strcspn.c standard libkern/strdup.c standard libkern/strlcat.c standard libkern/strlcpy.c standard libkern/strlen.c standard libkern/strncmp.c standard libkern/strncpy.c standard libkern/strnlen.c standard libkern/strsep.c standard libkern/strspn.c standard libkern/strstr.c standard libkern/strtol.c standard libkern/strtoq.c standard libkern/strtoul.c standard libkern/strtouq.c standard libkern/strvalid.c standard net/bpf.c standard net/bpf_buffer.c optional bpf net/bpf_jitter.c optional bpf_jitter net/bpf_filter.c optional bpf | netgraph_bpf net/bpf_zerocopy.c optional bpf net/bridgestp.c optional bridge | if_bridge net/flowtable.c optional flowtable inet | flowtable inet6 net/ieee8023ad_lacp.c optional lagg net/if.c standard net/if_arcsubr.c optional arcnet net/if_atmsubr.c optional atm net/if_bridge.c optional bridge inet | if_bridge inet net/if_clone.c standard net/if_dead.c standard net/if_debug.c optional ddb net/if_disc.c optional disc net/if_edsc.c optional edsc net/if_ef.c optional ef net/if_enc.c optional enc ipsec inet | enc ipsec inet6 net/if_epair.c optional epair net/if_ethersubr.c optional ether \ compile-with "${NORMAL_C} -I$S/contrib/pf" net/if_faith.c optional faith net/if_fddisubr.c optional fddi net/if_fwsubr.c optional fwip net/if_gif.c optional gif | netgraph_gif net/if_gre.c optional gre inet net/if_iso88025subr.c optional token net/if_lagg.c optional lagg net/if_loop.c optional loop net/if_llatbl.c standard net/if_media.c standard net/if_mib.c standard net/if_spppfr.c optional sppp | netgraph_sppp net/if_spppsubr.c optional sppp | netgraph_sppp net/if_stf.c optional stf inet inet6 net/if_tun.c optional tun net/if_tap.c optional tap net/if_vlan.c optional vlan net/mppcc.c optional netgraph_mppc_compression net/mppcd.c optional netgraph_mppc_compression net/netisr.c standard net/pfil.c optional ether | inet net/radix.c standard net/radix_mpath.c standard net/raw_cb.c standard net/raw_usrreq.c standard net/route.c standard net/rtsock.c standard net/slcompress.c optional netgraph_vjc | sppp | \ netgraph_sppp net/vnet.c optional vimage net/zlib.c optional crypto | geom_uzip | ipsec | \ mxge | netgraph_deflate | \ ddb_ctf | gzio | geom_uncompress net80211/ieee80211.c optional wlan net80211/ieee80211_acl.c optional wlan wlan_acl net80211/ieee80211_action.c optional wlan net80211/ieee80211_ageq.c optional wlan net80211/ieee80211_adhoc.c optional wlan \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_ageq.c optional wlan net80211/ieee80211_amrr.c optional wlan | wlan_amrr net80211/ieee80211_crypto.c optional wlan \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_crypto_ccmp.c optional wlan wlan_ccmp net80211/ieee80211_crypto_none.c optional wlan net80211/ieee80211_crypto_tkip.c optional wlan wlan_tkip net80211/ieee80211_crypto_wep.c optional wlan wlan_wep net80211/ieee80211_ddb.c optional wlan ddb net80211/ieee80211_dfs.c optional wlan net80211/ieee80211_freebsd.c optional wlan net80211/ieee80211_hostap.c optional wlan \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_ht.c optional wlan net80211/ieee80211_hwmp.c optional wlan ieee80211_support_mesh net80211/ieee80211_input.c optional wlan net80211/ieee80211_ioctl.c optional wlan net80211/ieee80211_mesh.c optional wlan ieee80211_support_mesh \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_monitor.c optional wlan net80211/ieee80211_node.c optional wlan net80211/ieee80211_output.c optional wlan net80211/ieee80211_phy.c optional wlan net80211/ieee80211_power.c optional wlan net80211/ieee80211_proto.c optional wlan net80211/ieee80211_radiotap.c optional wlan net80211/ieee80211_ratectl.c optional wlan net80211/ieee80211_ratectl_none.c optional wlan net80211/ieee80211_regdomain.c optional wlan net80211/ieee80211_rssadapt.c optional wlan wlan_rssadapt net80211/ieee80211_scan.c optional wlan net80211/ieee80211_scan_sta.c optional wlan net80211/ieee80211_sta.c optional wlan \ compile-with "${NORMAL_C} -Wno-unused-function" net80211/ieee80211_superg.c optional wlan ieee80211_support_superg net80211/ieee80211_tdma.c optional wlan ieee80211_support_tdma net80211/ieee80211_wds.c optional wlan net80211/ieee80211_xauth.c optional wlan wlan_xauth net80211/ieee80211_alq.c optional wlan ieee80211_alq netatalk/aarp.c optional netatalk netatalk/at_control.c optional netatalk netatalk/at_proto.c optional netatalk netatalk/at_rmx.c optional netatalk netatalk/ddp_input.c optional netatalk netatalk/ddp_output.c optional netatalk netatalk/ddp_pcb.c optional netatalk netatalk/ddp_usrreq.c optional netatalk netgraph/atm/ccatm/ng_ccatm.c optional ngatm_ccatm \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/ng_atm.c optional ngatm_atm netgraph/atm/ngatmbase.c optional ngatm_atmbase \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/sscfu/ng_sscfu.c optional ngatm_sscfu \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/sscop/ng_sscop.c optional ngatm_sscop \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/atm/uni/ng_uni.c optional ngatm_uni \ compile-with "${NORMAL_C} -I$S/contrib/ngatm" netgraph/bluetooth/common/ng_bluetooth.c optional netgraph_bluetooth netgraph/bluetooth/drivers/bt3c/ng_bt3c_pccard.c optional netgraph_bluetooth_bt3c netgraph/bluetooth/drivers/h4/ng_h4.c optional netgraph_bluetooth_h4 netgraph/bluetooth/drivers/ubt/ng_ubt.c optional netgraph_bluetooth_ubt usb netgraph/bluetooth/drivers/ubtbcmfw/ubtbcmfw.c optional netgraph_bluetooth_ubtbcmfw usb netgraph/bluetooth/hci/ng_hci_cmds.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_evnt.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_main.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_misc.c optional netgraph_bluetooth_hci netgraph/bluetooth/hci/ng_hci_ulpi.c optional netgraph_bluetooth_hci netgraph/bluetooth/l2cap/ng_l2cap_cmds.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_evnt.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_llpi.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_main.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_misc.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/l2cap/ng_l2cap_ulpi.c optional netgraph_bluetooth_l2cap netgraph/bluetooth/socket/ng_btsocket.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_hci_raw.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_l2cap.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_l2cap_raw.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_rfcomm.c optional netgraph_bluetooth_socket netgraph/bluetooth/socket/ng_btsocket_sco.c optional netgraph_bluetooth_socket netgraph/netflow/netflow.c optional netgraph_netflow netgraph/netflow/netflow_v9.c optional netgraph_netflow netgraph/netflow/ng_netflow.c optional netgraph_netflow netgraph/ng_UI.c optional netgraph_UI netgraph/ng_async.c optional netgraph_async netgraph/ng_atmllc.c optional netgraph_atmllc netgraph/ng_base.c optional netgraph netgraph/ng_bpf.c optional netgraph_bpf netgraph/ng_bridge.c optional netgraph_bridge netgraph/ng_car.c optional netgraph_car netgraph/ng_cisco.c optional netgraph_cisco netgraph/ng_deflate.c optional netgraph_deflate netgraph/ng_device.c optional netgraph_device netgraph/ng_echo.c optional netgraph_echo netgraph/ng_eiface.c optional netgraph_eiface netgraph/ng_ether.c optional netgraph_ether netgraph/ng_ether_echo.c optional netgraph_ether_echo netgraph/ng_fec.c optional netgraph_fec netgraph/ng_frame_relay.c optional netgraph_frame_relay netgraph/ng_gif.c optional netgraph_gif netgraph/ng_gif_demux.c optional netgraph_gif_demux netgraph/ng_hole.c optional netgraph_hole netgraph/ng_iface.c optional netgraph_iface netgraph/ng_ip_input.c optional netgraph_ip_input netgraph/ng_ipfw.c optional netgraph_ipfw inet ipfirewall netgraph/ng_ksocket.c optional netgraph_ksocket netgraph/ng_l2tp.c optional netgraph_l2tp netgraph/ng_lmi.c optional netgraph_lmi netgraph/ng_mppc.c optional netgraph_mppc_compression | \ netgraph_mppc_encryption netgraph/ng_nat.c optional netgraph_nat inet libalias netgraph/ng_one2many.c optional netgraph_one2many netgraph/ng_parse.c optional netgraph netgraph/ng_patch.c optional netgraph_patch netgraph/ng_pipe.c optional netgraph_pipe netgraph/ng_ppp.c optional netgraph_ppp netgraph/ng_pppoe.c optional netgraph_pppoe netgraph/ng_pptpgre.c optional netgraph_pptpgre netgraph/ng_pred1.c optional netgraph_pred1 netgraph/ng_rfc1490.c optional netgraph_rfc1490 netgraph/ng_socket.c optional netgraph_socket netgraph/ng_split.c optional netgraph_split netgraph/ng_sppp.c optional netgraph_sppp netgraph/ng_tag.c optional netgraph_tag netgraph/ng_tcpmss.c optional netgraph_tcpmss netgraph/ng_tee.c optional netgraph_tee netgraph/ng_tty.c optional netgraph_tty netgraph/ng_vjc.c optional netgraph_vjc netgraph/ng_vlan.c optional netgraph_vlan netinet/accf_data.c optional accept_filter_data inet netinet/accf_dns.c optional accept_filter_dns inet netinet/accf_http.c optional accept_filter_http inet netinet/if_atm.c optional atm netinet/if_ether.c optional inet ether netinet/igmp.c optional inet netinet/in.c optional inet netinet/in_debug.c optional inet ddb netinet/ip_carp.c optional inet carp | inet6 carp netinet/in_gif.c optional gif inet | netgraph_gif inet netinet/ip_gre.c optional gre inet netinet/ip_id.c optional inet netinet/in_mcast.c optional inet netinet/in_pcb.c optional inet | inet6 netinet/in_pcbgroup.c optional inet pcbgroup | inet6 pcbgroup netinet/in_proto.c optional inet | inet6 \ compile-with "${NORMAL_C} -I$S/contrib/pf" netinet/in_rmx.c optional inet netinet/ip_divert.c optional inet ipdivert ipfirewall netinet/ip_ecn.c optional inet | inet6 netinet/ip_encap.c optional inet | inet6 netinet/ip_fastfwd.c optional inet netinet/ip_icmp.c optional inet | inet6 netinet/ip_input.c optional inet netinet/ip_ipsec.c optional inet ipsec netinet/ip_mroute.c optional mrouting inet netinet/ip_options.c optional inet netinet/ip_output.c optional inet netinet/raw_ip.c optional inet | inet6 netinet/cc/cc.c optional inet | inet6 netinet/cc/cc_newreno.c optional inet | inet6 netinet/sctp_asconf.c optional inet sctp | inet6 sctp netinet/sctp_auth.c optional inet sctp | inet6 sctp netinet/sctp_bsd_addr.c optional inet sctp | inet6 sctp netinet/sctp_cc_functions.c optional inet sctp | inet6 sctp netinet/sctp_crc32.c optional inet sctp | inet6 sctp netinet/sctp_indata.c optional inet sctp | inet6 sctp netinet/sctp_input.c optional inet sctp | inet6 sctp netinet/sctp_output.c optional inet sctp | inet6 sctp netinet/sctp_pcb.c optional inet sctp | inet6 sctp netinet/sctp_peeloff.c optional inet sctp | inet6 sctp netinet/sctp_ss_functions.c optional inet sctp | inet6 sctp netinet/sctp_sysctl.c optional inet sctp | inet6 sctp netinet/sctp_timer.c optional inet sctp | inet6 sctp netinet/sctp_usrreq.c optional inet sctp | inet6 sctp netinet/sctputil.c optional inet sctp | inet6 sctp netinet/tcp_debug.c optional tcpdebug netinet/tcp_hostcache.c optional inet | inet6 netinet/tcp_input.c optional inet | inet6 netinet/tcp_lro.c optional inet | inet6 netinet/tcp_output.c optional inet | inet6 netinet/tcp_offload.c optional tcp_offload inet | tcp_offload inet6 netinet/tcp_reass.c optional inet | inet6 netinet/tcp_sack.c optional inet | inet6 netinet/tcp_subr.c optional inet | inet6 netinet/tcp_syncache.c optional inet | inet6 netinet/tcp_timer.c optional inet | inet6 netinet/tcp_timewait.c optional inet | inet6 netinet/tcp_usrreq.c optional inet | inet6 netinet/udp_usrreq.c optional inet | inet6 netinet/libalias/alias.c optional libalias inet | netgraph_nat inet netinet/libalias/alias_db.c optional libalias inet | netgraph_nat inet netinet/libalias/alias_mod.c optional libalias | netgraph_nat netinet/libalias/alias_proxy.c optional libalias inet | netgraph_nat inet netinet/libalias/alias_util.c optional libalias inet | netgraph_nat inet netinet/libalias/alias_sctp.c optional libalias inet | netgraph_nat inet netinet6/dest6.c optional inet6 netinet6/frag6.c optional inet6 netinet6/icmp6.c optional inet6 netinet6/in6.c optional inet6 netinet6/in6_cksum.c optional inet6 netinet6/in6_gif.c optional gif inet6 | netgraph_gif inet6 netinet6/in6_ifattach.c optional inet6 netinet6/in6_mcast.c optional inet6 netinet6/in6_pcb.c optional inet6 netinet6/in6_pcbgroup.c optional inet6 pcbgroup netinet6/in6_proto.c optional inet6 netinet6/in6_rmx.c optional inet6 netinet6/in6_src.c optional inet6 netinet6/ip6_forward.c optional inet6 netinet6/ip6_id.c optional inet6 netinet6/ip6_input.c optional inet6 netinet6/ip6_mroute.c optional mrouting inet6 netinet6/ip6_output.c optional inet6 netinet6/ip6_ipsec.c optional inet6 ipsec netinet6/mld6.c optional inet6 netinet6/nd6.c optional inet6 netinet6/nd6_nbr.c optional inet6 netinet6/nd6_rtr.c optional inet6 netinet6/raw_ip6.c optional inet6 netinet6/route6.c optional inet6 netinet6/scope6.c optional inet6 netinet6/sctp6_usrreq.c optional inet6 sctp netinet6/udp6_usrreq.c optional inet6 netipsec/ipsec.c optional ipsec inet | ipsec inet6 netipsec/ipsec_input.c optional ipsec inet | ipsec inet6 netipsec/ipsec_mbuf.c optional ipsec inet | ipsec inet6 netipsec/ipsec_output.c optional ipsec inet | ipsec inet6 netipsec/key.c optional ipsec inet | ipsec inet6 netipsec/key_debug.c optional ipsec inet | ipsec inet6 netipsec/keysock.c optional ipsec inet | ipsec inet6 netipsec/xform_ah.c optional ipsec inet | ipsec inet6 netipsec/xform_esp.c optional ipsec inet | ipsec inet6 netipsec/xform_ipcomp.c optional ipsec inet | ipsec inet6 netipsec/xform_ipip.c optional ipsec inet | ipsec inet6 netipsec/xform_tcp.c optional ipsec inet tcp_signature | \ ipsec inet6 tcp_signature netipx/ipx.c optional ipx netipx/ipx_cksum.c optional ipx netipx/ipx_input.c optional ipx netipx/ipx_outputfl.c optional ipx netipx/ipx_pcb.c optional ipx netipx/ipx_proto.c optional ipx netipx/ipx_usrreq.c optional ipx netipx/spx_debug.c optional ipx netipx/spx_reass.c optional ipx netipx/spx_usrreq.c optional ipx netnatm/natm.c optional natm netnatm/natm_pcb.c optional natm netnatm/natm_proto.c optional natm netncp/ncp_conn.c optional ncp netncp/ncp_crypt.c optional ncp netncp/ncp_login.c optional ncp netncp/ncp_mod.c optional ncp netncp/ncp_ncp.c optional ncp netncp/ncp_nls.c optional ncp netncp/ncp_rq.c optional ncp netncp/ncp_sock.c optional ncp netncp/ncp_subr.c optional ncp netpfil/ipfw/dn_heap.c optional inet dummynet netpfil/ipfw/dn_sched_fifo.c optional inet dummynet netpfil/ipfw/dn_sched_prio.c optional inet dummynet netpfil/ipfw/dn_sched_qfq.c optional inet dummynet netpfil/ipfw/dn_sched_rr.c optional inet dummynet netpfil/ipfw/dn_sched_wf2q.c optional inet dummynet netpfil/ipfw/ip_dummynet.c optional inet dummynet netpfil/ipfw/ip_dn_io.c optional inet dummynet netpfil/ipfw/ip_dn_glue.c optional inet dummynet netpfil/ipfw/ip_fw2.c optional inet ipfirewall \ compile-with "${NORMAL_C} -I$S/contrib/pf" netpfil/ipfw/ip_fw_dynamic.c optional inet ipfirewall netpfil/ipfw/ip_fw_log.c optional inet ipfirewall netpfil/ipfw/ip_fw_pfil.c optional inet ipfirewall netpfil/ipfw/ip_fw_sockopt.c optional inet ipfirewall netpfil/ipfw/ip_fw_table.c optional inet ipfirewall netpfil/ipfw/ip_fw_nat.c optional inet ipfirewall_nat netsmb/smb_conn.c optional netsmb netsmb/smb_crypt.c optional netsmb netsmb/smb_dev.c optional netsmb netsmb/smb_iod.c optional netsmb netsmb/smb_rq.c optional netsmb netsmb/smb_smb.c optional netsmb netsmb/smb_subr.c optional netsmb netsmb/smb_trantcp.c optional netsmb netsmb/smb_usr.c optional netsmb nfs/bootp_subr.c optional bootp nfsclient | bootp nfscl nfs/krpc_subr.c optional bootp nfsclient | bootp nfscl nfs/nfs_common.c optional nfsclient | nfsserver nfs/nfs_diskless.c optional nfsclient nfs_root | nfscl nfs_root nfs/nfs_fha.c optional nfsserver | nfsd nfs/nfs_lock.c optional nfsclient | nfscl | nfslockd | nfsd nfsclient/nfs_bio.c optional nfsclient nfsclient/nfs_node.c optional nfsclient nfsclient/nfs_krpc.c optional nfsclient nfsclient/nfs_subs.c optional nfsclient nfsclient/nfs_nfsiod.c optional nfsclient nfsclient/nfs_vfsops.c optional nfsclient nfsclient/nfs_vnops.c optional nfsclient nfsserver/nfs_fha_old.c optional nfsserver nfsserver/nfs_serv.c optional nfsserver nfsserver/nfs_srvkrpc.c optional nfsserver nfsserver/nfs_srvsubs.c optional nfsserver nfs/nfs_nfssvc.c optional nfsserver | nfscl | nfsd nlm/nlm_advlock.c optional nfslockd | nfsd nlm/nlm_prot_clnt.c optional nfslockd | nfsd nlm/nlm_prot_impl.c optional nfslockd | nfsd nlm/nlm_prot_server.c optional nfslockd | nfsd nlm/nlm_prot_svc.c optional nfslockd | nfsd nlm/nlm_prot_xdr.c optional nfslockd | nfsd nlm/sm_inter_xdr.c optional nfslockd | nfsd # OpenFabrics Enterprise Distribution (Infiniband) ofed/include/linux/linux_compat.c optional ofed \ no-depend compile-with "${OFED_C}" ofed/include/linux/linux_idr.c optional ofed \ no-depend compile-with "${OFED_C}" ofed/include/linux/linux_radix.c optional ofed \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/core/addr.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/agent.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/cache.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" # XXX Mad.c must be ordered before cm.c for sysinit sets to occur in # the correct order. ofed/drivers/infiniband/core/mad.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/cm.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/ -Wno-unused-function" ofed/drivers/infiniband/core/cma.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/device.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/fmr_pool.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/iwcm.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/local_sa.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/mad_rmpp.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/multicast.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/notice.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/packer.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/sa_query.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/smi.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/sysfs.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/ucm.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/ucma.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/ud_header.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/umem.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/user_mad.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/uverbs_cmd.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/uverbs_main.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/uverbs_marshall.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/core/verbs.c optional ofed \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/" ofed/drivers/infiniband/ulp/ipoib/ipoib_cm.c optional ipoib \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" #ofed/drivers/infiniband/ulp/ipoib/ipoib_fs.c optional ipoib \ # no-depend \ # compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/ipoib/ipoib_ib.c optional ipoib \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c optional ipoib \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/ipoib/ipoib_multicast.c optional ipoib \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/ipoib/ipoib_verbs.c optional ipoib \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" #ofed/drivers/infiniband/ulp/ipoib/ipoib_vlan.c optional ipoib \ # no-depend \ # compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/" ofed/drivers/infiniband/ulp/sdp/sdp_bcopy.c optional sdp inet \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" ofed/drivers/infiniband/ulp/sdp/sdp_main.c optional sdp inet \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" ofed/drivers/infiniband/ulp/sdp/sdp_rx.c optional sdp inet \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" ofed/drivers/infiniband/ulp/sdp/sdp_cma.c optional sdp inet \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" ofed/drivers/infiniband/ulp/sdp/sdp_tx.c optional sdp inet \ no-depend \ compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/" +ofed/drivers/infiniband/hw/mlx4/alias_GUID.c optional mlx4ib \ + no-depend obj-prefix "mlx4ib_" \ + compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" +ofed/drivers/infiniband/hw/mlx4/mcg.c optional mlx4ib \ + no-depend obj-prefix "mlx4ib_" \ + compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" +ofed/drivers/infiniband/hw/mlx4/sysfs.c optional mlx4ib \ + no-depend obj-prefix "mlx4ib_" \ + compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" +ofed/drivers/infiniband/hw/mlx4/cm.c optional mlx4ib \ + no-depend obj-prefix "mlx4ib_" \ + compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/ah.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/cq.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/doorbell.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/mad.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/main.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/mr.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/qp.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/srq.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/infiniband/hw/mlx4/wc.c optional mlx4ib \ no-depend obj-prefix "mlx4ib_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/infiniband/hw/mlx4/" ofed/drivers/net/mlx4/alloc.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/catas.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/cmd.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/cq.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/eq.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/fw.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/icm.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/intf.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/main.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/mcg.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/ -Wno-unused" ofed/drivers/net/mlx4/mr.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/pd.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/port.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/profile.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/qp.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/reset.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/sense.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/srq.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/xrcd.c optional mlx4ib | mlxen \ +ofed/drivers/net/mlx4/resource_tracker.c optional mlx4ib | mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" +ofed/drivers/net/mlx4/sys_tune.c optional mlx4ib | mlxen \ + no-depend obj-prefix "mlx4_" \ + compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_cq.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" -ofed/drivers/net/mlx4/en_frag.c optional mlxen \ +ofed/drivers/net/mlx4/utils.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_main.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_netdev.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_port.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_resources.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_rx.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/net/mlx4/en_tx.c optional mlxen \ no-depend obj-prefix "mlx4_" \ compile-with "${OFED_C_NOIMP} -I$S/ofed/drivers/net/mlx4/" ofed/drivers/infiniband/hw/mthca/mthca_allocator.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_av.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_catas.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_cmd.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_cq.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_eq.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_mad.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_main.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_mcg.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_memfree.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_mr.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_pd.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_profile.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_provider.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_qp.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_reset.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_srq.c optional mthca \ no-depend compile-with "${OFED_C}" ofed/drivers/infiniband/hw/mthca/mthca_uar.c optional mthca \ no-depend compile-with "${OFED_C}" # crypto support opencrypto/cast.c optional crypto | ipsec opencrypto/criov.c optional crypto opencrypto/crypto.c optional crypto opencrypto/cryptodev.c optional cryptodev opencrypto/cryptodev_if.m optional crypto opencrypto/cryptosoft.c optional crypto opencrypto/deflate.c optional crypto opencrypto/rmd160.c optional crypto | ipsec opencrypto/skipjack.c optional crypto opencrypto/xform.c optional crypto pci/alpm.c optional alpm pci pci/amdpm.c optional amdpm pci | nfpm pci pci/amdsmb.c optional amdsmb pci pci/if_rl.c optional rl pci pci/intpm.c optional intpm pci pci/ncr.c optional ncr pci \ compile-with "${NORMAL_C} -Wno-unused" pci/nfsmb.c optional nfsmb pci pci/viapm.c optional viapm pci rpc/auth_none.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd rpc/auth_unix.c optional krpc | nfslockd | nfsclient | nfscl | nfsd rpc/authunix_prot.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd rpc/clnt_dg.c optional krpc | nfslockd | nfsclient | nfscl | nfsd rpc/clnt_rc.c optional krpc | nfslockd | nfsclient | nfscl | nfsd rpc/clnt_vc.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd rpc/getnetconfig.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd rpc/replay.c optional krpc | nfslockd | nfsserver | nfscl | nfsd rpc/rpc_callmsg.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd rpc/rpc_generic.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd rpc/rpc_prot.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd rpc/rpcb_clnt.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd rpc/rpcb_prot.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd rpc/svc.c optional krpc | nfslockd | nfsserver | nfscl | nfsd rpc/svc_auth.c optional krpc | nfslockd | nfsserver | nfscl | nfsd rpc/svc_auth_unix.c optional krpc | nfslockd | nfsserver | nfscl | nfsd rpc/svc_dg.c optional krpc | nfslockd | nfsserver | nfscl | nfsd rpc/svc_generic.c optional krpc | nfslockd | nfsserver | nfscl | nfsd rpc/svc_vc.c optional krpc | nfslockd | nfsserver | nfscl | nfsd rpc/rpcsec_gss/rpcsec_gss.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi rpc/rpcsec_gss/rpcsec_gss_conf.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi rpc/rpcsec_gss/rpcsec_gss_misc.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi rpc/rpcsec_gss/rpcsec_gss_prot.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi rpc/rpcsec_gss/svc_rpcsec_gss.c optional krpc kgssapi | nfslockd kgssapi | nfscl kgssapi | nfsd kgssapi security/audit/audit.c optional audit security/audit/audit_arg.c optional audit security/audit/audit_bsm.c optional audit security/audit/audit_bsm_domain.c optional audit security/audit/audit_bsm_errno.c optional audit security/audit/audit_bsm_fcntl.c optional audit security/audit/audit_bsm_klib.c optional audit security/audit/audit_bsm_socket_type.c optional audit security/audit/audit_bsm_token.c optional audit security/audit/audit_pipe.c optional audit security/audit/audit_syscalls.c standard security/audit/audit_trigger.c optional audit security/audit/audit_worker.c optional audit security/mac/mac_atalk.c optional mac netatalk security/mac/mac_audit.c optional mac audit security/mac/mac_cred.c optional mac security/mac/mac_framework.c optional mac security/mac/mac_inet.c optional mac inet | mac inet6 security/mac/mac_inet6.c optional mac inet6 security/mac/mac_label.c optional mac security/mac/mac_net.c optional mac security/mac/mac_pipe.c optional mac security/mac/mac_posix_sem.c optional mac security/mac/mac_posix_shm.c optional mac security/mac/mac_priv.c optional mac security/mac/mac_process.c optional mac security/mac/mac_socket.c optional mac security/mac/mac_syscalls.c standard security/mac/mac_system.c optional mac security/mac/mac_sysv_msg.c optional mac security/mac/mac_sysv_sem.c optional mac security/mac/mac_sysv_shm.c optional mac security/mac/mac_vfs.c optional mac security/mac_biba/mac_biba.c optional mac_biba security/mac_bsdextended/mac_bsdextended.c optional mac_bsdextended security/mac_bsdextended/ugidfw_system.c optional mac_bsdextended security/mac_bsdextended/ugidfw_vnode.c optional mac_bsdextended security/mac_ifoff/mac_ifoff.c optional mac_ifoff security/mac_lomac/mac_lomac.c optional mac_lomac security/mac_mls/mac_mls.c optional mac_mls security/mac_none/mac_none.c optional mac_none security/mac_partition/mac_partition.c optional mac_partition security/mac_portacl/mac_portacl.c optional mac_portacl security/mac_seeotheruids/mac_seeotheruids.c optional mac_seeotheruids security/mac_stub/mac_stub.c optional mac_stub security/mac_test/mac_test.c optional mac_test teken/teken.c optional sc | vt ufs/ffs/ffs_alloc.c optional ffs ufs/ffs/ffs_balloc.c optional ffs ufs/ffs/ffs_inode.c optional ffs ufs/ffs/ffs_snapshot.c optional ffs ufs/ffs/ffs_softdep.c optional ffs ufs/ffs/ffs_subr.c optional ffs ufs/ffs/ffs_tables.c optional ffs ufs/ffs/ffs_vfsops.c optional ffs ufs/ffs/ffs_vnops.c optional ffs ufs/ffs/ffs_rawread.c optional directio ufs/ffs/ffs_suspend.c optional ffs ufs/ufs/ufs_acl.c optional ffs ufs/ufs/ufs_bmap.c optional ffs ufs/ufs/ufs_dirhash.c optional ffs ufs/ufs/ufs_extattr.c optional ffs ufs/ufs/ufs_gjournal.c optional ffs UFS_GJOURNAL ufs/ufs/ufs_inode.c optional ffs ufs/ufs/ufs_lookup.c optional ffs ufs/ufs/ufs_quota.c optional ffs ufs/ufs/ufs_vfsops.c optional ffs ufs/ufs/ufs_vnops.c optional ffs vm/default_pager.c standard vm/device_pager.c standard vm/phys_pager.c standard vm/redzone.c optional DEBUG_REDZONE vm/sg_pager.c standard vm/swap_pager.c standard vm/uma_core.c standard vm/uma_dbg.c standard vm/vm_contig.c standard vm/memguard.c optional DEBUG_MEMGUARD vm/vm_fault.c standard vm/vm_glue.c standard vm/vm_init.c standard vm/vm_kern.c standard vm/vm_map.c standard vm/vm_meter.c standard vm/vm_mmap.c standard vm/vm_object.c standard vm/vm_page.c standard vm/vm_pageout.c standard vm/vm_pager.c standard vm/vm_phys.c standard vm/vm_reserv.c standard vm/vm_unix.c standard vm/vm_zeroidle.c standard vm/vnode_pager.c standard xdr/xdr.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd xdr/xdr_array.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd xdr/xdr_mbuf.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd xdr/xdr_mem.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd xdr/xdr_reference.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd xdr/xdr_sizeof.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd # gnu/fs/xfs/xfs_alloc.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" \ warning "kernel contains GPL contaminated xfs filesystem" gnu/fs/xfs/xfs_alloc_btree.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_bit.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_bmap.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_bmap_btree.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_btree.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_buf_item.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_da_btree.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2_block.c optional xfs \ compile-with "${NORMAL_C} ${NO_WARRAY_BOUNDS} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2_data.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2_leaf.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2_node.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2_sf.c optional xfs \ compile-with "${NORMAL_C} ${NO_WARRAY_BOUNDS} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir2_trace.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dir_leaf.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_error.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_extfree_item.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_fsops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_ialloc.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_ialloc_btree.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_inode.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_inode_item.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_iocore.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_itable.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dfrag.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_log.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_log_recover.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_mount.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_rename.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_trans.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_trans_ail.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_trans_buf.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_trans_extfree.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_trans_inode.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_trans_item.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_utils.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_vfsops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_vnodeops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_rw.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_attr_leaf.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_attr.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_dmops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_qmops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_iget.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_freebsd_iget.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_mountops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_vnops.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_frw.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_buf.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_globals.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_dmistubs.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_super.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_stats.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_vfs.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_vnode.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_sysctl.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_fs_subr.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/xfs_ioctl.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/support/debug.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/support/ktrace.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/support/mrlock.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/support/uuid.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/FreeBSD/support/kmem.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_iomap.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" gnu/fs/xfs/xfs_behavior.c optional xfs \ compile-with "${NORMAL_C} -I$S/gnu/fs/xfs/FreeBSD -I$S/gnu/fs/xfs/FreeBSD/support -I$S/gnu/fs/xfs" xen/gnttab.c optional xen | xenhvm xen/features.c optional xen | xenhvm xen/evtchn/evtchn.c optional xen xen/evtchn/evtchn_dev.c optional xen | xenhvm xen/xenbus/xenbus_if.m optional xen | xenhvm xen/xenbus/xenbus.c optional xen | xenhvm xen/xenbus/xenbusb_if.m optional xen | xenhvm xen/xenbus/xenbusb.c optional xen | xenhvm xen/xenbus/xenbusb_front.c optional xen | xenhvm xen/xenbus/xenbusb_back.c optional xen | xenhvm xen/xenstore/xenstore.c optional xen | xenhvm xen/xenstore/xenstore_dev.c optional xen | xenhvm dev/xen/balloon/balloon.c optional xen | xenhvm dev/xen/blkfront/blkfront.c optional xen | xenhvm dev/xen/blkback/blkback.c optional xen | xenhvm dev/xen/console/console.c optional xen dev/xen/console/xencons_ring.c optional xen dev/xen/control/control.c optional xen | xenhvm dev/xen/netback/netback.c optional xen | xenhvm dev/xen/netfront/netfront.c optional xen | xenhvm dev/xen/xenpci/xenpci.c optional xenpci dev/xen/xenpci/evtchn.c optional xenpci Index: stable/9/sys/conf =================================================================== --- stable/9/sys/conf (revision 273863) +++ stable/9/sys/conf (revision 273864) Property changes on: stable/9/sys/conf ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/sys/conf:r255932 Index: stable/9/sys/contrib/dev/npe/LICENSE =================================================================== --- stable/9/sys/contrib/dev/npe/LICENSE (revision 273863) +++ stable/9/sys/contrib/dev/npe/LICENSE (revision 273864) @@ -1,12 +1,12 @@ $FreeBSD$ Copyright (c) 2007, Intel Corporation. All rights reserved. Redistribution. Redistribution and use in binary form, without modification, are permitted provided that the following conditions are met: - Redistributions must reproduce the above copyright notice and the following disclaimer in the documentation and/or other materials provided with the distribution. - Neither the name of Intel Corporation nor the names of its suppliers may be used to endorse or promote products derived from this software without specific prior written permission. - No reverse engineering, decompilation, or disassembly of this software is permitted. -Limited patent license. Intel Corporation grants a world-wide, royalty-free, non-exclusive license under patents it now or hereafter owns or controls to make, have made, use, import, offer to sell and sell (¿Utilize¿) this software, but solely to the extent that any such patent is necessary to Utilize the software alone. The patent license shall not apply to any combinations which include this software. No hardware per se is licensed hereunder. +Limited patent license. Intel Corporation grants a world-wide, royalty-free, non-exclusive license under patents it now or hereafter owns or controls to make, have made, use, import, offer to sell and sell ("Utilize") this software, but solely to the extent that any such patent is necessary to Utilize the software alone. The patent license shall not apply to any combinations which include this software. No hardware per se is licensed hereunder. DISCLAIMER. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Index: stable/9/sys/contrib/dev/nve/amd64/nvenetlib.README =================================================================== --- stable/9/sys/contrib/dev/nve/amd64/nvenetlib.README (revision 273863) +++ stable/9/sys/contrib/dev/nve/amd64/nvenetlib.README (revision 273864) @@ -1,52 +1,52 @@ $FreeBSD$ The installation and use of this software is subject to the following license terms and conditions: License For Customer Use of NVIDIA Software -IMPORTANT NOTICE -- READ CAREFULLY: This License For Customer Use of NVIDIA Software ("LICENSE") is the agreement which governs use of the software of NVIDIA Corporation and its subsidiaries (“NVIDIA”) enclosed herewith, including computer software and associated printed materials ("SOFTWARE"). By downloading, installing, copying, or otherwise using the SOFTWARE, you agree to be bound by the terms of this LICENSE. If you do not agree to the terms of this LICENSE, do not download, install or use the SOFTWARE. +IMPORTANT NOTICE -- READ CAREFULLY: This License For Customer Use of NVIDIA Software ("LICENSE") is the agreement which governs use of the software of NVIDIA Corporation and its subsidiaries ("NVIDIA") enclosed herewith, including computer software and associated printed materials ("SOFTWARE"). By downloading, installing, copying, or otherwise using the SOFTWARE, you agree to be bound by the terms of this LICENSE. If you do not agree to the terms of this LICENSE, do not download, install or use the SOFTWARE. RECITALS Use of NVIDIA's products requires three elements: the SOFTWARE, the hardware on a computer motherboard, and a personal computer. The SOFTWARE is protected by copyright laws and international copyright treaties, as well as other intellectual property laws and treaties. The SOFTWARE is not sold, and instead is only licensed for use, strictly in accordance with this document. The hardware is protected by various patents, and is sold, but this agreement does not cover that sale, since it may not necessarily be sold as a package with the SOFTWARE. This agreement sets forth the terms and conditions of the SOFTWARE LICENSE only. 1. DEFINITIONS 1.1 Customer. Customer means the entity or individual that installs or uses the SOFTWARE. 2. GRANT OF LICENSE 2.1 Rights and Limitations of Grant. NVIDIA hereby grants Customer the following non-exclusive, non-transferable right to use the SOFTWARE, with the following limitations: 2.1.1 Rights. Customer may install and use one copy of the SOFTWARE on a single computer, and except for making one back-up copy of the Software, may not otherwise copy the SOFTWARE. This LICENSE of SOFTWARE may not be shared or used concurrently on different computers. 2.1.2 Linux/FreeBSD Exception. Notwithstanding the foregoing terms of Section 2.1.1, SOFTWARE designed exclusively for use on the Linux operating system may be copied and redistributed, provided that the binary files thereof are not modified in any way (except for uncompressing/compressing files). SOFTWARE designed exclusively for use on the Linux Operating system but which has been authorized by NVIDIA for use on the FreeBSD Operating System may also be copied and redistributed, provided that the binary files thereof are not modified in any way (except for unzipping of compressed files). 2.1.3 Limitations. No Reverse Engineering. Customer may not reverse engineer, decompile, or disassemble the SOFTWARE, nor attempt in any other manner to obtain the source code. No Separation of Components. The SOFTWARE is licensed as a single product. Its component parts may not be separated for use on more than one computer, nor otherwise used separately from the other parts. No Rental. Customer may not rent or lease the SOFTWARE to someone else. 3. TERMINATION This LICENSE will automatically terminate if Customer fails to comply with any of the terms and conditions hereof. In such event, Customer must destroy all copies of the SOFTWARE and all of its component parts. 4. COPYRIGHT All title and copyrights in and to the SOFTWARE (including but not limited to all images, photographs, animations, video, audio, music, text, and other information incorporated into the SOFTWARE), the accompanying printed materials, and any copies of the SOFTWARE, are owned by NVIDIA, or its suppliers. The SOFTWARE is protected by copyright laws and international treaty provisions. Accordingly, Customer is required to treat the SOFTWARE like any other copyrighted material, except as otherwise allowed pursuant to this LICENSE and that it may make one copy of the SOFTWARE solely for backup or archive purposes. 5. APPLICABLE LAW This agreement shall be deemed to have been made in, and shall be construed pursuant to, the laws of the State of California. 6. DISCLAIMER OF WARRANTIES AND LIMITATION ON LIABILITY 6.1 No Warranties. TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE SOFTWARE IS PROVIDED "AS IS" AND NVIDIA AND ITS SUPPLIERS DISCLAIM ALL WARRANTIES, EITHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. 6.2 No Liability for Consequential Damages. TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT SHALL NVIDIA OR ITS SUPPLIERS BE LIABLE FOR ANY SPECIAL, INCIDENTAL, INDIRECT, OR CONSEQUENTIAL DAMAGES WHATSOEVER (INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF BUSINESS INFORMATION, OR ANY OTHER PECUNIARY LOSS) ARISING OUT OF THE USE OF OR INABILITY TO USE THE SOFTWARE, EVEN IF NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 7. MISCELLANEOUS The United Nations Convention on Contracts for the International Sale of Goods is specifically disclaimed. If any provision of this LICENSE is inconsistent with, or cannot be fully enforced under, the law, such provision will be construed as limited to the extent necessary to be consistent with and fully enforceable under the law. This agreement is the final, complete and exclusive agreement between the parties relating to the subject matter hereof, and supersedes all prior or contemporaneous understandings and agreements relating to such subject matter, whether oral or written. Customer agrees that it will not ship, transfer or export the SOFTWARE into any country, or use the SOFTWARE in any manner, prohibited by the United States Bureau of Export Administration or any export laws, restrictions or regulations. This LICENSE may only be modified in writing signed by an authorized officer of NVIDIA. Index: stable/9/sys/contrib/dev/nve/i386/nvenetlib.README =================================================================== --- stable/9/sys/contrib/dev/nve/i386/nvenetlib.README (revision 273863) +++ stable/9/sys/contrib/dev/nve/i386/nvenetlib.README (revision 273864) @@ -1,52 +1,52 @@ $FreeBSD$ The installation and use of this software is subject to the following license terms and conditions: License For Customer Use of NVIDIA Software -IMPORTANT NOTICE -- READ CAREFULLY: This License For Customer Use of NVIDIA Software ("LICENSE") is the agreement which governs use of the software of NVIDIA Corporation and its subsidiaries (“NVIDIA”) enclosed herewith, including computer software and associated printed materials ("SOFTWARE"). By downloading, installing, copying, or otherwise using the SOFTWARE, you agree to be bound by the terms of this LICENSE. If you do not agree to the terms of this LICENSE, do not download, install or use the SOFTWARE. +IMPORTANT NOTICE -- READ CAREFULLY: This License For Customer Use of NVIDIA Software ("LICENSE") is the agreement which governs use of the software of NVIDIA Corporation and its subsidiaries ("NVIDIA") enclosed herewith, including computer software and associated printed materials ("SOFTWARE"). By downloading, installing, copying, or otherwise using the SOFTWARE, you agree to be bound by the terms of this LICENSE. If you do not agree to the terms of this LICENSE, do not download, install or use the SOFTWARE. RECITALS Use of NVIDIA's products requires three elements: the SOFTWARE, the hardware on a computer motherboard, and a personal computer. The SOFTWARE is protected by copyright laws and international copyright treaties, as well as other intellectual property laws and treaties. The SOFTWARE is not sold, and instead is only licensed for use, strictly in accordance with this document. The hardware is protected by various patents, and is sold, but this agreement does not cover that sale, since it may not necessarily be sold as a package with the SOFTWARE. This agreement sets forth the terms and conditions of the SOFTWARE LICENSE only. 1. DEFINITIONS 1.1 Customer. Customer means the entity or individual that installs or uses the SOFTWARE. 2. GRANT OF LICENSE 2.1 Rights and Limitations of Grant. NVIDIA hereby grants Customer the following non-exclusive, non-transferable right to use the SOFTWARE, with the following limitations: 2.1.1 Rights. Customer may install and use one copy of the SOFTWARE on a single computer, and except for making one back-up copy of the Software, may not otherwise copy the SOFTWARE. This LICENSE of SOFTWARE may not be shared or used concurrently on different computers. 2.1.2 Linux/FreeBSD Exception. Notwithstanding the foregoing terms of Section 2.1.1, SOFTWARE designed exclusively for use on the Linux operating system may be copied and redistributed, provided that the binary files thereof are not modified in any way (except for uncompressing/compressing files). SOFTWARE designed exclusively for use on the Linux Operating system but which has been authorized by NVIDIA for use on the FreeBSD Operating System may also be copied and redistributed, provided that the binary files thereof are not modified in any way (except for unzipping of compressed files). 2.1.3 Limitations. No Reverse Engineering. Customer may not reverse engineer, decompile, or disassemble the SOFTWARE, nor attempt in any other manner to obtain the source code. No Separation of Components. The SOFTWARE is licensed as a single product. Its component parts may not be separated for use on more than one computer, nor otherwise used separately from the other parts. No Rental. Customer may not rent or lease the SOFTWARE to someone else. 3. TERMINATION This LICENSE will automatically terminate if Customer fails to comply with any of the terms and conditions hereof. In such event, Customer must destroy all copies of the SOFTWARE and all of its component parts. 4. COPYRIGHT All title and copyrights in and to the SOFTWARE (including but not limited to all images, photographs, animations, video, audio, music, text, and other information incorporated into the SOFTWARE), the accompanying printed materials, and any copies of the SOFTWARE, are owned by NVIDIA, or its suppliers. The SOFTWARE is protected by copyright laws and international treaty provisions. Accordingly, Customer is required to treat the SOFTWARE like any other copyrighted material, except as otherwise allowed pursuant to this LICENSE and that it may make one copy of the SOFTWARE solely for backup or archive purposes. 5. APPLICABLE LAW This agreement shall be deemed to have been made in, and shall be construed pursuant to, the laws of the State of California. 6. DISCLAIMER OF WARRANTIES AND LIMITATION ON LIABILITY 6.1 No Warranties. TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE SOFTWARE IS PROVIDED "AS IS" AND NVIDIA AND ITS SUPPLIERS DISCLAIM ALL WARRANTIES, EITHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. 6.2 No Liability for Consequential Damages. TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT SHALL NVIDIA OR ITS SUPPLIERS BE LIABLE FOR ANY SPECIAL, INCIDENTAL, INDIRECT, OR CONSEQUENTIAL DAMAGES WHATSOEVER (INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF BUSINESS INFORMATION, OR ANY OTHER PECUNIARY LOSS) ARISING OUT OF THE USE OF OR INABILITY TO USE THE SOFTWARE, EVEN IF NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 7. MISCELLANEOUS The United Nations Convention on Contracts for the International Sale of Goods is specifically disclaimed. If any provision of this LICENSE is inconsistent with, or cannot be fully enforced under, the law, such provision will be construed as limited to the extent necessary to be consistent with and fully enforceable under the law. This agreement is the final, complete and exclusive agreement between the parties relating to the subject matter hereof, and supersedes all prior or contemporaneous understandings and agreements relating to such subject matter, whether oral or written. Customer agrees that it will not ship, transfer or export the SOFTWARE into any country, or use the SOFTWARE in any manner, prohibited by the United States Bureau of Export Administration or any export laws, restrictions or regulations. This LICENSE may only be modified in writing signed by an authorized officer of NVIDIA. Index: stable/9/sys/contrib/rdma/ib_umem.h =================================================================== --- stable/9/sys/contrib/rdma/ib_umem.h (revision 273863) +++ stable/9/sys/contrib/rdma/ib_umem.h (revision 273864) @@ -1,81 +1,125 @@ /* * Copyright (c) 2007 Cisco Systems. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $FreeBSD$ */ #ifndef IB_UMEM_H #define IB_UMEM_H +#include +#include +#include +#include + struct ib_ucontext; -struct ib_umem_chunk { - TAILQ_ENTRY(ib_umem_chunk) entry; - int nents; - int nmap; - struct rdma_scatterlist page_list[0]; -}; - struct ib_umem { struct ib_ucontext *context; size_t length; int offset; int page_size; int writable; - TAILQ_HEAD(, ib_umem_chunk) chunk_list; -#ifdef notyet + int hugetlb; + struct list_head chunk_list; struct work_struct work; struct mm_struct *mm; -#endif unsigned long diff; }; +/* contiguous memory structure */ +struct ib_cmem { + + struct ib_ucontext *context; + size_t length; + /* Link list of contiguous blocks being part of that cmem */ + struct list_head ib_cmem_block; + + /* Order of cmem block, 2^ block_order will equal number + of physical pages per block + */ + unsigned long block_order; + /* Refernce counter for that memory area + - When value became 0 pages will be returned to the kernel. + */ + struct kref refcount; +}; + + +struct ib_cmem_block { + + struct list_head list; + /* page will point to the page struct of the head page + in the current compound page. + block order is saved once as part of ib_cmem. + */ + struct page *page; +}; + + + +struct ib_umem_chunk { + struct list_head list; + int nents; + int nmap; + struct dma_attrs attrs; + struct scatterlist page_list[0]; +}; + #ifdef CONFIG_INFINIBAND_USER_MEM struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, - size_t size, int access); + size_t size, int access, int dmasync); void ib_umem_release(struct ib_umem *umem); int ib_umem_page_count(struct ib_umem *umem); +int ib_cmem_map_contiguous_pages_to_vma(struct ib_cmem *ib_cmem, + struct vm_area_struct *vma); +struct ib_cmem *ib_cmem_alloc_contiguous_pages(struct ib_ucontext *context, + unsigned long total_size, + unsigned long page_size_order); +void ib_cmem_release_contiguous_pages(struct ib_cmem *cmem); +int ib_umem_map_to_vma(struct ib_umem *umem, + struct vm_area_struct *vma); + #else /* CONFIG_INFINIBAND_USER_MEM */ +#include static inline struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, size_t size, - int access) { - return ERR_PTR(EINVAL); + int access, int dmasync) { + return ERR_PTR(-EINVAL); } static inline void ib_umem_release(struct ib_umem *umem) { } static inline int ib_umem_page_count(struct ib_umem *umem) { return 0; } #endif /* CONFIG_INFINIBAND_USER_MEM */ #endif /* IB_UMEM_H */ Index: stable/9/sys/contrib/rdma/krping/getopt.c =================================================================== --- stable/9/sys/contrib/rdma/krping/getopt.c (revision 273863) +++ stable/9/sys/contrib/rdma/krping/getopt.c (revision 273864) @@ -1,77 +1,78 @@ /* * lifted from fs/ncpfs/getopt.c * */ #include __FBSDID("$FreeBSD$"); -#include -#include -#include +#include +#include +#include + #include "getopt.h" /** * krping_getopt - option parser * @caller: name of the caller, for error messages * @options: the options string * @opts: an array of &struct option entries controlling parser operations * @optopt: output; will contain the current option * @optarg: output; will contain the value (if one exists) * @flag: output; may be NULL; should point to a long for or'ing flags * @value: output; may be NULL; will be overwritten with the integer value * of the current argument. * * Helper to parse options on the format used by mount ("a=b,c=d,e,f"). * Returns opts->val if a matching entry in the 'opts' array is found, * 0 when no more tokens are found, -1 if an error is encountered. */ int krping_getopt(const char *caller, char **options, const struct krping_option *opts, char **optopt, char **optarg, unsigned long *value) { char *token; char *val; do { if ((token = strsep(options, ",")) == NULL) return 0; } while (*token == '\0'); if (optopt) *optopt = token; if ((val = strchr (token, '=')) != NULL) { *val++ = 0; } *optarg = val; for (; opts->name; opts++) { if (!strcmp(opts->name, token)) { if (!val) { if (opts->has_arg & OPT_NOPARAM) { return opts->val; } - printf("%s: the %s option requires " + printk(KERN_INFO "%s: the %s option requires " "an argument\n", caller, token); return -EINVAL; } if (opts->has_arg & OPT_INT) { char* v; - *value = strtoul(val, &v, 0); + *value = simple_strtoul(val, &v, 0); if (!*v) { return opts->val; } - printf("%s: invalid numeric value " + printk(KERN_INFO "%s: invalid numeric value " "in %s=%s\n", caller, token, val); return -EDOM; } if (opts->has_arg & OPT_STRING) { return opts->val; } - printf("%s: unexpected argument %s to the " + printk(KERN_INFO "%s: unexpected argument %s to the " "%s option\n", caller, val, token); return -EINVAL; } } - printf("%s: Unrecognized option %s\n", caller, token); + printk(KERN_INFO "%s: Unrecognized option %s\n", caller, token); return -EOPNOTSUPP; } Index: stable/9/sys/contrib/rdma/krping/krping.c =================================================================== --- stable/9/sys/contrib/rdma/krping/krping.c (revision 273863) +++ stable/9/sys/contrib/rdma/krping/krping.c (revision 273864) @@ -1,1792 +1,2328 @@ /* * Copyright (c) 2005 Ammasso, Inc. All rights reserved. - * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved. + * Copyright (c) 2006-2009 Open Grid Computing, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include __FBSDID("$FreeBSD$"); -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include +#include #include -#include "getopt.h" #include "krping.h" +#include "getopt.h" -#define PFX "krping: " +extern int krping_debug; +#define DEBUG_LOG(cb, x...) if (krping_debug) krping_printf((cb)->cookie, x) +#define PRINTF(cb, x...) krping_printf((cb)->cookie, x) -static int debug = 0; -#define DEBUG_LOG if (debug) printf +MODULE_AUTHOR("Steve Wise"); +MODULE_DESCRIPTION("RDMA ping client/server"); +MODULE_LICENSE("Dual BSD/GPL"); +static __inline uint64_t +get_cycles(void) +{ + uint32_t low, high; + __asm __volatile("rdtsc" : "=a" (low), "=d" (high)); + return (low | ((u_int64_t)high << 32)); +} + +typedef uint64_t cycles_t; + +enum mem_type { + DMA = 1, + FASTREG = 2, + MW = 3, + MR = 4 +}; + static const struct krping_option krping_opts[] = { {"count", OPT_INT, 'C'}, {"size", OPT_INT, 'S'}, {"addr", OPT_STRING, 'a'}, {"port", OPT_INT, 'p'}, {"verbose", OPT_NOPARAM, 'v'}, {"validate", OPT_NOPARAM, 'V'}, {"server", OPT_NOPARAM, 's'}, {"client", OPT_NOPARAM, 'c'}, - {"dmamr", OPT_NOPARAM, 'D'}, - {"debug", OPT_NOPARAM, 'd'}, - {"wlat", OPT_NOPARAM, 'l'}, - {"rlat", OPT_NOPARAM, 'L'}, - {"bw", OPT_NOPARAM, 'B'}, - {"tx-depth", OPT_INT, 't'}, - {"poll", OPT_NOPARAM, 'P'}, - {"memlimit", OPT_INT, 'm'}, + {"mem_mode", OPT_STRING, 'm'}, + {"server_inv", OPT_NOPARAM, 'I'}, + {"wlat", OPT_NOPARAM, 'l'}, + {"rlat", OPT_NOPARAM, 'L'}, + {"bw", OPT_NOPARAM, 'B'}, + {"duplex", OPT_NOPARAM, 'd'}, + {"txdepth", OPT_INT, 'T'}, + {"poll", OPT_NOPARAM, 'P'}, + {"local_dma_lkey", OPT_NOPARAM, 'Z'}, + {"read_inv", OPT_NOPARAM, 'R'}, + {"fr", OPT_NOPARAM, 'f'}, {NULL, 0, 0} }; -struct mtx krping_mutex; +#define htonll(x) cpu_to_be64((x)) +#define ntohll(x) cpu_to_be64((x)) +static struct mutex krping_mutex; + /* * List of running krping threads. */ -struct krping_cb_list krping_cbs; +static LIST_HEAD(krping_cbs); /* * krping "ping/pong" loop: * client sends source rkey/addr/len * server receives source rkey/add/len * server rdma reads "ping" data from source * server sends "go ahead" on rdma read completion * client sends sink rkey/addr/len * server receives sink rkey/addr/len * server rdma writes "pong" data to sink * server sends "go ahead" on rdma write completion * */ /* + * These states are used to signal events between the completion handler + * and the main client or server thread. + * + * Once CONNECTED, they cycle through RDMA_READ_ADV, RDMA_WRITE_ADV, + * and RDMA_WRITE_COMPLETE for each ping. + */ +enum test_state { + IDLE = 1, + CONNECT_REQUEST, + ADDR_RESOLVED, + ROUTE_RESOLVED, + CONNECTED, + RDMA_READ_ADV, + RDMA_READ_COMPLETE, + RDMA_WRITE_ADV, + RDMA_WRITE_COMPLETE, + ERROR +}; + +struct krping_rdma_info { + uint64_t buf; + uint32_t rkey; + uint32_t size; +}; + +/* * Default max buffer size for IO... */ #define RPING_BUFSIZE 128*1024 -#define RPING_SQ_DEPTH 32 +#define RPING_SQ_DEPTH 64 -static void krping_wait(struct krping_cb *cb, int state) -{ - int rc; - mtx_lock(&cb->lock); - while (cb->state < state) { - rc = msleep(cb, &cb->lock, 0, "krping", 0); - if (rc && rc != ERESTART) { - cb->state = ERROR; - break; - } - } - mtx_unlock(&cb->lock); -} +/* + * Control block struct. + */ +struct krping_cb { + void *cookie; + int server; /* 0 iff client */ + struct ib_cq *cq; + struct ib_pd *pd; + struct ib_qp *qp; + enum mem_type mem; + struct ib_mr *dma_mr; + + struct ib_fast_reg_page_list *page_list; + int page_list_len; + struct ib_send_wr fastreg_wr; + struct ib_send_wr invalidate_wr; + struct ib_mr *fastreg_mr; + int server_invalidate; + int read_inv; + u8 key; + + struct ib_mw *mw; + struct ib_mw_bind bind_attr; + + struct ib_recv_wr rq_wr; /* recv work request record */ + struct ib_sge recv_sgl; /* recv single SGE */ + struct krping_rdma_info recv_buf;/* malloc'd buffer */ + u64 recv_dma_addr; + DECLARE_PCI_UNMAP_ADDR(recv_mapping) + struct ib_mr *recv_mr; + + struct ib_send_wr sq_wr; /* send work requrest record */ + struct ib_sge send_sgl; + struct krping_rdma_info send_buf;/* single send buf */ + u64 send_dma_addr; + DECLARE_PCI_UNMAP_ADDR(send_mapping) + struct ib_mr *send_mr; + + struct ib_send_wr rdma_sq_wr; /* rdma work request record */ + struct ib_sge rdma_sgl; /* rdma single SGE */ + char *rdma_buf; /* used as rdma sink */ + u64 rdma_dma_addr; + DECLARE_PCI_UNMAP_ADDR(rdma_mapping) + struct ib_mr *rdma_mr; + + uint32_t remote_rkey; /* remote guys RKEY */ + uint64_t remote_addr; /* remote guys TO */ + uint32_t remote_len; /* remote guys LEN */ + + char *start_buf; /* rdma read src */ + u64 start_dma_addr; + DECLARE_PCI_UNMAP_ADDR(start_mapping) + struct ib_mr *start_mr; + + enum test_state state; /* used for cond/signalling */ + wait_queue_head_t sem; + struct krping_stats stats; + + uint16_t port; /* dst port in NBO */ + struct in_addr addr; /* dst addr in NBO */ + char *addr_str; /* dst addr string */ + int verbose; /* verbose logging */ + int count; /* ping count */ + int size; /* ping data size */ + int validate; /* validate ping data */ + int wlat; /* run wlat test */ + int rlat; /* run rlat test */ + int bw; /* run bw test */ + int duplex; /* run bw full duplex test */ + int poll; /* poll or block for rlat test */ + int txdepth; /* SQ depth */ + int local_dma_lkey; /* use 0 for lkey */ + int frtest; /* fastreg test */ + + /* CM stuff */ + struct rdma_cm_id *cm_id; /* connection on client side,*/ + /* listener on server side. */ + struct rdma_cm_id *child_cm_id; /* connection on server side */ + struct list_head list; +}; + static int krping_cma_event_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) { int ret; struct krping_cb *cb = cma_id->context; - DEBUG_LOG(PFX "cma_event type %d cma_id %p (%s)\n", event->event, cma_id, - (cma_id == cb->cm_id) ? "parent" : "child"); + DEBUG_LOG(cb, "cma_event type %d cma_id %p (%s)\n", event->event, + cma_id, (cma_id == cb->cm_id) ? "parent" : "child"); - mtx_lock(&cb->lock); switch (event->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: cb->state = ADDR_RESOLVED; ret = rdma_resolve_route(cma_id, 2000); if (ret) { - log(LOG_ERR, "rdma_resolve_route error %d\n", - ret); - wakeup(cb); + PRINTF(cb, "rdma_resolve_route error %d\n", ret); + wake_up_interruptible(&cb->sem); } break; case RDMA_CM_EVENT_ROUTE_RESOLVED: cb->state = ROUTE_RESOLVED; - wakeup(cb); + wake_up_interruptible(&cb->sem); break; case RDMA_CM_EVENT_CONNECT_REQUEST: cb->state = CONNECT_REQUEST; cb->child_cm_id = cma_id; - DEBUG_LOG(PFX "child cma %p\n", cb->child_cm_id); - wakeup(cb); + DEBUG_LOG(cb, "child cma %p\n", cb->child_cm_id); + wake_up_interruptible(&cb->sem); break; case RDMA_CM_EVENT_ESTABLISHED: - DEBUG_LOG(PFX "ESTABLISHED\n"); + DEBUG_LOG(cb, "ESTABLISHED\n"); if (!cb->server) { cb->state = CONNECTED; - wakeup(cb); } + wake_up_interruptible(&cb->sem); break; case RDMA_CM_EVENT_ADDR_ERROR: case RDMA_CM_EVENT_ROUTE_ERROR: case RDMA_CM_EVENT_CONNECT_ERROR: case RDMA_CM_EVENT_UNREACHABLE: case RDMA_CM_EVENT_REJECTED: - log(LOG_ERR, "cma event %d, error %d\n", event->event, + PRINTF(cb, "cma event %d, error %d\n", event->event, event->status); cb->state = ERROR; - wakeup(cb); + wake_up_interruptible(&cb->sem); break; case RDMA_CM_EVENT_DISCONNECTED: - DEBUG_LOG(PFX "DISCONNECT EVENT...\n"); + PRINTF(cb, "DISCONNECT EVENT...\n"); cb->state = ERROR; - wakeup(cb); + wake_up_interruptible(&cb->sem); break; case RDMA_CM_EVENT_DEVICE_REMOVAL: - DEBUG_LOG(PFX "cma detected device removal!!!!\n"); + PRINTF(cb, "cma detected device removal!!!!\n"); break; default: - log(LOG_ERR, "oof bad type!\n"); - wakeup(cb); + PRINTF(cb, "oof bad type!\n"); + wake_up_interruptible(&cb->sem); break; } - mtx_unlock(&cb->lock); return 0; } static int server_recv(struct krping_cb *cb, struct ib_wc *wc) { if (wc->byte_len != sizeof(cb->recv_buf)) { - log(LOG_ERR, "Received bogus data, size %d\n", + PRINTF(cb, "Received bogus data, size %d\n", wc->byte_len); return -1; } cb->remote_rkey = ntohl(cb->recv_buf.rkey); cb->remote_addr = ntohll(cb->recv_buf.buf); cb->remote_len = ntohl(cb->recv_buf.size); - DEBUG_LOG(PFX "Received rkey %x addr %llx len %d from peer\n", + DEBUG_LOG(cb, "Received rkey %x addr %llx len %d from peer\n", cb->remote_rkey, (unsigned long long)cb->remote_addr, cb->remote_len); if (cb->state <= CONNECTED || cb->state == RDMA_WRITE_COMPLETE) cb->state = RDMA_READ_ADV; else cb->state = RDMA_WRITE_ADV; return 0; } static int client_recv(struct krping_cb *cb, struct ib_wc *wc) { if (wc->byte_len != sizeof(cb->recv_buf)) { - log(LOG_ERR, "Received bogus data, size %d\n", + PRINTF(cb, "Received bogus data, size %d\n", wc->byte_len); return -1; } if (cb->state == RDMA_READ_ADV) cb->state = RDMA_WRITE_ADV; else cb->state = RDMA_WRITE_COMPLETE; return 0; } static void krping_cq_event_handler(struct ib_cq *cq, void *ctx) { struct krping_cb *cb = ctx; struct ib_wc wc; struct ib_recv_wr *bad_wr; int ret; - mtx_lock(&cb->lock); - KASSERT(cb->cq == cq, ("bad condition")); + BUG_ON(cb->cq != cq); if (cb->state == ERROR) { - log(LOG_ERR, "cq completion in ERROR state\n"); - mtx_unlock(&cb->lock); + PRINTF(cb, "cq completion in ERROR state\n"); return; } + if (cb->frtest) { + PRINTF(cb, "cq completion event in frtest!\n"); + return; + } if (!cb->wlat && !cb->rlat && !cb->bw) ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) { if (wc.status) { if (wc.status == IB_WC_WR_FLUSH_ERR) { - DEBUG_LOG("cq flushed\n"); + DEBUG_LOG(cb, "cq flushed\n"); continue; } else { - log(LOG_CRIT, "cq completion failed status %d\n", - wc.status); + PRINTF(cb, "cq completion failed with " + "wr_id %Lx status %d opcode %d vender_err %x\n", + wc.wr_id, wc.status, wc.opcode, wc.vendor_err); goto error; } } switch (wc.opcode) { case IB_WC_SEND: - DEBUG_LOG(PFX "send completion\n"); + DEBUG_LOG(cb, "send completion\n"); cb->stats.send_bytes += cb->send_sgl.length; cb->stats.send_msgs++; break; case IB_WC_RDMA_WRITE: - DEBUG_LOG(PFX "rdma write completion\n"); + DEBUG_LOG(cb, "rdma write completion\n"); cb->stats.write_bytes += cb->rdma_sq_wr.sg_list->length; cb->stats.write_msgs++; cb->state = RDMA_WRITE_COMPLETE; - wakeup(cb); + wake_up_interruptible(&cb->sem); break; case IB_WC_RDMA_READ: - DEBUG_LOG(PFX "rdma read completion\n"); + DEBUG_LOG(cb, "rdma read completion\n"); cb->stats.read_bytes += cb->rdma_sq_wr.sg_list->length; cb->stats.read_msgs++; cb->state = RDMA_READ_COMPLETE; - wakeup(cb); + wake_up_interruptible(&cb->sem); break; case IB_WC_RECV: - DEBUG_LOG(PFX "recv completion\n"); + DEBUG_LOG(cb, "recv completion\n"); cb->stats.recv_bytes += sizeof(cb->recv_buf); cb->stats.recv_msgs++; if (cb->wlat || cb->rlat || cb->bw) ret = server_recv(cb, &wc); else ret = cb->server ? server_recv(cb, &wc) : - client_recv(cb, &wc); + client_recv(cb, &wc); if (ret) { - log(LOG_ERR, "recv wc error: %d\n", ret); + PRINTF(cb, "recv wc error: %d\n", ret); goto error; } ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "post recv error: %d\n", + PRINTF(cb, "post recv error: %d\n", ret); goto error; } - wakeup(cb); + wake_up_interruptible(&cb->sem); break; default: - log(LOG_ERR, "unknown!!!!! completion\n"); + PRINTF(cb, + "%s:%d Unexpected opcode %d, Shutting down\n", + __func__, __LINE__, wc.opcode); goto error; } } if (ret) { - log(LOG_ERR, "poll error %d\n", ret); + PRINTF(cb, "poll error %d\n", ret); goto error; } - mtx_unlock(&cb->lock); return; error: cb->state = ERROR; - wakeup(cb); - mtx_unlock(&cb->lock); + wake_up_interruptible(&cb->sem); } static int krping_accept(struct krping_cb *cb) { struct rdma_conn_param conn_param; int ret; - DEBUG_LOG(PFX "accepting client connection request\n"); + DEBUG_LOG(cb, "accepting client connection request\n"); memset(&conn_param, 0, sizeof conn_param); conn_param.responder_resources = 1; conn_param.initiator_depth = 1; ret = rdma_accept(cb->child_cm_id, &conn_param); if (ret) { - log(LOG_ERR, "rdma_accept error: %d\n", ret); + PRINTF(cb, "rdma_accept error: %d\n", ret); return ret; } if (!cb->wlat && !cb->rlat && !cb->bw) { - krping_wait(cb, CONNECTED); + wait_event_interruptible(cb->sem, cb->state >= CONNECTED); if (cb->state == ERROR) { - log(LOG_ERR, "wait for CONNECTED state %d\n", cb->state); + PRINTF(cb, "wait for CONNECTED state %d\n", + cb->state); return -1; } } return 0; } static void krping_setup_wr(struct krping_cb *cb) { - /* XXX X86 only here... not mapping for dma! */ - cb->recv_sgl.addr = vtophys(&cb->recv_buf); + cb->recv_sgl.addr = cb->recv_dma_addr; cb->recv_sgl.length = sizeof cb->recv_buf; - if (cb->use_dmamr) + if (cb->local_dma_lkey) + cb->recv_sgl.lkey = cb->qp->device->local_dma_lkey; + else if (cb->mem == DMA) cb->recv_sgl.lkey = cb->dma_mr->lkey; else cb->recv_sgl.lkey = cb->recv_mr->lkey; cb->rq_wr.sg_list = &cb->recv_sgl; cb->rq_wr.num_sge = 1; - cb->send_sgl.addr = vtophys(&cb->send_buf); + cb->send_sgl.addr = cb->send_dma_addr; cb->send_sgl.length = sizeof cb->send_buf; - if (cb->use_dmamr) + if (cb->local_dma_lkey) + cb->send_sgl.lkey = cb->qp->device->local_dma_lkey; + else if (cb->mem == DMA) cb->send_sgl.lkey = cb->dma_mr->lkey; else cb->send_sgl.lkey = cb->send_mr->lkey; cb->sq_wr.opcode = IB_WR_SEND; cb->sq_wr.send_flags = IB_SEND_SIGNALED; cb->sq_wr.sg_list = &cb->send_sgl; cb->sq_wr.num_sge = 1; - cb->rdma_addr = vtophys(cb->rdma_buf); - cb->rdma_sgl.addr = cb->rdma_addr; - if (cb->use_dmamr) - cb->rdma_sgl.lkey = cb->dma_mr->lkey; - else - cb->rdma_sgl.lkey = cb->rdma_mr->lkey; - cb->rdma_sq_wr.send_flags = IB_SEND_SIGNALED; - cb->rdma_sq_wr.sg_list = &cb->rdma_sgl; - cb->rdma_sq_wr.num_sge = 1; + if (cb->server || cb->wlat || cb->rlat || cb->bw) { + cb->rdma_sgl.addr = cb->rdma_dma_addr; + if (cb->mem == MR) + cb->rdma_sgl.lkey = cb->rdma_mr->lkey; + cb->rdma_sq_wr.send_flags = IB_SEND_SIGNALED; + cb->rdma_sq_wr.sg_list = &cb->rdma_sgl; + cb->rdma_sq_wr.num_sge = 1; + } - if (!cb->server || cb->wlat || cb->rlat || cb->bw) { - cb->start_addr = vtophys(cb->start_buf); + switch(cb->mem) { + case FASTREG: + + /* + * A chain of 2 WRs, INVALDATE_MR + FAST_REG_MR. + * both unsignaled. The client uses them to reregister + * the rdma buffers with a new key each iteration. + */ + cb->fastreg_wr.opcode = IB_WR_FAST_REG_MR; + cb->fastreg_wr.wr.fast_reg.page_shift = PAGE_SHIFT; + cb->fastreg_wr.wr.fast_reg.length = cb->size; + cb->fastreg_wr.wr.fast_reg.page_list = cb->page_list; + cb->fastreg_wr.wr.fast_reg.page_list_len = cb->page_list_len; + + cb->invalidate_wr.next = &cb->fastreg_wr; + cb->invalidate_wr.opcode = IB_WR_LOCAL_INV; + break; + case MW: + cb->bind_attr.wr_id = 0xabbaabba; + cb->bind_attr.send_flags = 0; /* unsignaled */ + cb->bind_attr.length = cb->size; + break; + default: + break; } } static int krping_setup_buffers(struct krping_cb *cb) { int ret; struct ib_phys_buf buf; u64 iovbase; - DEBUG_LOG(PFX "krping_setup_buffers called on cb %p\n", cb); + DEBUG_LOG(cb, "krping_setup_buffers called on cb %p\n", cb); - if (cb->use_dmamr) { + cb->recv_dma_addr = dma_map_single(cb->pd->device->dma_device, + &cb->recv_buf, + sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); + pci_unmap_addr_set(cb, recv_mapping, cb->recv_dma_addr); + cb->send_dma_addr = dma_map_single(cb->pd->device->dma_device, + &cb->send_buf, sizeof(cb->send_buf), + DMA_BIDIRECTIONAL); + pci_unmap_addr_set(cb, send_mapping, cb->send_dma_addr); + + if (cb->mem == DMA) { cb->dma_mr = ib_get_dma_mr(cb->pd, IB_ACCESS_LOCAL_WRITE| IB_ACCESS_REMOTE_READ| IB_ACCESS_REMOTE_WRITE); if (IS_ERR(cb->dma_mr)) { - log(LOG_ERR, "reg_dmamr failed\n"); - return PTR_ERR(cb->dma_mr); + DEBUG_LOG(cb, "reg_dmamr failed\n"); + ret = PTR_ERR(cb->dma_mr); + goto bail; } } else { + if (!cb->local_dma_lkey) { + buf.addr = cb->recv_dma_addr; + buf.size = sizeof cb->recv_buf; + DEBUG_LOG(cb, "recv buf dma_addr %llx size %d\n", buf.addr, + (int)buf.size); + iovbase = cb->recv_dma_addr; + cb->recv_mr = ib_reg_phys_mr(cb->pd, &buf, 1, + IB_ACCESS_LOCAL_WRITE, + &iovbase); - buf.addr = vtophys(&cb->recv_buf); - buf.size = sizeof cb->recv_buf; - iovbase = vtophys(&cb->recv_buf); - cb->recv_mr = ib_reg_phys_mr(cb->pd, &buf, 1, - IB_ACCESS_LOCAL_WRITE, - &iovbase); + if (IS_ERR(cb->recv_mr)) { + DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); + ret = PTR_ERR(cb->recv_mr); + goto bail; + } - if (IS_ERR(cb->recv_mr)) { - log(LOG_ERR, "recv_buf reg_mr failed\n"); - return PTR_ERR(cb->recv_mr); - } + buf.addr = cb->send_dma_addr; + buf.size = sizeof cb->send_buf; + DEBUG_LOG(cb, "send buf dma_addr %llx size %d\n", buf.addr, + (int)buf.size); + iovbase = cb->send_dma_addr; + cb->send_mr = ib_reg_phys_mr(cb->pd, &buf, 1, + 0, &iovbase); - buf.addr = vtophys(&cb->send_buf); - buf.size = sizeof cb->send_buf; - iovbase = vtophys(&cb->send_buf); - cb->send_mr = ib_reg_phys_mr(cb->pd, &buf, 1, - 0, &iovbase); - - if (IS_ERR(cb->send_mr)) { - log(LOG_ERR, "send_buf reg_mr failed\n"); - ib_dereg_mr(cb->recv_mr); - return PTR_ERR(cb->send_mr); + if (IS_ERR(cb->send_mr)) { + DEBUG_LOG(cb, "send_buf reg_mr failed\n"); + ret = PTR_ERR(cb->send_mr); + goto bail; + } } } - /* RNIC adapters have a limit upto which it can register physical memory - * If DMA-MR memory mode is set then normally driver registers maximum - * supported memory. After that if contigmalloc allocates memory beyond the - * specified RNIC limit then Krping may not work. - */ - if (cb->use_dmamr && cb->memlimit) - cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, cb->memlimit, - PAGE_SIZE, 0); - else - cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, -1UL, - PAGE_SIZE, 0); - + cb->rdma_buf = kmalloc(cb->size, GFP_KERNEL); if (!cb->rdma_buf) { - log(LOG_ERR, "rdma_buf malloc failed\n"); - ret = ENOMEM; - goto err1; + DEBUG_LOG(cb, "rdma_buf malloc failed\n"); + ret = -ENOMEM; + goto bail; } - if (!cb->use_dmamr) { - buf.addr = vtophys(cb->rdma_buf); - buf.size = cb->size; - iovbase = vtophys(cb->rdma_buf); - cb->rdma_mr = ib_reg_phys_mr(cb->pd, &buf, 1, + cb->rdma_dma_addr = dma_map_single(cb->pd->device->dma_device, + cb->rdma_buf, cb->size, + DMA_BIDIRECTIONAL); + pci_unmap_addr_set(cb, rdma_mapping, cb->rdma_dma_addr); + if (cb->mem != DMA) { + switch (cb->mem) { + case FASTREG: + cb->page_list_len = (((cb->size - 1) & PAGE_MASK) + + PAGE_SIZE) >> PAGE_SHIFT; + cb->page_list = ib_alloc_fast_reg_page_list( + cb->pd->device, + cb->page_list_len); + if (IS_ERR(cb->page_list)) { + DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); + ret = PTR_ERR(cb->page_list); + goto bail; + } + cb->fastreg_mr = ib_alloc_fast_reg_mr(cb->pd, + cb->page_list->max_page_list_len); + if (IS_ERR(cb->fastreg_mr)) { + DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); + ret = PTR_ERR(cb->fastreg_mr); + goto bail; + } + DEBUG_LOG(cb, "fastreg rkey 0x%x page_list %p" + " page_list_len %u\n", cb->fastreg_mr->rkey, + cb->page_list, cb->page_list_len); + break; + case MW: + cb->mw = ib_alloc_mw(cb->pd); + if (IS_ERR(cb->mw)) { + DEBUG_LOG(cb, "recv_buf alloc_mw failed\n"); + ret = PTR_ERR(cb->mw); + goto bail; + } + DEBUG_LOG(cb, "mw rkey 0x%x\n", cb->mw->rkey); + /*FALLTHROUGH*/ + case MR: + buf.addr = cb->rdma_dma_addr; + buf.size = cb->size; + iovbase = cb->rdma_dma_addr; + cb->rdma_mr = ib_reg_phys_mr(cb->pd, &buf, 1, IB_ACCESS_REMOTE_READ| IB_ACCESS_REMOTE_WRITE, &iovbase); - - if (IS_ERR(cb->rdma_mr)) { - log(LOG_ERR, "rdma_buf reg_mr failed\n"); - ret = PTR_ERR(cb->rdma_mr); - goto err2; + if (IS_ERR(cb->rdma_mr)) { + DEBUG_LOG(cb, "rdma_buf reg_mr failed\n"); + ret = PTR_ERR(cb->rdma_mr); + goto bail; + } + DEBUG_LOG(cb, "rdma buf dma_addr %llx size %d mr rkey 0x%x\n", + buf.addr, (int)buf.size, cb->rdma_mr->rkey); + break; + default: + ret = -EINVAL; + goto bail; + break; } } if (!cb->server || cb->wlat || cb->rlat || cb->bw) { - if (cb->use_dmamr && cb->memlimit) - cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, - 0, cb->memlimit, PAGE_SIZE, 0); - else - cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, - 0, -1UL, PAGE_SIZE, 0); + + cb->start_buf = kmalloc(cb->size, GFP_KERNEL); if (!cb->start_buf) { - log(LOG_ERR, "start_buf malloc failed\n"); - ret = ENOMEM; - goto err2; + DEBUG_LOG(cb, "start_buf malloc failed\n"); + ret = -ENOMEM; + goto bail; } - if (!cb->use_dmamr) { + + cb->start_dma_addr = dma_map_single(cb->pd->device->dma_device, + cb->start_buf, cb->size, + DMA_BIDIRECTIONAL); + pci_unmap_addr_set(cb, start_mapping, cb->start_dma_addr); + + if (cb->mem == MR || cb->mem == MW) { unsigned flags = IB_ACCESS_REMOTE_READ; - if (cb->wlat || cb->rlat || cb->bw) + if (cb->wlat || cb->rlat || cb->bw) flags |= IB_ACCESS_REMOTE_WRITE; - buf.addr = vtophys(cb->start_buf); + + buf.addr = cb->start_dma_addr; buf.size = cb->size; - iovbase = vtophys(cb->start_buf); + DEBUG_LOG(cb, "start buf dma_addr %llx size %d\n", + buf.addr, (int)buf.size); + iovbase = cb->start_dma_addr; cb->start_mr = ib_reg_phys_mr(cb->pd, &buf, 1, flags, &iovbase); if (IS_ERR(cb->start_mr)) { - log(LOG_ERR, "start_buf reg_mr failed\n"); + DEBUG_LOG(cb, "start_buf reg_mr failed\n"); ret = PTR_ERR(cb->start_mr); - goto err3; + goto bail; } } } krping_setup_wr(cb); - DEBUG_LOG(PFX "allocated & registered buffers...\n"); + DEBUG_LOG(cb, "allocated & registered buffers...\n"); return 0; -err3: - contigfree(cb->start_buf, cb->size, M_DEVBUF); - - if (!cb->use_dmamr) +bail: + if (cb->fastreg_mr && !IS_ERR(cb->fastreg_mr)) + ib_dereg_mr(cb->fastreg_mr); + if (cb->mw && !IS_ERR(cb->mw)) + ib_dealloc_mw(cb->mw); + if (cb->rdma_mr && !IS_ERR(cb->rdma_mr)) ib_dereg_mr(cb->rdma_mr); -err2: - contigfree(cb->rdma_buf, cb->size, M_DEVBUF); -err1: - if (cb->use_dmamr) + if (cb->page_list && !IS_ERR(cb->page_list)) + ib_free_fast_reg_page_list(cb->page_list); + if (cb->dma_mr && !IS_ERR(cb->dma_mr)) ib_dereg_mr(cb->dma_mr); - else { + if (cb->recv_mr && !IS_ERR(cb->recv_mr)) ib_dereg_mr(cb->recv_mr); + if (cb->send_mr && !IS_ERR(cb->send_mr)) ib_dereg_mr(cb->send_mr); - } + if (cb->rdma_buf) + kfree(cb->rdma_buf); + if (cb->start_buf) + kfree(cb->start_buf); return ret; } static void krping_free_buffers(struct krping_cb *cb) { - DEBUG_LOG(PFX "krping_free_buffers called on cb %p\n", cb); + DEBUG_LOG(cb, "krping_free_buffers called on cb %p\n", cb); -#if 0 + if (cb->dma_mr) + ib_dereg_mr(cb->dma_mr); + if (cb->send_mr) + ib_dereg_mr(cb->send_mr); + if (cb->recv_mr) + ib_dereg_mr(cb->recv_mr); + if (cb->rdma_mr) + ib_dereg_mr(cb->rdma_mr); + if (cb->start_mr) + ib_dereg_mr(cb->start_mr); + if (cb->fastreg_mr) + ib_dereg_mr(cb->fastreg_mr); + if (cb->mw) + ib_dealloc_mw(cb->mw); + dma_unmap_single(cb->pd->device->dma_device, pci_unmap_addr(cb, recv_mapping), sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); dma_unmap_single(cb->pd->device->dma_device, pci_unmap_addr(cb, send_mapping), sizeof(cb->send_buf), DMA_BIDIRECTIONAL); dma_unmap_single(cb->pd->device->dma_device, pci_unmap_addr(cb, rdma_mapping), cb->size, DMA_BIDIRECTIONAL); -#endif - contigfree(cb->rdma_buf, cb->size, M_DEVBUF); - if (!cb->server || cb->wlat || cb->rlat || cb->bw) { -#if 0 + kfree(cb->rdma_buf); + if (cb->start_buf) { dma_unmap_single(cb->pd->device->dma_device, pci_unmap_addr(cb, start_mapping), cb->size, DMA_BIDIRECTIONAL); -#endif - contigfree(cb->start_buf, cb->size, M_DEVBUF); + kfree(cb->start_buf); } - if (cb->use_dmamr) - ib_dereg_mr(cb->dma_mr); - else { - ib_dereg_mr(cb->send_mr); - ib_dereg_mr(cb->recv_mr); - ib_dereg_mr(cb->rdma_mr); - if (!cb->server) - ib_dereg_mr(cb->start_mr); - } } static int krping_create_qp(struct krping_cb *cb) { struct ib_qp_init_attr init_attr; int ret; memset(&init_attr, 0, sizeof(init_attr)); init_attr.cap.max_send_wr = cb->txdepth; init_attr.cap.max_recv_wr = 2; init_attr.cap.max_recv_sge = 1; init_attr.cap.max_send_sge = 1; init_attr.qp_type = IB_QPT_RC; init_attr.send_cq = cb->cq; init_attr.recv_cq = cb->cq; + init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; if (cb->server) { ret = rdma_create_qp(cb->child_cm_id, cb->pd, &init_attr); if (!ret) cb->qp = cb->child_cm_id->qp; } else { ret = rdma_create_qp(cb->cm_id, cb->pd, &init_attr); if (!ret) cb->qp = cb->cm_id->qp; } return ret; } static void krping_free_qp(struct krping_cb *cb) { ib_destroy_qp(cb->qp); ib_destroy_cq(cb->cq); ib_dealloc_pd(cb->pd); } static int krping_setup_qp(struct krping_cb *cb, struct rdma_cm_id *cm_id) { int ret; cb->pd = ib_alloc_pd(cm_id->device); if (IS_ERR(cb->pd)) { - log(LOG_ERR, "ib_alloc_pd failed\n"); + PRINTF(cb, "ib_alloc_pd failed\n"); return PTR_ERR(cb->pd); } - DEBUG_LOG(PFX "created pd %p\n", cb->pd); + DEBUG_LOG(cb, "created pd %p\n", cb->pd); + strlcpy(cb->stats.name, cb->pd->device->name, sizeof(cb->stats.name)); + cb->cq = ib_create_cq(cm_id->device, krping_cq_event_handler, NULL, cb, cb->txdepth * 2, 0); if (IS_ERR(cb->cq)) { - log(LOG_ERR, "ib_create_cq failed\n"); + PRINTF(cb, "ib_create_cq failed\n"); ret = PTR_ERR(cb->cq); goto err1; } - DEBUG_LOG(PFX "created cq %p\n", cb->cq); + DEBUG_LOG(cb, "created cq %p\n", cb->cq); - if (!cb->wlat && !cb->rlat && !cb->bw) { + if (!cb->wlat && !cb->rlat && !cb->bw && !cb->frtest) { ret = ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); if (ret) { - log(LOG_ERR, "ib_create_cq failed\n"); + PRINTF(cb, "ib_create_cq failed\n"); goto err2; } } ret = krping_create_qp(cb); if (ret) { - log(LOG_ERR, "krping_create_qp failed: %d\n", ret); + PRINTF(cb, "krping_create_qp failed: %d\n", ret); goto err2; } - DEBUG_LOG(PFX "created qp %p\n", cb->qp); + DEBUG_LOG(cb, "created qp %p\n", cb->qp); return 0; err2: ib_destroy_cq(cb->cq); err1: ib_dealloc_pd(cb->pd); return ret; } -static void krping_format_send(struct krping_cb *cb, u64 buf, - struct ib_mr *mr) +/* + * return the (possibly rebound) rkey for the rdma buffer. + * FASTREG mode: invalidate and rebind via fastreg wr. + * MW mode: rebind the MW. + * other modes: just return the mr rkey. + */ +static u32 krping_rdma_rkey(struct krping_cb *cb, u64 buf, int post_inv) { - struct krping_rdma_info *info = &cb->send_buf; + u32 rkey = 0xffffffff; + u64 p; + struct ib_send_wr *bad_wr; + int i; + int ret; - info->buf = htonll(buf); - info->rkey = htonl(mr->rkey); - info->size = htonl(cb->size); + switch (cb->mem) { + case FASTREG: + cb->invalidate_wr.ex.invalidate_rkey = cb->fastreg_mr->rkey; - DEBUG_LOG(PFX "RDMA addr %llx rkey %x len %d\n", - (unsigned long long)buf, mr->rkey, cb->size); + /* + * Update the fastreg key. + */ + ib_update_fast_reg_key(cb->fastreg_mr, ++cb->key); + cb->fastreg_wr.wr.fast_reg.rkey = cb->fastreg_mr->rkey; + + /* + * Update the fastreg WR with new buf info. + */ + if (buf == (u64)cb->start_dma_addr) + cb->fastreg_wr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_READ; + else + cb->fastreg_wr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; + cb->fastreg_wr.wr.fast_reg.iova_start = buf; + p = (u64)(buf & PAGE_MASK); + for (i=0; i < cb->fastreg_wr.wr.fast_reg.page_list_len; + i++, p += PAGE_SIZE) { + cb->page_list->page_list[i] = p; + DEBUG_LOG(cb, "page_list[%d] 0x%llx\n", i, p); + } + + DEBUG_LOG(cb, "post_inv = %d, fastreg new rkey 0x%x shift %u len %u" + " iova_start %llx page_list_len %u\n", + post_inv, + cb->fastreg_wr.wr.fast_reg.rkey, + cb->fastreg_wr.wr.fast_reg.page_shift, + cb->fastreg_wr.wr.fast_reg.length, + cb->fastreg_wr.wr.fast_reg.iova_start, + cb->fastreg_wr.wr.fast_reg.page_list_len); + + if (post_inv) + ret = ib_post_send(cb->qp, &cb->invalidate_wr, &bad_wr); + else + ret = ib_post_send(cb->qp, &cb->fastreg_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + cb->state = ERROR; + } + rkey = cb->fastreg_mr->rkey; + break; + case MW: + /* + * Update the MW with new buf info. + */ + if (buf == (u64)cb->start_dma_addr) { + cb->bind_attr.mw_access_flags = IB_ACCESS_REMOTE_READ; + cb->bind_attr.mr = cb->start_mr; + } else { + cb->bind_attr.mw_access_flags = IB_ACCESS_REMOTE_WRITE; + cb->bind_attr.mr = cb->rdma_mr; + } + cb->bind_attr.addr = buf; + DEBUG_LOG(cb, "binding mw rkey 0x%x to buf %llx mr rkey 0x%x\n", + cb->mw->rkey, buf, cb->bind_attr.mr->rkey); + ret = ib_bind_mw(cb->qp, cb->mw, &cb->bind_attr); + if (ret) { + PRINTF(cb, "bind mw error %d\n", ret); + cb->state = ERROR; + } else + rkey = cb->mw->rkey; + break; + case MR: + if (buf == (u64)cb->start_dma_addr) + rkey = cb->start_mr->rkey; + else + rkey = cb->rdma_mr->rkey; + break; + case DMA: + rkey = cb->dma_mr->rkey; + break; + default: + PRINTF(cb, "%s:%d case ERROR\n", __func__, __LINE__); + cb->state = ERROR; + break; + } + return rkey; } +static void krping_format_send(struct krping_cb *cb, u64 buf) +{ + struct krping_rdma_info *info = &cb->send_buf; + u32 rkey; + + /* + * Client side will do fastreg or mw bind before + * advertising the rdma buffer. Server side + * sends have no data. + */ + if (!cb->server || cb->wlat || cb->rlat || cb->bw) { + rkey = krping_rdma_rkey(cb, buf, !cb->server_invalidate); + info->buf = htonll(buf); + info->rkey = htonl(rkey); + info->size = htonl(cb->size); + DEBUG_LOG(cb, "RDMA addr %llx rkey %x len %d\n", + (unsigned long long)buf, rkey, cb->size); + } +} + static void krping_test_server(struct krping_cb *cb) { - struct ib_send_wr *bad_wr; + struct ib_send_wr *bad_wr, inv; int ret; while (1) { /* Wait for client's Start STAG/TO/Len */ - krping_wait(cb, RDMA_READ_ADV); + wait_event_interruptible(cb->sem, cb->state >= RDMA_READ_ADV); if (cb->state != RDMA_READ_ADV) { - DEBUG_LOG(PFX "wait for RDMA_READ_ADV state %d\n", + PRINTF(cb, "wait for RDMA_READ_ADV state %d\n", cb->state); break; } - DEBUG_LOG(PFX "server received sink adv\n"); + DEBUG_LOG(cb, "server received sink adv\n"); - /* Issue RDMA Read. */ - cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ; cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; cb->rdma_sq_wr.sg_list->length = cb->remote_len; + cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, 1); + cb->rdma_sq_wr.next = NULL; + /* Issue RDMA Read. */ + if (cb->read_inv) + cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ_WITH_INV; + else { + + cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ; + if (cb->mem == FASTREG) { + /* + * Immediately follow the read with a + * fenced LOCAL_INV. + */ + cb->rdma_sq_wr.next = &inv; + memset(&inv, 0, sizeof inv); + inv.opcode = IB_WR_LOCAL_INV; + inv.ex.invalidate_rkey = cb->fastreg_mr->rkey; + inv.send_flags = IB_SEND_FENCE; + } + } + ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "post send error %d\n", ret); + PRINTF(cb, "post send error %d\n", ret); break; } - DEBUG_LOG(PFX "server posted rdma read req \n"); + cb->rdma_sq_wr.next = NULL; + DEBUG_LOG(cb, "server posted rdma read req \n"); + /* Wait for read completion */ - krping_wait(cb, RDMA_READ_COMPLETE); + wait_event_interruptible(cb->sem, + cb->state >= RDMA_READ_COMPLETE); if (cb->state != RDMA_READ_COMPLETE) { - log(LOG_ERR, + PRINTF(cb, "wait for RDMA_READ_COMPLETE state %d\n", cb->state); break; } - DEBUG_LOG(PFX "server received read complete\n"); + DEBUG_LOG(cb, "server received read complete\n"); /* Display data in recv buf */ if (cb->verbose) - DEBUG_LOG("server ping data: %s\n", cb->rdma_buf); + PRINTF(cb, "server ping data: %s\n", + cb->rdma_buf); /* Tell client to continue */ + if (cb->server && cb->server_invalidate) { + cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; + cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; + DEBUG_LOG(cb, "send-w-inv rkey 0x%x\n", cb->remote_rkey); + } ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "post send error %d\n", ret); + PRINTF(cb, "post send error %d\n", ret); break; } - DEBUG_LOG(PFX "server posted go ahead\n"); + DEBUG_LOG(cb, "server posted go ahead\n"); /* Wait for client's RDMA STAG/TO/Len */ - krping_wait(cb, RDMA_WRITE_ADV); + wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); if (cb->state != RDMA_WRITE_ADV) { - log(LOG_ERR, + PRINTF(cb, "wait for RDMA_WRITE_ADV state %d\n", cb->state); break; } - DEBUG_LOG(PFX "server received sink adv\n"); + DEBUG_LOG(cb, "server received sink adv\n"); /* RDMA Write echo data */ cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; cb->rdma_sq_wr.sg_list->length = strlen(cb->rdma_buf) + 1; - DEBUG_LOG(PFX "rdma write from lkey %x laddr %llx len %d\n", + if (cb->local_dma_lkey) + cb->rdma_sgl.lkey = cb->qp->device->local_dma_lkey; + else + cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, 0); + + DEBUG_LOG(cb, "rdma write from lkey %x laddr %llx len %d\n", cb->rdma_sq_wr.sg_list->lkey, (unsigned long long)cb->rdma_sq_wr.sg_list->addr, cb->rdma_sq_wr.sg_list->length); ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "post send error %d\n", ret); + PRINTF(cb, "post send error %d\n", ret); break; } /* Wait for completion */ - krping_wait(cb, RDMA_WRITE_COMPLETE); + ret = wait_event_interruptible(cb->sem, cb->state >= + RDMA_WRITE_COMPLETE); if (cb->state != RDMA_WRITE_COMPLETE) { - log(LOG_ERR, + PRINTF(cb, "wait for RDMA_WRITE_COMPLETE state %d\n", cb->state); break; } - DEBUG_LOG(PFX "server rdma write complete \n"); + DEBUG_LOG(cb, "server rdma write complete \n"); cb->state = CONNECTED; /* Tell client to begin again */ + if (cb->server && cb->server_invalidate) { + cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; + cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; + DEBUG_LOG(cb, "send-w-inv rkey 0x%x\n", cb->remote_rkey); + } ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "post send error %d\n", ret); + PRINTF(cb, "post send error %d\n", ret); break; } - DEBUG_LOG(PFX "server posted go ahead\n"); + DEBUG_LOG(cb, "server posted go ahead\n"); } } static void rlat_test(struct krping_cb *cb) { int scnt; int iters = cb->count; struct timeval start_tv, stop_tv; int ret; struct ib_wc wc; struct ib_send_wr *bad_wr; int ne; scnt = 0; cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ; cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; cb->rdma_sq_wr.sg_list->length = cb->size; microtime(&start_tv); - if (!cb->poll) { - cb->state = RDMA_READ_ADV; - ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); - } + if (!cb->poll) { + cb->state = RDMA_READ_ADV; + ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); + } while (scnt < iters) { - cb->state = RDMA_READ_ADV; + cb->state = RDMA_READ_ADV; ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); if (ret) { - log(LOG_ERR, + PRINTF(cb, "Couldn't post send: ret=%d scnt %d\n", ret, scnt); return; } do { if (!cb->poll) { - krping_wait(cb, RDMA_READ_COMPLETE); + wait_event_interruptible(cb->sem, + cb->state != RDMA_READ_ADV); if (cb->state == RDMA_READ_COMPLETE) { ne = 1; - ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); + ib_req_notify_cq(cb->cq, + IB_CQ_NEXT_COMP); } else { ne = -1; } } else ne = ib_poll_cq(cb->cq, 1, &wc); if (cb->state == ERROR) { - log(LOG_ERR, - "state == ERROR...bailing scnt %d\n", scnt); + PRINTF(cb, + "state == ERROR...bailing scnt %d\n", + scnt); return; } } while (ne == 0); if (ne < 0) { - log(LOG_ERR, "poll CQ failed %d\n", ne); + PRINTF(cb, "poll CQ failed %d\n", ne); return; } - if (cb->poll && wc.status != IB_WC_SUCCESS) { - log(LOG_ERR, "Completion wth error at %s:\n", + if (cb->poll && wc.status != IB_WC_SUCCESS) { + PRINTF(cb, "Completion wth error at %s:\n", cb->server ? "server" : "client"); - log(LOG_ERR, "Failed status %d: wr_id %d\n", + PRINTF(cb, "Failed status %d: wr_id %d\n", wc.status, (int) wc.wr_id); return; } ++scnt; } microtime(&stop_tv); if (stop_tv.tv_usec < start_tv.tv_usec) { stop_tv.tv_usec += 1000000; stop_tv.tv_sec -= 1; } - log(LOG_ERR, "delta sec %zu delta usec %lu iter %d size %d\n", + PRINTF(cb, "delta sec %lu delta usec %lu iter %d size %d\n", stop_tv.tv_sec - start_tv.tv_sec, stop_tv.tv_usec - start_tv.tv_usec, scnt, cb->size); } -static int alloc_cycle_mem(int cycle_iters, - cycles_t **post_cycles_start, - cycles_t **post_cycles_stop, - cycles_t **poll_cycles_start, - cycles_t **poll_cycles_stop, - cycles_t **last_poll_cycles_start) -{ - *post_cycles_start = malloc(cycle_iters * sizeof(cycles_t), M_DEVBUF, M_WAITOK); - if (!*post_cycles_start) { - goto fail1; - } - *post_cycles_stop = malloc(cycle_iters * sizeof(cycles_t), M_DEVBUF, M_WAITOK); - if (!*post_cycles_stop) { - goto fail2; - } - *poll_cycles_start = malloc(cycle_iters * sizeof(cycles_t), M_DEVBUF, M_WAITOK); - if (!*poll_cycles_start) { - goto fail3; - } - *poll_cycles_stop = malloc(cycle_iters * sizeof(cycles_t), M_DEVBUF, M_WAITOK); - if (!*poll_cycles_stop) { - goto fail4; - } - *last_poll_cycles_start = malloc(cycle_iters * sizeof(cycles_t), M_DEVBUF, M_WAITOK); - if (!*last_poll_cycles_start) { - goto fail5; - } - return 0; -fail5: - free(*poll_cycles_stop, M_DEVBUF); -fail4: - free(*poll_cycles_start, M_DEVBUF); -fail3: - free(*post_cycles_stop, M_DEVBUF); -fail2: - free(*post_cycles_start, M_DEVBUF); -fail1: - log(LOG_ERR, "%s malloc failed\n", __FUNCTION__); - return ENOMEM; -} - -static void free_cycle_mem(cycles_t *post_cycles_start, - cycles_t *post_cycles_stop, - cycles_t *poll_cycles_start, - cycles_t *poll_cycles_stop, - cycles_t *last_poll_cycles_start) -{ - free(last_poll_cycles_start, M_DEVBUF); - free(poll_cycles_stop, M_DEVBUF); - free(poll_cycles_start, M_DEVBUF); - free(post_cycles_stop, M_DEVBUF); - free(post_cycles_start, M_DEVBUF); -} - static void wlat_test(struct krping_cb *cb) { int ccnt, scnt, rcnt; int iters=cb->count; volatile char *poll_buf = (char *) cb->start_buf; char *buf = (char *)cb->rdma_buf; - ccnt = 0; - scnt = 0; - rcnt = 0; struct timeval start_tv, stop_tv; cycles_t *post_cycles_start, *post_cycles_stop; cycles_t *poll_cycles_start, *poll_cycles_stop; cycles_t *last_poll_cycles_start; cycles_t sum_poll = 0, sum_post = 0, sum_last_poll = 0; int i; int cycle_iters = 1000; - int err; - err = alloc_cycle_mem(cycle_iters, &post_cycles_start, &post_cycles_stop, - &poll_cycles_start, &poll_cycles_stop, &last_poll_cycles_start); - - if (err) { - log(LOG_ERR, "%s malloc failed\n", __FUNCTION__); + ccnt = 0; + scnt = 0; + rcnt = 0; + + post_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!post_cycles_start) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); return; } - + post_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!post_cycles_stop) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!poll_cycles_start) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + poll_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!poll_cycles_stop) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + last_poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), + GFP_KERNEL); + if (!last_poll_cycles_start) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; cb->rdma_sq_wr.sg_list->length = cb->size; if (cycle_iters > iters) cycle_iters = iters; microtime(&start_tv); while (scnt < iters || ccnt < iters || rcnt < iters) { /* Wait till buffer changes. */ if (rcnt < iters && !(scnt < 1 && !cb->server)) { ++rcnt; while (*poll_buf != (char)rcnt) { if (cb->state == ERROR) { - log(LOG_ERR, "state = ERROR, bailing\n"); + PRINTF(cb, + "state = ERROR, bailing\n"); return; } } } if (scnt < iters) { struct ib_send_wr *bad_wr; *buf = (char)scnt+1; if (scnt < cycle_iters) post_cycles_start[scnt] = get_cycles(); if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { - log(LOG_ERR, "Couldn't post send: scnt=%d\n", + PRINTF(cb, + "Couldn't post send: scnt=%d\n", scnt); return; } if (scnt < cycle_iters) post_cycles_stop[scnt] = get_cycles(); scnt++; } if (ccnt < iters) { struct ib_wc wc; int ne; if (ccnt < cycle_iters) poll_cycles_start[ccnt] = get_cycles(); do { if (ccnt < cycle_iters) - last_poll_cycles_start[ccnt] = get_cycles(); + last_poll_cycles_start[ccnt] = + get_cycles(); ne = ib_poll_cq(cb->cq, 1, &wc); } while (ne == 0); if (ccnt < cycle_iters) poll_cycles_stop[ccnt] = get_cycles(); ++ccnt; if (ne < 0) { - log(LOG_ERR, "poll CQ failed %d\n", ne); + PRINTF(cb, "poll CQ failed %d\n", ne); return; } if (wc.status != IB_WC_SUCCESS) { - log(LOG_ERR, "Completion wth error at %s:\n", + PRINTF(cb, + "Completion wth error at %s:\n", cb->server ? "server" : "client"); - log(LOG_ERR, "Failed status %d: wr_id %d\n", + PRINTF(cb, + "Failed status %d: wr_id %d\n", wc.status, (int) wc.wr_id); - log(LOG_ERR, "scnt=%d, rcnt=%d, ccnt=%d\n", + PRINTF(cb, + "scnt=%d, rcnt=%d, ccnt=%d\n", scnt, rcnt, ccnt); return; } } } microtime(&stop_tv); if (stop_tv.tv_usec < start_tv.tv_usec) { stop_tv.tv_usec += 1000000; stop_tv.tv_sec -= 1; } for (i=0; i < cycle_iters; i++) { sum_post += post_cycles_stop[i] - post_cycles_start[i]; sum_poll += poll_cycles_stop[i] - poll_cycles_start[i]; - sum_last_poll += poll_cycles_stop[i] - last_poll_cycles_start[i]; + sum_last_poll += poll_cycles_stop[i]-last_poll_cycles_start[i]; } - - log(LOG_ERR, "delta sec %zu delta usec %lu iter %d size %d cycle_iters %d sum_post %llu sum_poll %llu sum_last_poll %llu\n", + PRINTF(cb, + "delta sec %lu delta usec %lu iter %d size %d cycle_iters %d" + " sum_post %llu sum_poll %llu sum_last_poll %llu\n", stop_tv.tv_sec - start_tv.tv_sec, stop_tv.tv_usec - start_tv.tv_usec, scnt, cb->size, cycle_iters, (unsigned long long)sum_post, (unsigned long long)sum_poll, (unsigned long long)sum_last_poll); - - free_cycle_mem(post_cycles_start, post_cycles_stop, poll_cycles_start, - poll_cycles_stop, last_poll_cycles_start); + kfree(post_cycles_start); + kfree(post_cycles_stop); + kfree(poll_cycles_start); + kfree(poll_cycles_stop); + kfree(last_poll_cycles_start); } static void bw_test(struct krping_cb *cb) { int ccnt, scnt, rcnt; int iters=cb->count; - ccnt = 0; - scnt = 0; - rcnt = 0; struct timeval start_tv, stop_tv; cycles_t *post_cycles_start, *post_cycles_stop; cycles_t *poll_cycles_start, *poll_cycles_stop; cycles_t *last_poll_cycles_start; cycles_t sum_poll = 0, sum_post = 0, sum_last_poll = 0; int i; int cycle_iters = 1000; - int err; - err = alloc_cycle_mem(cycle_iters, &post_cycles_start, &post_cycles_stop, - &poll_cycles_start, &poll_cycles_stop, &last_poll_cycles_start); - - if (err) { - log(LOG_ERR, "%s kmalloc failed\n", __FUNCTION__); + ccnt = 0; + scnt = 0; + rcnt = 0; + + post_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!post_cycles_start) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); return; } - + post_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!post_cycles_stop) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!poll_cycles_start) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + poll_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!poll_cycles_stop) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + last_poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), + GFP_KERNEL); + if (!last_poll_cycles_start) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; cb->rdma_sq_wr.sg_list->length = cb->size; if (cycle_iters > iters) cycle_iters = iters; microtime(&start_tv); while (scnt < iters || ccnt < iters) { while (scnt < iters && scnt - ccnt < cb->txdepth) { struct ib_send_wr *bad_wr; if (scnt < cycle_iters) post_cycles_start[scnt] = get_cycles(); if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { - log(LOG_ERR, "Couldn't post send: scnt=%d\n", + PRINTF(cb, + "Couldn't post send: scnt=%d\n", scnt); return; } if (scnt < cycle_iters) post_cycles_stop[scnt] = get_cycles(); ++scnt; } if (ccnt < iters) { int ne; struct ib_wc wc; if (ccnt < cycle_iters) poll_cycles_start[ccnt] = get_cycles(); do { if (ccnt < cycle_iters) - last_poll_cycles_start[ccnt] = get_cycles(); + last_poll_cycles_start[ccnt] = + get_cycles(); ne = ib_poll_cq(cb->cq, 1, &wc); } while (ne == 0); if (ccnt < cycle_iters) poll_cycles_stop[ccnt] = get_cycles(); ccnt += 1; if (ne < 0) { - log(LOG_ERR, "poll CQ failed %d\n", ne); + PRINTF(cb, "poll CQ failed %d\n", ne); return; } if (wc.status != IB_WC_SUCCESS) { - log(LOG_ERR, "Completion wth error at %s:\n", + PRINTF(cb, + "Completion wth error at %s:\n", cb->server ? "server" : "client"); - log(LOG_ERR, "Failed status %d: wr_id %d\n", + PRINTF(cb, + "Failed status %d: wr_id %d\n", wc.status, (int) wc.wr_id); return; } } } microtime(&stop_tv); if (stop_tv.tv_usec < start_tv.tv_usec) { stop_tv.tv_usec += 1000000; stop_tv.tv_sec -= 1; } for (i=0; i < cycle_iters; i++) { sum_post += post_cycles_stop[i] - post_cycles_start[i]; sum_poll += poll_cycles_stop[i] - poll_cycles_start[i]; - sum_last_poll += poll_cycles_stop[i] - last_poll_cycles_start[i]; + sum_last_poll += poll_cycles_stop[i]-last_poll_cycles_start[i]; } - - log(LOG_ERR, "delta sec %zu delta usec %lu iter %d size %d cycle_iters %d sum_post %llu sum_poll %llu sum_last_poll %llu\n", + PRINTF(cb, + "delta sec %lu delta usec %lu iter %d size %d cycle_iters %d" + " sum_post %llu sum_poll %llu sum_last_poll %llu\n", stop_tv.tv_sec - start_tv.tv_sec, stop_tv.tv_usec - start_tv.tv_usec, scnt, cb->size, cycle_iters, (unsigned long long)sum_post, (unsigned long long)sum_poll, (unsigned long long)sum_last_poll); - - free_cycle_mem(post_cycles_start, post_cycles_stop, poll_cycles_start, - poll_cycles_stop, last_poll_cycles_start); + kfree(post_cycles_start); + kfree(post_cycles_stop); + kfree(poll_cycles_start); + kfree(poll_cycles_stop); + kfree(last_poll_cycles_start); } static void krping_rlat_test_server(struct krping_cb *cb) { struct ib_send_wr *bad_wr; struct ib_wc wc; int ret; /* Spin waiting for client's Start STAG/TO/Len */ while (cb->state < RDMA_READ_ADV) { krping_cq_event_handler(cb->cq, cb); } /* Send STAG/TO/Len to client */ - if (cb->dma_mr) - krping_format_send(cb, cb->start_addr, cb->dma_mr); - else - krping_format_send(cb, cb->start_addr, cb->start_mr); + krping_format_send(cb, cb->start_dma_addr); ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "post send error %d\n", ret); + PRINTF(cb, "post send error %d\n", ret); return; } /* Spin waiting for send completion */ while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); if (ret < 0) { - log(LOG_ERR, "poll error %d\n", ret); + PRINTF(cb, "poll error %d\n", ret); return; } if (wc.status) { - log(LOG_ERR, "send completiong error %d\n", wc.status); + PRINTF(cb, "send completiong error %d\n", wc.status); return; } - krping_wait(cb, ERROR); + wait_event_interruptible(cb->sem, cb->state == ERROR); } static void krping_wlat_test_server(struct krping_cb *cb) { struct ib_send_wr *bad_wr; struct ib_wc wc; int ret; /* Spin waiting for client's Start STAG/TO/Len */ while (cb->state < RDMA_READ_ADV) { krping_cq_event_handler(cb->cq, cb); } /* Send STAG/TO/Len to client */ - if (cb->dma_mr) - krping_format_send(cb, cb->start_addr, cb->dma_mr); - else - krping_format_send(cb, cb->start_addr, cb->start_mr); + krping_format_send(cb, cb->start_dma_addr); ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "post send error %d\n", ret); + PRINTF(cb, "post send error %d\n", ret); return; } /* Spin waiting for send completion */ while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); if (ret < 0) { - log(LOG_ERR, "poll error %d\n", ret); + PRINTF(cb, "poll error %d\n", ret); return; } if (wc.status) { - log(LOG_ERR, "send completiong error %d\n", wc.status); + PRINTF(cb, "send completiong error %d\n", wc.status); return; } wlat_test(cb); - + wait_event_interruptible(cb->sem, cb->state == ERROR); } static void krping_bw_test_server(struct krping_cb *cb) { struct ib_send_wr *bad_wr; struct ib_wc wc; int ret; /* Spin waiting for client's Start STAG/TO/Len */ while (cb->state < RDMA_READ_ADV) { krping_cq_event_handler(cb->cq, cb); } /* Send STAG/TO/Len to client */ - if (cb->dma_mr) - krping_format_send(cb, cb->start_addr, cb->dma_mr); - else - krping_format_send(cb, cb->start_addr, cb->start_mr); + krping_format_send(cb, cb->start_dma_addr); ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "post send error %d\n", ret); + PRINTF(cb, "post send error %d\n", ret); return; } /* Spin waiting for send completion */ while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); if (ret < 0) { - log(LOG_ERR, "poll error %d\n", ret); + PRINTF(cb, "poll error %d\n", ret); return; } if (wc.status) { - log(LOG_ERR, "send completiong error %d\n", wc.status); + PRINTF(cb, "send completiong error %d\n", wc.status); return; } if (cb->duplex) bw_test(cb); - krping_wait(cb, ERROR); + wait_event_interruptible(cb->sem, cb->state == ERROR); } +static int fastreg_supported(struct krping_cb *cb) +{ + struct ib_device *dev = cb->child_cm_id->device; + struct ib_device_attr attr; + int ret; + + ret = ib_query_device(dev, &attr); + if (ret) { + PRINTF(cb, "ib_query_device failed ret %d\n", ret); + return 0; + } + if (!(attr.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) { + PRINTF(cb, "Fastreg not supported - device_cap_flags 0x%x\n", + attr.device_cap_flags); + return 0; + } + DEBUG_LOG(cb, "Fastreg supported - device_cap_flags 0x%x\n", + attr.device_cap_flags); + return 1; +} + static int krping_bind_server(struct krping_cb *cb) { struct sockaddr_in sin; int ret; memset(&sin, 0, sizeof(sin)); sin.sin_len = sizeof sin; sin.sin_family = AF_INET; sin.sin_addr.s_addr = cb->addr.s_addr; sin.sin_port = cb->port; ret = rdma_bind_addr(cb->cm_id, (struct sockaddr *) &sin); if (ret) { - log(LOG_ERR, "rdma_bind_addr error %d\n", ret); + PRINTF(cb, "rdma_bind_addr error %d\n", ret); return ret; } - DEBUG_LOG(PFX "rdma_bind_addr successful\n"); + DEBUG_LOG(cb, "rdma_bind_addr successful\n"); - DEBUG_LOG(PFX "rdma_listen\n"); + DEBUG_LOG(cb, "rdma_listen\n"); ret = rdma_listen(cb->cm_id, 3); if (ret) { - log(LOG_ERR, "rdma_listen failed: %d\n", ret); + PRINTF(cb, "rdma_listen failed: %d\n", ret); return ret; } - krping_wait(cb, CONNECT_REQUEST); + wait_event_interruptible(cb->sem, cb->state >= CONNECT_REQUEST); if (cb->state != CONNECT_REQUEST) { - log(LOG_ERR, "wait for CONNECT_REQUEST state %d\n", + PRINTF(cb, "wait for CONNECT_REQUEST state %d\n", cb->state); return -1; } + if (cb->mem == FASTREG && !fastreg_supported(cb)) + return -EINVAL; + return 0; } static void krping_run_server(struct krping_cb *cb) { struct ib_recv_wr *bad_wr; int ret; ret = krping_bind_server(cb); if (ret) return; ret = krping_setup_qp(cb, cb->child_cm_id); if (ret) { - log(LOG_ERR, "setup_qp failed: %d\n", ret); - return; + PRINTF(cb, "setup_qp failed: %d\n", ret); + goto err0; } ret = krping_setup_buffers(cb); if (ret) { - log(LOG_ERR, "krping_setup_buffers failed: %d\n", ret); + PRINTF(cb, "krping_setup_buffers failed: %d\n", ret); goto err1; } ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "ib_post_recv failed: %d\n", ret); + PRINTF(cb, "ib_post_recv failed: %d\n", ret); goto err2; } ret = krping_accept(cb); if (ret) { - log(LOG_ERR, "connect error %d\n", ret); + PRINTF(cb, "connect error %d\n", ret); goto err2; } if (cb->wlat) krping_wlat_test_server(cb); else if (cb->rlat) krping_rlat_test_server(cb); else if (cb->bw) krping_bw_test_server(cb); else krping_test_server(cb); - rdma_disconnect(cb->child_cm_id); - rdma_destroy_id(cb->child_cm_id); err2: krping_free_buffers(cb); err1: krping_free_qp(cb); +err0: + rdma_destroy_id(cb->child_cm_id); } static void krping_test_client(struct krping_cb *cb) { int ping, start, cc, i, ret; struct ib_send_wr *bad_wr; unsigned char c; start = 65; for (ping = 0; !cb->count || ping < cb->count; ping++) { cb->state = RDMA_READ_ADV; /* Put some ascii text in the buffer. */ cc = sprintf(cb->start_buf, "rdma-ping-%d: ", ping); for (i = cc, c = start; i < cb->size; i++) { cb->start_buf[i] = c; c++; if (c > 122) c = 65; } start++; if (start > 122) start = 65; cb->start_buf[cb->size - 1] = 0; - if (cb->dma_mr) - krping_format_send(cb, cb->start_addr, cb->dma_mr); - else - krping_format_send(cb, cb->start_addr, cb->start_mr); - + krping_format_send(cb, cb->start_dma_addr); + if (cb->state == ERROR) { + PRINTF(cb, "krping_format_send failed\n"); + break; + } ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "post send error %d\n", ret); + PRINTF(cb, "post send error %d\n", ret); break; } /* Wait for server to ACK */ - krping_wait(cb, RDMA_WRITE_ADV); + wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); if (cb->state != RDMA_WRITE_ADV) { - log(LOG_ERR, + PRINTF(cb, "wait for RDMA_WRITE_ADV state %d\n", cb->state); break; } - if (cb->dma_mr) - krping_format_send(cb, cb->rdma_addr, cb->dma_mr); - else - krping_format_send(cb, cb->rdma_addr, cb->rdma_mr); - + krping_format_send(cb, cb->rdma_dma_addr); ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "post send error %d\n", ret); + PRINTF(cb, "post send error %d\n", ret); break; } /* Wait for the server to say the RDMA Write is complete. */ - krping_wait(cb, RDMA_WRITE_COMPLETE); + wait_event_interruptible(cb->sem, + cb->state >= RDMA_WRITE_COMPLETE); if (cb->state != RDMA_WRITE_COMPLETE) { - log(LOG_ERR, + PRINTF(cb, "wait for RDMA_WRITE_COMPLETE state %d\n", cb->state); break; } if (cb->validate) if (memcmp(cb->start_buf, cb->rdma_buf, cb->size)) { - log(LOG_ERR, "data mismatch!\n"); + PRINTF(cb, "data mismatch!\n"); break; } if (cb->verbose) - DEBUG_LOG("ping data: %s\n", cb->rdma_buf); + PRINTF(cb, "ping data: %s\n", cb->rdma_buf); +#ifdef SLOW_KRPING + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); +#endif } } static void krping_rlat_test_client(struct krping_cb *cb) { struct ib_send_wr *bad_wr; struct ib_wc wc; int ret; cb->state = RDMA_READ_ADV; /* Send STAG/TO/Len to client */ - if (cb->dma_mr) - krping_format_send(cb, cb->start_addr, cb->dma_mr); - else - krping_format_send(cb, cb->start_addr, cb->rdma_mr); + krping_format_send(cb, cb->start_dma_addr); + if (cb->state == ERROR) { + PRINTF(cb, "krping_format_send failed\n"); + return; + } ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "post send error %d\n", ret); + PRINTF(cb, "post send error %d\n", ret); return; } /* Spin waiting for send completion */ while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); if (ret < 0) { - log(LOG_ERR, "poll error %d\n", ret); + PRINTF(cb, "poll error %d\n", ret); return; } if (wc.status) { - log(LOG_ERR, "send completion error %d\n", wc.status); + PRINTF(cb, "send completion error %d\n", wc.status); return; } /* Spin waiting for server's Start STAG/TO/Len */ while (cb->state < RDMA_WRITE_ADV) { krping_cq_event_handler(cb->cq, cb); } #if 0 { int i; struct timeval start, stop; time_t sec; suseconds_t usec; unsigned long long elapsed; struct ib_wc wc; struct ib_send_wr *bad_wr; int ne; cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; cb->rdma_sq_wr.sg_list->length = 0; cb->rdma_sq_wr.num_sge = 0; microtime(&start); for (i=0; i < 100000; i++) { if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { - log(LOG_ERR, "Couldn't post send\n"); + PRINTF(cb, "Couldn't post send\n"); return; } do { ne = ib_poll_cq(cb->cq, 1, &wc); } while (ne == 0); if (ne < 0) { - log(LOG_ERR, "poll CQ failed %d\n", ne); + PRINTF(cb, "poll CQ failed %d\n", ne); return; } if (wc.status != IB_WC_SUCCESS) { - log(LOG_ERR, "Completion wth error at %s:\n", + PRINTF(cb, "Completion wth error at %s:\n", cb->server ? "server" : "client"); - log(LOG_ERR, "Failed status %d: wr_id %d\n", + PRINTF(cb, "Failed status %d: wr_id %d\n", wc.status, (int) wc.wr_id); return; } } microtime(&stop); if (stop.tv_usec < start.tv_usec) { stop.tv_usec += 1000000; stop.tv_sec -= 1; } sec = stop.tv_sec - start.tv_sec; usec = stop.tv_usec - start.tv_usec; elapsed = sec * 1000000 + usec; - log(LOG_ERR, "0B-write-lat iters 100000 usec %llu\n", elapsed); + PRINTF(cb, "0B-write-lat iters 100000 usec %llu\n", elapsed); } #endif rlat_test(cb); } static void krping_wlat_test_client(struct krping_cb *cb) { struct ib_send_wr *bad_wr; struct ib_wc wc; int ret; cb->state = RDMA_READ_ADV; /* Send STAG/TO/Len to client */ - if (cb->dma_mr) - krping_format_send(cb, cb->start_addr, cb->dma_mr); - else - krping_format_send(cb, cb->start_addr, cb->start_mr); + krping_format_send(cb, cb->start_dma_addr); + if (cb->state == ERROR) { + PRINTF(cb, "krping_format_send failed\n"); + return; + } ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "post send error %d\n", ret); + PRINTF(cb, "post send error %d\n", ret); return; } /* Spin waiting for send completion */ while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); if (ret < 0) { - log(LOG_ERR, "poll error %d\n", ret); + PRINTF(cb, "poll error %d\n", ret); return; } if (wc.status) { - log(LOG_ERR, "send completion error %d\n", wc.status); + PRINTF(cb, "send completion error %d\n", wc.status); return; } /* Spin waiting for server's Start STAG/TO/Len */ while (cb->state < RDMA_WRITE_ADV) { krping_cq_event_handler(cb->cq, cb); } wlat_test(cb); } static void krping_bw_test_client(struct krping_cb *cb) { struct ib_send_wr *bad_wr; struct ib_wc wc; int ret; cb->state = RDMA_READ_ADV; /* Send STAG/TO/Len to client */ - if (cb->dma_mr) - krping_format_send(cb, cb->start_addr, cb->dma_mr); - else - krping_format_send(cb, cb->start_addr, cb->start_mr); + krping_format_send(cb, cb->start_dma_addr); + if (cb->state == ERROR) { + PRINTF(cb, "krping_format_send failed\n"); + return; + } ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "post send error %d\n", ret); + PRINTF(cb, "post send error %d\n", ret); return; } /* Spin waiting for send completion */ while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); if (ret < 0) { - log(LOG_ERR, "poll error %d\n", ret); + PRINTF(cb, "poll error %d\n", ret); return; } if (wc.status) { - log(LOG_ERR, "send completion error %d\n", wc.status); + PRINTF(cb, "send completion error %d\n", wc.status); return; } /* Spin waiting for server's Start STAG/TO/Len */ while (cb->state < RDMA_WRITE_ADV) { krping_cq_event_handler(cb->cq, cb); } bw_test(cb); } +static void krping_fr_test(struct krping_cb *cb) +{ + struct ib_fast_reg_page_list *pl; + struct ib_send_wr fr, inv, *bad; + struct ib_wc wc; + u8 key = 0; + struct ib_mr *mr; + int i; + int ret; + int size = cb->size; + int plen = (((size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; + time_t start; + int count = 0; + int scnt = 0; + + pl = ib_alloc_fast_reg_page_list(cb->qp->device, plen); + if (IS_ERR(pl)) { + PRINTF(cb, "ib_alloc_fast_reg_page_list failed %ld\n", PTR_ERR(pl)); + return; + } + + mr = ib_alloc_fast_reg_mr(cb->pd, plen); + if (IS_ERR(mr)) { + PRINTF(cb, "ib_alloc_fast_reg_mr failed %ld\n", PTR_ERR(pl)); + goto err1; + } + + for (i=0; ipage_list[i] = 0xcafebabe | i; + + memset(&fr, 0, sizeof fr); + fr.opcode = IB_WR_FAST_REG_MR; + fr.wr.fast_reg.page_shift = PAGE_SHIFT; + fr.wr.fast_reg.length = size; + fr.wr.fast_reg.page_list = pl; + fr.wr.fast_reg.page_list_len = plen; + fr.wr.fast_reg.iova_start = 0; + fr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; + fr.next = &inv; + memset(&inv, 0, sizeof inv); + inv.opcode = IB_WR_LOCAL_INV; + inv.send_flags = IB_SEND_SIGNALED; + + DEBUG_LOG(cb, "fr_test: stag index 0x%x plen %u size %u depth %u\n", mr->rkey >> 8, plen, cb->size, cb->txdepth); + start = time_uptime; + while (1) { + if ((time_uptime - start) >= 9) { + DEBUG_LOG(cb, "fr_test: pausing 1 second! count %u latest size %u plen %u\n", count, size, plen); + wait_event_interruptible(cb->sem, cb->state == ERROR); + if (cb->state == ERROR) + break; + start = time_uptime; + } + while (scnt < (cb->txdepth>>1)) { + ib_update_fast_reg_key(mr, ++key); + fr.wr.fast_reg.rkey = mr->rkey; + inv.ex.invalidate_rkey = mr->rkey; + size = arc4random() % cb->size; + if (size == 0) + size = cb->size; + plen = (((size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; + fr.wr.fast_reg.length = size; + fr.wr.fast_reg.page_list_len = plen; + ret = ib_post_send(cb->qp, &fr, &bad); + if (ret) { + PRINTF(cb, "ib_post_send failed %d\n", ret); + goto err2; + } + scnt++; + } + + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", ret); + goto err2; + } + if (ret == 1) { + if (wc.status) { + PRINTF(cb, "completion error %u\n", wc.status); + goto err2; + } + count++; + scnt--; + } + else if (krping_sigpending()) { + PRINTF(cb, "signal!\n"); + goto err2; + } + } while (ret == 1); + } +err2: +#if 0 + DEBUG_LOG(cb, "sleeping 1 second\n"); + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); +#endif + DEBUG_LOG(cb, "draining the cq...\n"); + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", ret); + break; + } + if (ret == 1) { + if (wc.status) { + PRINTF(cb, "completion error %u opcode %u\n", wc.status, wc.opcode); + } + } + } while (ret == 1); + DEBUG_LOG(cb, "fr_test: done!\n"); + ib_dereg_mr(mr); +err1: + ib_free_fast_reg_page_list(pl); +} + static int krping_connect_client(struct krping_cb *cb) { struct rdma_conn_param conn_param; int ret; memset(&conn_param, 0, sizeof conn_param); conn_param.responder_resources = 1; conn_param.initiator_depth = 1; conn_param.retry_count = 10; ret = rdma_connect(cb->cm_id, &conn_param); if (ret) { - log(LOG_ERR, "rdma_connect error %d\n", ret); + PRINTF(cb, "rdma_connect error %d\n", ret); return ret; } - krping_wait(cb, CONNECTED); + wait_event_interruptible(cb->sem, cb->state >= CONNECTED); if (cb->state == ERROR) { - log(LOG_ERR, "wait for CONNECTED state %d\n", cb->state); + PRINTF(cb, "wait for CONNECTED state %d\n", cb->state); return -1; } - DEBUG_LOG(PFX "rdma_connect successful\n"); + DEBUG_LOG(cb, "rdma_connect successful\n"); return 0; } static int krping_bind_client(struct krping_cb *cb) { struct sockaddr_in sin; int ret; memset(&sin, 0, sizeof(sin)); sin.sin_len = sizeof sin; sin.sin_family = AF_INET; sin.sin_addr.s_addr = cb->addr.s_addr; sin.sin_port = cb->port; ret = rdma_resolve_addr(cb->cm_id, NULL, (struct sockaddr *) &sin, 2000); if (ret) { - log(LOG_ERR, "rdma_resolve_addr error %d\n", ret); + PRINTF(cb, "rdma_resolve_addr error %d\n", ret); return ret; } - krping_wait(cb, ROUTE_RESOLVED); + wait_event_interruptible(cb->sem, cb->state >= ROUTE_RESOLVED); if (cb->state != ROUTE_RESOLVED) { - log(LOG_ERR, + PRINTF(cb, "addr/route resolution did not resolve: state %d\n", cb->state); - return EINTR; + return -EINTR; } - DEBUG_LOG(PFX "rdma_resolve_addr - rdma_resolve_route successful\n"); + if (cb->mem == FASTREG && !fastreg_supported(cb)) + return -EINVAL; + + DEBUG_LOG(cb, "rdma_resolve_addr - rdma_resolve_route successful\n"); return 0; } static void krping_run_client(struct krping_cb *cb) { struct ib_recv_wr *bad_wr; int ret; ret = krping_bind_client(cb); if (ret) return; ret = krping_setup_qp(cb, cb->cm_id); if (ret) { - log(LOG_ERR, "setup_qp failed: %d\n", ret); + PRINTF(cb, "setup_qp failed: %d\n", ret); return; } ret = krping_setup_buffers(cb); if (ret) { - log(LOG_ERR, "krping_setup_buffers failed: %d\n", ret); + PRINTF(cb, "krping_setup_buffers failed: %d\n", ret); goto err1; } ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); if (ret) { - log(LOG_ERR, "ib_post_recv failed: %d\n", ret); + PRINTF(cb, "ib_post_recv failed: %d\n", ret); goto err2; } ret = krping_connect_client(cb); if (ret) { - log(LOG_ERR, "connect error %d\n", ret); + PRINTF(cb, "connect error %d\n", ret); goto err2; } if (cb->wlat) krping_wlat_test_client(cb); else if (cb->rlat) krping_rlat_test_client(cb); else if (cb->bw) krping_bw_test_client(cb); + else if (cb->frtest) + krping_fr_test(cb); else krping_test_client(cb); rdma_disconnect(cb->cm_id); err2: krping_free_buffers(cb); err1: krping_free_qp(cb); } -int krping_doit(char *cmd) +int krping_doit(char *cmd, void *cookie) { struct krping_cb *cb; int op; int ret = 0; char *optarg; unsigned long optint; - debug = 0; - cb = malloc(sizeof(*cb), M_DEVBUF, M_WAITOK); + cb = kzalloc(sizeof(*cb), GFP_KERNEL); if (!cb) - return ENOMEM; - bzero(cb, sizeof *cb); + return -ENOMEM; - mtx_lock(&krping_mutex); - TAILQ_INSERT_TAIL(&krping_cbs, cb, list); - mtx_unlock(&krping_mutex); + mutex_lock(&krping_mutex); + list_add_tail(&cb->list, &krping_cbs); + mutex_unlock(&krping_mutex); + cb->cookie = cookie; cb->server = -1; cb->state = IDLE; cb->size = 64; cb->txdepth = RPING_SQ_DEPTH; - cb->use_dmamr = 1; - cb->memlimit = 0; - mtx_init(&cb->lock, "krping mtx", NULL, MTX_DUPOK|MTX_DEF); + cb->mem = DMA; + init_waitqueue_head(&cb->sem); while ((op = krping_getopt("krping", &cmd, krping_opts, NULL, &optarg, &optint)) != 0) { switch (op) { case 'a': cb->addr_str = optarg; - DEBUG_LOG(PFX "ipaddr (%s)\n", optarg); + DEBUG_LOG(cb, "ipaddr (%s)\n", optarg); if (!inet_aton(optarg, &cb->addr)) { - log(LOG_ERR, "bad addr string %s\n", optarg); + PRINTF(cb, "bad addr string %s\n", + optarg); ret = EINVAL; } break; - case 'D': - cb->use_dmamr = 1; - DEBUG_LOG(PFX "using dma mr\n"); - break; case 'p': cb->port = htons(optint); - DEBUG_LOG(PFX "port %d\n", (int)optint); + DEBUG_LOG(cb, "port %d\n", (int)optint); break; case 'P': cb->poll = 1; - DEBUG_LOG("server\n"); + DEBUG_LOG(cb, "server\n"); break; case 's': cb->server = 1; - DEBUG_LOG(PFX "server\n"); + DEBUG_LOG(cb, "server\n"); break; case 'c': cb->server = 0; - DEBUG_LOG(PFX "client\n"); + DEBUG_LOG(cb, "client\n"); break; case 'S': cb->size = optint; if ((cb->size < 1) || (cb->size > RPING_BUFSIZE)) { - log(LOG_ERR, "Invalid size %d " + PRINTF(cb, "Invalid size %d " "(valid range is 1 to %d)\n", cb->size, RPING_BUFSIZE); ret = EINVAL; } else - DEBUG_LOG(PFX "size %d\n", (int)optint); + DEBUG_LOG(cb, "size %d\n", (int)optint); break; case 'C': cb->count = optint; if (cb->count < 0) { - log(LOG_ERR, "Invalid count %d\n", + PRINTF(cb, "Invalid count %d\n", cb->count); ret = EINVAL; } else - DEBUG_LOG(PFX "count %d\n", (int) cb->count); + DEBUG_LOG(cb, "count %d\n", (int) cb->count); break; case 'v': cb->verbose++; - DEBUG_LOG(PFX "verbose\n"); + DEBUG_LOG(cb, "verbose\n"); break; case 'V': cb->validate++; - DEBUG_LOG(PFX "validate data\n"); + DEBUG_LOG(cb, "validate data\n"); break; - case 'L': - cb->rlat++; - break; case 'l': cb->wlat++; break; + case 'L': + cb->rlat++; + break; case 'B': cb->bw++; break; - case 't': - cb->txdepth = optint; - DEBUG_LOG(PFX "txdepth %d\n", cb->txdepth); - break; case 'd': - debug++; + cb->duplex++; break; case 'm': - cb->memlimit = optint; - if (cb->memlimit < 1) { - log(LOG_ERR, "Invalid memory limit %ju\n", - cb->memlimit); - ret = EINVAL; - } else - DEBUG_LOG(PFX "memory limit %d\n", (int)optint); - break; + if (!strncmp(optarg, "dma", 3)) + cb->mem = DMA; + else if (!strncmp(optarg, "fastreg", 7)) + cb->mem = FASTREG; + else if (!strncmp(optarg, "mw", 2)) + cb->mem = MW; + else if (!strncmp(optarg, "mr", 2)) + cb->mem = MR; + else { + PRINTF(cb, "unknown mem mode %s. " + "Must be dma, fastreg, mw, or mr\n", + optarg); + ret = -EINVAL; + break; + } + break; + case 'I': + cb->server_invalidate = 1; + break; + case 'T': + cb->txdepth = optint; + DEBUG_LOG(cb, "txdepth %d\n", (int) cb->txdepth); + break; + case 'Z': + cb->local_dma_lkey = 1; + DEBUG_LOG(cb, "using local dma lkey\n"); + break; + case 'R': + cb->read_inv = 1; + DEBUG_LOG(cb, "using read-with-inv\n"); + break; + case 'f': + cb->frtest = 1; + DEBUG_LOG(cb, "fast-reg test!\n"); + break; default: - log(LOG_ERR, "unknown opt %s\n", optarg); - ret = EINVAL; + PRINTF(cb, "unknown opt %s\n", optarg); + ret = -EINVAL; break; } } if (ret) goto out; if (cb->server == -1) { - log(LOG_ERR, "must be either client or server\n"); - ret = EINVAL; + PRINTF(cb, "must be either client or server\n"); + ret = -EINVAL; goto out; } - if ((cb->bw + cb->rlat + cb->wlat) > 1) { - log(LOG_ERR, "Pick only one test: bw, rlat, wlat\n"); - ret = EINVAL; + + if (cb->server && cb->frtest) { + PRINTF(cb, "must be client to run frtest\n"); + ret = -EINVAL; goto out; } + if ((cb->frtest + cb->bw + cb->rlat + cb->wlat) > 1) { + PRINTF(cb, "Pick only one test: fr, bw, rlat, wlat\n"); + ret = -EINVAL; + goto out; + } + if (cb->server_invalidate && cb->mem != FASTREG) { + PRINTF(cb, "server_invalidate only valid with fastreg mem_mode\n"); + ret = -EINVAL; + goto out; + } + + if (cb->read_inv && cb->mem != FASTREG) { + PRINTF(cb, "read_inv only valid with fastreg mem_mode\n"); + ret = -EINVAL; + goto out; + } + + if (cb->mem != MR && (cb->wlat || cb->rlat || cb->bw)) { + PRINTF(cb, "wlat, rlat, and bw tests only support mem_mode MR\n"); + ret = -EINVAL; + goto out; + } + cb->cm_id = rdma_create_id(krping_cma_event_handler, cb, RDMA_PS_TCP); if (IS_ERR(cb->cm_id)) { ret = PTR_ERR(cb->cm_id); - log(LOG_ERR, "rdma_create_id error %d\n", ret); + PRINTF(cb, "rdma_create_id error %d\n", ret); goto out; } - DEBUG_LOG(PFX "created cm_id %p\n", cb->cm_id); + DEBUG_LOG(cb, "created cm_id %p\n", cb->cm_id); + if (cb->server) krping_run_server(cb); else krping_run_client(cb); - DEBUG_LOG(PFX "destroy cm_id %p\n", cb->cm_id); + + DEBUG_LOG(cb, "destroy cm_id %p\n", cb->cm_id); rdma_destroy_id(cb->cm_id); out: - mtx_lock(&krping_mutex); - TAILQ_REMOVE(&krping_cbs, cb, list); - mtx_unlock(&krping_mutex); - free(cb, M_DEVBUF); + mutex_lock(&krping_mutex); + list_del(&cb->list); + mutex_unlock(&krping_mutex); + kfree(cb); return ret; } +void +krping_walk_cb_list(void (*f)(struct krping_stats *, void *), void *arg) +{ + struct krping_cb *cb; + + mutex_lock(&krping_mutex); + list_for_each_entry(cb, &krping_cbs, list) + (*f)(cb->pd ? &cb->stats : NULL, arg); + mutex_unlock(&krping_mutex); +} + void krping_init(void) { - mtx_init(&krping_mutex, "krping lock", NULL, MTX_DEF); - TAILQ_INIT(&krping_cbs); + + mutex_init(&krping_mutex); } Index: stable/9/sys/contrib/rdma/krping/krping.h =================================================================== --- stable/9/sys/contrib/rdma/krping/krping.h (revision 273863) +++ stable/9/sys/contrib/rdma/krping/krping.h (revision 273864) @@ -1,130 +1,21 @@ /* * $FreeBSD$ */ -#include -#include -/* - * Krping header stuffs... - */ - struct krping_stats { - unsigned send_bytes; - unsigned send_msgs; - unsigned recv_bytes; - unsigned recv_msgs; - unsigned write_bytes; - unsigned write_msgs; - unsigned read_bytes; - unsigned read_msgs; + unsigned long long send_bytes; + unsigned long long send_msgs; + unsigned long long recv_bytes; + unsigned long long recv_msgs; + unsigned long long write_bytes; + unsigned long long write_msgs; + unsigned long long read_bytes; + unsigned long long read_msgs; + char name[16]; }; - -/* - * These states are used to signal events between the completion handler - * and the main client or server thread. - * - * Once CONNECTED, they cycle through RDMA_READ_ADV, RDMA_WRITE_ADV, - * and RDMA_WRITE_COMPLETE for each ping. - */ -enum test_state { - IDLE = 1, - CONNECT_REQUEST, - ADDR_RESOLVED, - ROUTE_RESOLVED, - CONNECTED, - RDMA_READ_ADV, - RDMA_READ_COMPLETE, - RDMA_WRITE_ADV, - RDMA_WRITE_COMPLETE, - ERROR -}; - -struct krping_rdma_info { - uint64_t buf; - uint32_t rkey; - uint32_t size; -}; - -/* - * Control block struct. - */ -struct krping_cb { - int server; /* 0 iff client */ - struct ib_cq *cq; - struct ib_pd *pd; - struct ib_qp *qp; - struct ib_mr *dma_mr; - int use_dmamr; - - struct ib_recv_wr rq_wr; /* recv work request record */ - struct ib_sge recv_sgl; /* recv single SGE */ - struct krping_rdma_info recv_buf;/* malloc'd buffer */ - struct ib_mr *recv_mr; - - struct ib_send_wr sq_wr; /* send work requrest record */ - struct ib_sge send_sgl; - struct krping_rdma_info send_buf;/* single send buf */ - struct ib_mr *send_mr; - - struct ib_send_wr rdma_sq_wr; /* rdma work request record */ - struct ib_sge rdma_sgl; /* rdma single SGE */ - char *rdma_buf; /* used as rdma sink */ - u64 rdma_addr; - struct ib_mr *rdma_mr; - - uint32_t remote_rkey; /* remote guys RKEY */ - uint64_t remote_addr; /* remote guys TO */ - uint32_t remote_len; /* remote guys LEN */ - - char *start_buf; /* rdma read src */ - u64 start_addr; - struct ib_mr *start_mr; - - enum test_state state; /* used for cond/signalling */ - struct mtx lock; - struct krping_stats stats; - - uint16_t port; /* dst port in NBO */ - struct in_addr addr; /* dst addr in NBO */ - char *addr_str; /* dst addr string */ - int verbose; /* verbose logging */ - int count; /* ping count */ - int size; /* ping data size */ - int validate; /* validate ping data */ - uint64_t memlimit; /* limit of the physical memory that - can be registered with dma_mr mode */ - - /* CM stuff */ - struct rdma_cm_id *cm_id; /* connection on client side,*/ - /* listener on service side. */ - struct rdma_cm_id *child_cm_id; /* connection on server side */ - TAILQ_ENTRY(krping_cb) list; - - int rlat; /* run read latency test */ - int wlat; /* run write latency test */ - int bw; /* run write bw test */ - int duplex; /* run write bw full duplex test */ - int poll; /* poll vs block in rlat */ - int txdepth; -}; - -static __inline uint64_t -get_cycles(void) -{ - u_int32_t low, high; - __asm __volatile("rdtsc" : "=a" (low), "=d" (high)); - return (low | ((u_int64_t)high << 32)); -} - -#define htonll(x) htobe64((x)) -#define ntohll(x) be64toh((x)) - -typedef uint64_t cycles_t; - -extern struct mtx krping_mutex; -TAILQ_HEAD(krping_cb_list, krping_cb); -extern struct krping_cb_list krping_cbs; - -int krping_doit(char *cmd); +int krping_doit(char *, void *); +void krping_walk_cb_list(void (*)(struct krping_stats *, void *), void *); void krping_init(void); +void krping_printf(void *, const char *, ...); +int krping_sigpending(void); Index: stable/9/sys/contrib/rdma/krping/krping_dev.c =================================================================== --- stable/9/sys/contrib/rdma/krping/krping_dev.c (revision 273863) +++ stable/9/sys/contrib/rdma/krping/krping_dev.c (revision 273864) @@ -1,180 +1,232 @@ /* - * This code lifted from: + * This code lifted from: * Simple `echo' pseudo-device KLD * Murray Stokely * Converted to 5.X by Søren (Xride) Straarup */ /* - * /bin/echo "server,port=9999,addr=192.168.69.142,validate" > /dev/krping - * /bin/echo "client,port=9999,addr=192.168.69.142,validate" > /dev/krping + * /bin/echo "server,port=9999,addr=192.168.69.142,validate" > /dev/krping + * /bin/echo "client,port=9999,addr=192.168.69.142,validate" > /dev/krping */ #include __FBSDID("$FreeBSD$"); #include +#include #include /* uprintf */ #include #include /* defines used in kernel.h */ #include /* types used in module initialization */ #include /* cdevsw struct */ #include /* uio struct */ #include +#include +#include +#include #include "krping.h" #define BUFFERSIZE 512 +SYSCTL_NODE(_dev, OID_AUTO, krping, CTLFLAG_RW, 0, "kernel rping module"); + +int krping_debug = 0; +SYSCTL_INT(_dev_krping, OID_AUTO, debug, CTLFLAG_RW, &krping_debug, 0 , ""); + /* Function prototypes */ static d_open_t krping_open; static d_close_t krping_close; static d_read_t krping_read; static d_write_t krping_write; /* Character device entry points */ static struct cdevsw krping_cdevsw = { .d_version = D_VERSION, .d_open = krping_open, .d_close = krping_close, .d_read = krping_read, .d_write = krping_write, .d_name = "krping", }; typedef struct s_krping { char msg[BUFFERSIZE]; int len; } krping_t; +struct stats_list_entry { + STAILQ_ENTRY(stats_list_entry) link; + struct krping_stats *stats; +}; +STAILQ_HEAD(stats_list, stats_list_entry); + /* vars */ static struct cdev *krping_dev; -#undef MODULE_VERSION -#include - static int krping_loader(struct module *m, int what, void *arg) { int err = 0; switch (what) { case MOD_LOAD: /* kldload */ krping_init(); - krping_dev = make_dev(&krping_cdevsw, 0, UID_ROOT, GID_WHEEL, + krping_dev = make_dev(&krping_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "krping"); printf("Krping device loaded.\n"); break; case MOD_UNLOAD: destroy_dev(krping_dev); printf("Krping device unloaded.\n"); break; default: err = EOPNOTSUPP; break; } - return err; + + return (err); } static int krping_open(struct cdev *dev, int oflags, int devtype, struct thread *p) { - int err = 0; - return err; + + return (0); } static int krping_close(struct cdev *dev, int fflag, int devtype, struct thread *p) { + return 0; } +static void +krping_copy_stats(struct krping_stats *stats, void *arg) +{ + struct stats_list_entry *s; + struct stats_list *list = arg; + + s = malloc(sizeof(*s), M_DEVBUF, M_NOWAIT | M_ZERO); + if (s == NULL) + return; + if (stats != NULL) { + s->stats = malloc(sizeof(*stats), M_DEVBUF, M_NOWAIT | M_ZERO); + if (s->stats == NULL) { + free(s, M_DEVBUF); + return; + } + *s->stats = *stats; + } + STAILQ_INSERT_TAIL(list, s, link); +} + static int krping_read(struct cdev *dev, struct uio *uio, int ioflag) { - struct krping_cb *cb, *cb2; - int num=1; - struct krping_cb_list copy_cbs; + int num = 1; + struct stats_list list; + struct stats_list_entry *e; + STAILQ_INIT(&list); + krping_walk_cb_list(krping_copy_stats, &list); + + if (STAILQ_EMPTY(&list)) + return (0); + uprintf("krping: %4s %10s %10s %10s %10s %10s %10s %10s %10s %10s\n", - "num", "device", "snd bytes", "snd msgs", "rcv bytes", - "rcv msgs", "wr bytes", "wr msgs", "rd bytes", "rd msgs"); - TAILQ_INIT(©_cbs); + "num", "device", "snd bytes", "snd msgs", "rcv bytes", "rcv msgs", + "wr bytes", "wr msgs", "rd bytes", "rd msgs"); - mtx_lock(&krping_mutex); - TAILQ_FOREACH(cb, &krping_cbs, list) { - cb2 = malloc(sizeof(*cb), M_DEVBUF, M_NOWAIT|M_ZERO); - if (!cb2) - break; - bcopy(cb, cb2, sizeof(*cb)); - TAILQ_INSERT_TAIL(©_cbs, cb2, list); - } - mtx_unlock(&krping_mutex); + while (!STAILQ_EMPTY(&list)) { + e = STAILQ_FIRST(&list); + STAILQ_REMOVE_HEAD(&list, link); + if (e->stats == NULL) + uprintf("krping: %d listen\n", num); + else { + struct krping_stats *stats = e->stats; - while (!TAILQ_EMPTY(©_cbs)) { - - cb = TAILQ_FIRST(©_cbs); - TAILQ_REMOVE(©_cbs, cb, list); - if (cb->pd) { - uprintf("krping: %4d %10s %10u %10u %10u %10u %10u %10u %10u %10u\n", - num++, cb->pd->device->name, cb->stats.send_bytes, - cb->stats.send_msgs, cb->stats.recv_bytes, - cb->stats.recv_msgs, cb->stats.write_bytes, - cb->stats.write_msgs, - cb->stats.read_bytes, - cb->stats.read_msgs); - } else { - uprintf("krping: %d listen\n", num++); + uprintf("krping: %4d %10s %10llu %10llu %10llu %10llu " + "%10llu %10llu %10llu %10llu\n", num, stats->name, + stats->send_bytes, stats->send_msgs, + stats->recv_bytes, stats->recv_msgs, + stats->write_bytes, stats->write_msgs, + stats->read_bytes, stats->read_msgs); + free(stats, M_DEVBUF); } - free(cb, M_DEVBUF); + num++; + free(e, M_DEVBUF); } - return 0; + + return (0); } static int krping_write(struct cdev *dev, struct uio *uio, int ioflag) { int err = 0; int amt; int remain = BUFFERSIZE; char *cp; krping_t *krpingmsg; krpingmsg = malloc(sizeof *krpingmsg, M_DEVBUF, M_WAITOK|M_ZERO); if (!krpingmsg) { uprintf("Could not malloc mem!\n"); return ENOMEM; } cp = krpingmsg->msg; while (uio->uio_resid) { amt = MIN(uio->uio_resid, remain); if (amt == 0) break; /* Copy the string in from user memory to kernel memory */ err = uiomove(cp, amt, uio); if (err) { uprintf("Write failed: bad address!\n"); return err; } cp += amt; remain -= amt; } if (uio->uio_resid != 0) { uprintf("Message too big. max size is %d!\n", BUFFERSIZE); return EMSGSIZE; } /* null terminate and remove the \n */ cp--; *cp = 0; krpingmsg->len = (unsigned long)(cp - krpingmsg->msg); uprintf("krping: write string = |%s|\n", krpingmsg->msg); - err = krping_doit(krpingmsg->msg); + err = krping_doit(krpingmsg->msg, curproc); free(krpingmsg, M_DEVBUF); return(err); } -DEV_MODULE(krping,krping_loader,NULL); +void +krping_printf(void *cookie, const char *fmt, ...) +{ + va_list ap; + + printf("krping: "); + + va_start(ap, fmt); + vprintf(fmt, ap); + va_end(ap); +} + +int +krping_sigpending(void) +{ + + return (SIGPENDING(curthread)); +} + +DEV_MODULE(krping, krping_loader, NULL); +MODULE_DEPEND(krping, ibcore, 1, 1, 1); Index: stable/9/sys/dev/cxgb/cxgb_osdep.h =================================================================== --- stable/9/sys/dev/cxgb/cxgb_osdep.h (revision 273863) +++ stable/9/sys/dev/cxgb/cxgb_osdep.h (revision 273864) @@ -1,310 +1,308 @@ /************************************************************************** Copyright (c) 2007, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. $FreeBSD$ ***************************************************************************/ #include #include #include #include #include #include #include #include #ifndef _CXGB_OSDEP_H_ #define _CXGB_OSDEP_H_ typedef struct adapter adapter_t; typedef struct port_info pinfo_t; struct sge_rspq; enum { TP_TMR_RES = 200, /* TP timer resolution in usec */ MAX_NPORTS = 4, /* max # of ports */ TP_SRAM_OFFSET = 4096, /* TP SRAM content offset in eeprom */ TP_SRAM_LEN = 2112, /* TP SRAM content offset in eeprom */ }; struct t3_mbuf_hdr { struct mbuf *mh_head; struct mbuf *mh_tail; }; #ifndef PANIC_IF #define PANIC_IF(exp) do { \ if (exp) \ panic("BUG: %s", #exp); \ } while (0) #endif #if __FreeBSD_version < 800054 #if defined (__GNUC__) #if #cpu(i386) || defined __i386 || defined i386 || defined __i386__ || #cpu(x86_64) || defined __x86_64__ #define mb() __asm__ __volatile__ ("mfence;": : :"memory") #define wmb() __asm__ __volatile__ ("sfence;": : :"memory") #define rmb() __asm__ __volatile__ ("lfence;": : :"memory") #elif #cpu(sparc64) || defined sparc64 || defined __sparcv9 #define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory") #define wmb() mb() #define rmb() mb() #elif #cpu(sparc) || defined sparc || defined __sparc__ #define mb() __asm__ __volatile__ ("stbar;": : :"memory") #define wmb() mb() #define rmb() mb() #else #define wmb() mb() #define rmb() mb() #define mb() /* XXX just to make this compile */ #endif #else #error "unknown compiler" #endif #endif -#define __read_mostly __attribute__((__section__(".data.read_mostly"))) - /* * Workaround for weird Chelsio issue */ #if __FreeBSD_version > 700029 #define PRIV_SUPPORTED #endif #define CXGB_TX_CLEANUP_THRESHOLD 32 #define TX_MAX_SIZE (1 << 16) /* 64KB */ #define TX_MAX_SEGS 36 /* maximum supported by card */ #define TX_MAX_DESC 4 /* max descriptors per packet */ #define TX_START_MAX_DESC (TX_MAX_DESC << 2) /* maximum number of descriptors * call to start used per */ #define TX_CLEAN_MAX_DESC (TX_MAX_DESC << 4) /* maximum tx descriptors * to clean per iteration */ #define TX_WR_SIZE_MAX 11*1024 /* the maximum total size of packets aggregated into a single * TX WR */ #define TX_WR_COUNT_MAX 7 /* the maximum total number of packets that can be * aggregated into a single TX WR */ #if defined(__i386__) || defined(__amd64__) static __inline void prefetch(void *x) { __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); } #define smp_mb() mb() #define L1_CACHE_BYTES 128 extern void kdb_backtrace(void); #define WARN_ON(condition) do { \ if (__predict_false((condition)!=0)) { \ log(LOG_WARNING, "BUG: warning at %s:%d/%s()\n", __FILE__, __LINE__, __FUNCTION__); \ kdb_backtrace(); \ } \ } while (0) #else #define smp_mb() #define prefetch(x) #define L1_CACHE_BYTES 32 #endif #define DBG_RX (1 << 0) static const int debug_flags = DBG_RX; #ifdef DEBUG_PRINT #define DBG(flag, msg) do { \ if ((flag & debug_flags)) \ printf msg; \ } while (0) #else #define DBG(...) #endif #include #define promisc_rx_mode(rm) ((rm)->port->ifp->if_flags & IFF_PROMISC) #define allmulti_rx_mode(rm) ((rm)->port->ifp->if_flags & IFF_ALLMULTI) #define CH_ERR(adap, fmt, ...) log(LOG_ERR, fmt, ##__VA_ARGS__) #define CH_WARN(adap, fmt, ...) log(LOG_WARNING, fmt, ##__VA_ARGS__) #define CH_ALERT(adap, fmt, ...) log(LOG_ALERT, fmt, ##__VA_ARGS__) #define t3_os_sleep(x) DELAY((x) * 1000) #define test_and_clear_bit(bit, p) atomic_cmpset_int((p), ((*(p)) | (1< __FBSDID("$FreeBSD$"); #include "opt_inet.h" #ifdef TCP_OFFLOAD #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static int iwch_modify_port(struct ib_device *ibdev, u8 port, int port_modify_mask, struct ib_port_modify *props) { return (-ENOSYS); } static struct ib_ah * iwch_ah_create(struct ib_pd *pd, struct ib_ah_attr *ah_attr) { return ERR_PTR(-ENOSYS); } static int iwch_ah_destroy(struct ib_ah *ah) { return (-ENOSYS); } static int iwch_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) { return (-ENOSYS); } static int iwch_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) { return (-ENOSYS); } static int iwch_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, struct ib_wc *in_wc, struct ib_grh *in_grh, struct ib_mad *in_mad, struct ib_mad *out_mad) { return (-ENOSYS); } static int iwch_dealloc_ucontext(struct ib_ucontext *context) { struct iwch_dev *rhp = to_iwch_dev(context->device); struct iwch_ucontext *ucontext = to_iwch_ucontext(context); struct iwch_mm_entry *mm, *tmp; CTR2(KTR_IW_CXGB, "%s context %p", __FUNCTION__, context); TAILQ_FOREACH_SAFE(mm, &ucontext->mmaps, entry, tmp) { TAILQ_REMOVE(&ucontext->mmaps, mm, entry); cxfree(mm); } cxio_release_ucontext(&rhp->rdev, &ucontext->uctx); cxfree(ucontext); return 0; } static struct ib_ucontext * iwch_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata) { struct iwch_ucontext *context; struct iwch_dev *rhp = to_iwch_dev(ibdev); CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev); context = malloc(sizeof(*context), M_DEVBUF, M_ZERO|M_NOWAIT); if (!context) return ERR_PTR(-ENOMEM); cxio_init_ucontext(&rhp->rdev, &context->uctx); TAILQ_INIT(&context->mmaps); mtx_init(&context->mmap_lock, "ucontext mmap", NULL, MTX_DEF); return &context->ibucontext; } static int iwch_destroy_cq(struct ib_cq *ib_cq) { struct iwch_cq *chp; CTR2(KTR_IW_CXGB, "%s ib_cq %p", __FUNCTION__, ib_cq); chp = to_iwch_cq(ib_cq); remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid); mtx_lock(&chp->lock); if (--chp->refcnt) msleep(chp, &chp->lock, 0, "iwch_destroy_cq", 0); mtx_unlock(&chp->lock); cxio_destroy_cq(&chp->rhp->rdev, &chp->cq); cxfree(chp); return 0; } static struct ib_cq * iwch_create_cq(struct ib_device *ibdev, int entries, int vector, struct ib_ucontext *ib_context, struct ib_udata *udata) { struct iwch_dev *rhp; struct iwch_cq *chp; struct iwch_create_cq_resp uresp; struct iwch_create_cq_req ureq; struct iwch_ucontext *ucontext = NULL; static int warned; size_t resplen; CTR3(KTR_IW_CXGB, "%s ib_dev %p entries %d", __FUNCTION__, ibdev, entries); rhp = to_iwch_dev(ibdev); chp = malloc(sizeof(*chp), M_DEVBUF, M_NOWAIT|M_ZERO); if (!chp) { return ERR_PTR(-ENOMEM); } if (ib_context) { ucontext = to_iwch_ucontext(ib_context); if (!t3a_device(rhp)) { if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) { cxfree(chp); return ERR_PTR(-EFAULT); } chp->user_rptr_addr = (u32 /*__user */*)(unsigned long)ureq.user_rptr_addr; } } if (t3a_device(rhp)) { /* * T3A: Add some fluff to handle extra CQEs inserted * for various errors. * Additional CQE possibilities: * TERMINATE, * incoming RDMA WRITE Failures * incoming RDMA READ REQUEST FAILUREs * NOTE: We cannot ensure the CQ won't overflow. */ entries += 16; } entries = roundup_pow_of_two(entries); chp->cq.size_log2 = ilog2(entries); if (cxio_create_cq(&rhp->rdev, &chp->cq, !ucontext)) { cxfree(chp); return ERR_PTR(-ENOMEM); } chp->rhp = rhp; chp->ibcq.cqe = 1 << chp->cq.size_log2; mtx_init(&chp->lock, "cxgb cq", NULL, MTX_DEF|MTX_DUPOK); chp->refcnt = 1; if (insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid)) { cxio_destroy_cq(&chp->rhp->rdev, &chp->cq); cxfree(chp); return ERR_PTR(-ENOMEM); } if (ucontext) { struct iwch_mm_entry *mm; mm = kmalloc(sizeof *mm, M_NOWAIT); if (!mm) { iwch_destroy_cq(&chp->ibcq); return ERR_PTR(-ENOMEM); } uresp.cqid = chp->cq.cqid; uresp.size_log2 = chp->cq.size_log2; mtx_lock(&ucontext->mmap_lock); uresp.key = ucontext->key; ucontext->key += PAGE_SIZE; mtx_unlock(&ucontext->mmap_lock); mm->key = uresp.key; mm->addr = vtophys(chp->cq.queue); if (udata->outlen < sizeof uresp) { if (!warned++) CTR1(KTR_IW_CXGB, "%s Warning - " "downlevel libcxgb3 (non-fatal).\n", __func__); mm->len = PAGE_ALIGN((1UL << uresp.size_log2) * sizeof(struct t3_cqe)); resplen = sizeof(struct iwch_create_cq_resp_v0); } else { mm->len = PAGE_ALIGN(((1UL << uresp.size_log2) + 1) * sizeof(struct t3_cqe)); uresp.memsize = mm->len; resplen = sizeof uresp; } if (ib_copy_to_udata(udata, &uresp, resplen)) { cxfree(mm); iwch_destroy_cq(&chp->ibcq); return ERR_PTR(-EFAULT); } insert_mmap(ucontext, mm); } CTR4(KTR_IW_CXGB, "created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx", chp->cq.cqid, chp, (1 << chp->cq.size_log2), (unsigned long long) chp->cq.dma_addr); return &chp->ibcq; } static int iwch_resize_cq(struct ib_cq *cq __unused, int cqe __unused, struct ib_udata *udata __unused) { return (-ENOSYS); } static int iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { struct iwch_dev *rhp; struct iwch_cq *chp; enum t3_cq_opcode cq_op; int err; u32 rptr; chp = to_iwch_cq(ibcq); rhp = chp->rhp; if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED) cq_op = CQ_ARM_SE; else cq_op = CQ_ARM_AN; if (chp->user_rptr_addr) { if (copyin(&rptr, chp->user_rptr_addr, 4)) return (-EFAULT); mtx_lock(&chp->lock); chp->cq.rptr = rptr; } else mtx_lock(&chp->lock); CTR2(KTR_IW_CXGB, "%s rptr 0x%x", __FUNCTION__, chp->cq.rptr); err = cxio_hal_cq_op(&rhp->rdev, &chp->cq, cq_op, 0); mtx_unlock(&chp->lock); if (err < 0) log(LOG_ERR, "Error %d rearming CQID 0x%x\n", err, chp->cq.cqid); if (err > 0 && !(flags & IB_CQ_REPORT_MISSED_EVENTS)) err = 0; return err; } static int iwch_mmap(struct ib_ucontext *context __unused, struct vm_area_struct *vma __unused) { return (-ENOSYS); } static int iwch_deallocate_pd(struct ib_pd *pd) { struct iwch_dev *rhp; struct iwch_pd *php; php = to_iwch_pd(pd); rhp = php->rhp; CTR3(KTR_IW_CXGB, "%s ibpd %p pdid 0x%x", __FUNCTION__, pd, php->pdid); cxio_hal_put_pdid(rhp->rdev.rscp, php->pdid); cxfree(php); return 0; } static struct ib_pd *iwch_allocate_pd(struct ib_device *ibdev, struct ib_ucontext *context, struct ib_udata *udata) { struct iwch_pd *php; u32 pdid; struct iwch_dev *rhp; CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev); rhp = (struct iwch_dev *) ibdev; pdid = cxio_hal_get_pdid(rhp->rdev.rscp); if (!pdid) return ERR_PTR(-EINVAL); php = malloc(sizeof(*php), M_DEVBUF, M_ZERO|M_NOWAIT); if (!php) { cxio_hal_put_pdid(rhp->rdev.rscp, pdid); return ERR_PTR(-ENOMEM); } php->pdid = pdid; php->rhp = rhp; if (context) { if (ib_copy_to_udata(udata, &php->pdid, sizeof (__u32))) { iwch_deallocate_pd(&php->ibpd); return ERR_PTR(-EFAULT); } } CTR3(KTR_IW_CXGB, "%s pdid 0x%0x ptr 0x%p", __FUNCTION__, pdid, php); return &php->ibpd; } static int iwch_dereg_mr(struct ib_mr *ib_mr) { struct iwch_dev *rhp; struct iwch_mr *mhp; u32 mmid; CTR2(KTR_IW_CXGB, "%s ib_mr %p", __FUNCTION__, ib_mr); /* There can be no memory windows */ if (atomic_load_acq_int(&ib_mr->usecnt.counter)) return (-EINVAL); mhp = to_iwch_mr(ib_mr); rhp = mhp->rhp; mmid = mhp->attr.stag >> 8; cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, mhp->attr.pbl_addr); iwch_free_pbl(mhp); remove_handle(rhp, &rhp->mmidr, mmid); if (mhp->kva) cxfree((void *) (unsigned long) mhp->kva); if (mhp->umem) ib_umem_release(mhp->umem); CTR3(KTR_IW_CXGB, "%s mmid 0x%x ptr %p", __FUNCTION__, mmid, mhp); cxfree(mhp); return 0; } static struct ib_mr *iwch_register_phys_mem(struct ib_pd *pd, struct ib_phys_buf *buffer_list, int num_phys_buf, int acc, u64 *iova_start) { __be64 *page_list; int shift; u64 total_size; int npages; struct iwch_dev *rhp; struct iwch_pd *php; struct iwch_mr *mhp; int ret; CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd); php = to_iwch_pd(pd); rhp = php->rhp; mhp = malloc(sizeof(*mhp), M_DEVBUF, M_ZERO|M_NOWAIT); if (!mhp) return ERR_PTR(-ENOMEM); mhp->rhp = rhp; /* First check that we have enough alignment */ if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) { ret = -EINVAL; goto err; } if (num_phys_buf > 1 && ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK)) { ret = -EINVAL; goto err; } ret = build_phys_page_list(buffer_list, num_phys_buf, iova_start, &total_size, &npages, &shift, &page_list); if (ret) goto err; ret = iwch_alloc_pbl(mhp, npages); if (ret) { cxfree(page_list); goto err_pbl; } ret = iwch_write_pbl(mhp, page_list, npages, 0); cxfree(page_list); if (ret) goto err; mhp->attr.pdid = php->pdid; mhp->attr.zbva = 0; mhp->attr.perms = iwch_ib_to_tpt_access(acc); mhp->attr.va_fbo = *iova_start; mhp->attr.page_size = shift - 12; mhp->attr.len = (u32) total_size; mhp->attr.pbl_size = npages; ret = iwch_register_mem(rhp, php, mhp, shift); if (ret) goto err_pbl; return &mhp->ibmr; err_pbl: iwch_free_pbl(mhp); err: cxfree(mhp); return ERR_PTR(ret); } static int iwch_reregister_phys_mem(struct ib_mr *mr, int mr_rereg_mask, struct ib_pd *pd, struct ib_phys_buf *buffer_list, int num_phys_buf, int acc, u64 * iova_start) { struct iwch_mr mh, *mhp; struct iwch_pd *php; struct iwch_dev *rhp; __be64 *page_list = NULL; int shift = 0; u64 total_size; int npages = 0; int ret; CTR3(KTR_IW_CXGB, "%s ib_mr %p ib_pd %p", __FUNCTION__, mr, pd); /* There can be no memory windows */ if (atomic_load_acq_int(&mr->usecnt.counter)) return (-EINVAL); mhp = to_iwch_mr(mr); rhp = mhp->rhp; php = to_iwch_pd(mr->pd); /* make sure we are on the same adapter */ if (rhp != php->rhp) return (-EINVAL); memcpy(&mh, mhp, sizeof *mhp); if (mr_rereg_mask & IB_MR_REREG_PD) php = to_iwch_pd(pd); if (mr_rereg_mask & IB_MR_REREG_ACCESS) mh.attr.perms = iwch_ib_to_tpt_access(acc); if (mr_rereg_mask & IB_MR_REREG_TRANS) { ret = build_phys_page_list(buffer_list, num_phys_buf, iova_start, &total_size, &npages, &shift, &page_list); if (ret) return ret; } ret = iwch_reregister_mem(rhp, php, &mh, shift, npages); cxfree(page_list); if (ret) { return ret; } if (mr_rereg_mask & IB_MR_REREG_PD) mhp->attr.pdid = php->pdid; if (mr_rereg_mask & IB_MR_REREG_ACCESS) mhp->attr.perms = iwch_ib_to_tpt_access(acc); if (mr_rereg_mask & IB_MR_REREG_TRANS) { mhp->attr.zbva = 0; mhp->attr.va_fbo = *iova_start; mhp->attr.page_size = shift - 12; mhp->attr.len = (u32) total_size; mhp->attr.pbl_size = npages; } return 0; } static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, - u64 virt, int acc, struct ib_udata *udata) + u64 virt, int acc, struct ib_udata *udata, + int mr_id) { __be64 *pages; int shift, i, n; int err = 0; struct ib_umem_chunk *chunk; struct iwch_dev *rhp; struct iwch_pd *php; struct iwch_mr *mhp; struct iwch_reg_user_mr_resp uresp; #ifdef notyet int j, k, len; #endif CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd); php = to_iwch_pd(pd); rhp = php->rhp; mhp = malloc(sizeof(*mhp), M_DEVBUF, M_NOWAIT|M_ZERO); if (!mhp) return ERR_PTR(-ENOMEM); mhp->rhp = rhp; mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0); if (IS_ERR(mhp->umem)) { err = PTR_ERR(mhp->umem); cxfree(mhp); return ERR_PTR(-err); } shift = ffs(mhp->umem->page_size) - 1; n = 0; list_for_each_entry(chunk, &mhp->umem->chunk_list, list) n += chunk->nents; err = iwch_alloc_pbl(mhp, n); if (err) goto err; pages = (__be64 *) kmalloc(n * sizeof(u64), M_NOWAIT); if (!pages) { err = -ENOMEM; goto err_pbl; } i = n = 0; #ifdef notyet TAILQ_FOREACH(chunk, &mhp->umem->chunk_list, entry) for (j = 0; j < chunk->nmap; ++j) { len = sg_dma_len(&chunk->page_list[j]) >> shift; for (k = 0; k < len; ++k) { pages[i++] = htobe64(sg_dma_address( &chunk->page_list[j]) + mhp->umem->page_size * k); if (i == PAGE_SIZE / sizeof *pages) { err = iwch_write_pbl(mhp, pages, i, n); if (err) goto pbl_done; n += i; i = 0; } } } #endif if (i) err = iwch_write_pbl(mhp, pages, i, n); #ifdef notyet pbl_done: #endif cxfree(pages); if (err) goto err_pbl; mhp->attr.pdid = php->pdid; mhp->attr.zbva = 0; mhp->attr.perms = iwch_ib_to_tpt_access(acc); mhp->attr.va_fbo = virt; mhp->attr.page_size = shift - 12; mhp->attr.len = (u32) length; err = iwch_register_mem(rhp, php, mhp, shift); if (err) goto err_pbl; if (udata && !t3a_device(rhp)) { uresp.pbl_addr = (mhp->attr.pbl_addr - rhp->rdev.rnic_info.pbl_base) >> 3; CTR2(KTR_IW_CXGB, "%s user resp pbl_addr 0x%x", __FUNCTION__, uresp.pbl_addr); if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) { iwch_dereg_mr(&mhp->ibmr); err = EFAULT; goto err; } } return &mhp->ibmr; err_pbl: iwch_free_pbl(mhp); err: ib_umem_release(mhp->umem); cxfree(mhp); return ERR_PTR(-err); } static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc) { struct ib_phys_buf bl; u64 kva; struct ib_mr *ibmr; CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd); /* * T3 only supports 32 bits of size. */ bl.size = 0xffffffff; bl.addr = 0; kva = 0; ibmr = iwch_register_phys_mem(pd, &bl, 1, acc, &kva); return ibmr; } static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd) { struct iwch_dev *rhp; struct iwch_pd *php; struct iwch_mw *mhp; u32 mmid; u32 stag = 0; int ret; php = to_iwch_pd(pd); rhp = php->rhp; mhp = malloc(sizeof(*mhp), M_DEVBUF, M_ZERO|M_NOWAIT); if (!mhp) return ERR_PTR(-ENOMEM); ret = cxio_allocate_window(&rhp->rdev, &stag, php->pdid); if (ret) { cxfree(mhp); return ERR_PTR(-ret); } mhp->rhp = rhp; mhp->attr.pdid = php->pdid; mhp->attr.type = TPT_MW; mhp->attr.stag = stag; mmid = (stag) >> 8; mhp->ibmw.rkey = stag; if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { cxio_deallocate_window(&rhp->rdev, mhp->attr.stag); cxfree(mhp); return ERR_PTR(-ENOMEM); } CTR4(KTR_IW_CXGB, "%s mmid 0x%x mhp %p stag 0x%x", __FUNCTION__, mmid, mhp, stag); return &(mhp->ibmw); } static int iwch_dealloc_mw(struct ib_mw *mw) { struct iwch_dev *rhp; struct iwch_mw *mhp; u32 mmid; mhp = to_iwch_mw(mw); rhp = mhp->rhp; mmid = (mw->rkey) >> 8; cxio_deallocate_window(&rhp->rdev, mhp->attr.stag); remove_handle(rhp, &rhp->mmidr, mmid); cxfree(mhp); CTR4(KTR_IW_CXGB, "%s ib_mw %p mmid 0x%x ptr %p", __FUNCTION__, mw, mmid, mhp); return 0; } static int iwch_destroy_qp(struct ib_qp *ib_qp) { struct iwch_dev *rhp; struct iwch_qp *qhp; struct iwch_qp_attributes attrs; struct iwch_ucontext *ucontext; qhp = to_iwch_qp(ib_qp); rhp = qhp->rhp; attrs.next_state = IWCH_QP_STATE_ERROR; iwch_modify_qp(rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, &attrs, 0); mtx_lock(&qhp->lock); if (qhp->ep) msleep(qhp, &qhp->lock, 0, "iwch_destroy_qp1", 0); mtx_unlock(&qhp->lock); remove_handle(rhp, &rhp->qpidr, qhp->wq.qpid); mtx_lock(&qhp->lock); if (--qhp->refcnt) msleep(qhp, &qhp->lock, 0, "iwch_destroy_qp2", 0); mtx_unlock(&qhp->lock); ucontext = ib_qp->uobject ? to_iwch_ucontext(ib_qp->uobject->context) : NULL; cxio_destroy_qp(&rhp->rdev, &qhp->wq, ucontext ? &ucontext->uctx : &rhp->rdev.uctx); CTR4(KTR_IW_CXGB, "%s ib_qp %p qpid 0x%0x qhp %p", __FUNCTION__, ib_qp, qhp->wq.qpid, qhp); cxfree(qhp); return 0; } static struct ib_qp *iwch_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, struct ib_udata *udata) { struct iwch_dev *rhp; struct iwch_qp *qhp; struct iwch_pd *php; struct iwch_cq *schp; struct iwch_cq *rchp; struct iwch_create_qp_resp uresp; int wqsize, sqsize, rqsize; struct iwch_ucontext *ucontext; CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd); if (attrs->qp_type != IB_QPT_RC) return ERR_PTR(-EINVAL); php = to_iwch_pd(pd); rhp = php->rhp; schp = get_chp(rhp, ((struct iwch_cq *) attrs->send_cq)->cq.cqid); rchp = get_chp(rhp, ((struct iwch_cq *) attrs->recv_cq)->cq.cqid); if (!schp || !rchp) return ERR_PTR(-EINVAL); /* The RQT size must be # of entries + 1 rounded up to a power of two */ rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr); if (rqsize == attrs->cap.max_recv_wr) rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr+1); /* T3 doesn't support RQT depth < 16 */ if (rqsize < 16) rqsize = 16; if (rqsize > T3_MAX_RQ_SIZE) return ERR_PTR(-EINVAL); if (attrs->cap.max_inline_data > T3_MAX_INLINE) return ERR_PTR(-EINVAL); /* * NOTE: The SQ and total WQ sizes don't need to be * a power of two. However, all the code assumes * they are. EG: Q_FREECNT() and friends. */ sqsize = roundup_pow_of_two(attrs->cap.max_send_wr); wqsize = roundup_pow_of_two(rqsize + sqsize); CTR4(KTR_IW_CXGB, "%s wqsize %d sqsize %d rqsize %d", __FUNCTION__, wqsize, sqsize, rqsize); qhp = malloc(sizeof(*qhp), M_DEVBUF, M_ZERO|M_NOWAIT); if (!qhp) return ERR_PTR(-ENOMEM); qhp->wq.size_log2 = ilog2(wqsize); qhp->wq.rq_size_log2 = ilog2(rqsize); qhp->wq.sq_size_log2 = ilog2(sqsize); ucontext = pd->uobject ? to_iwch_ucontext(pd->uobject->context) : NULL; if (cxio_create_qp(&rhp->rdev, !udata, &qhp->wq, ucontext ? &ucontext->uctx : &rhp->rdev.uctx)) { cxfree(qhp); return ERR_PTR(-ENOMEM); } attrs->cap.max_recv_wr = rqsize - 1; attrs->cap.max_send_wr = sqsize; attrs->cap.max_inline_data = T3_MAX_INLINE; qhp->rhp = rhp; qhp->attr.pd = php->pdid; qhp->attr.scq = ((struct iwch_cq *) attrs->send_cq)->cq.cqid; qhp->attr.rcq = ((struct iwch_cq *) attrs->recv_cq)->cq.cqid; qhp->attr.sq_num_entries = attrs->cap.max_send_wr; qhp->attr.rq_num_entries = attrs->cap.max_recv_wr; qhp->attr.sq_max_sges = attrs->cap.max_send_sge; qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge; qhp->attr.rq_max_sges = attrs->cap.max_recv_sge; qhp->attr.state = IWCH_QP_STATE_IDLE; qhp->attr.next_state = IWCH_QP_STATE_IDLE; /* * XXX - These don't get passed in from the openib user * at create time. The CM sets them via a QP modify. * Need to fix... I think the CM should */ qhp->attr.enable_rdma_read = 1; qhp->attr.enable_rdma_write = 1; qhp->attr.enable_bind = 1; qhp->attr.max_ord = 1; qhp->attr.max_ird = 1; mtx_init(&qhp->lock, "cxgb qp", NULL, MTX_DEF|MTX_DUPOK); qhp->refcnt = 1; if (insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.qpid)) { cxio_destroy_qp(&rhp->rdev, &qhp->wq, ucontext ? &ucontext->uctx : &rhp->rdev.uctx); cxfree(qhp); return ERR_PTR(-ENOMEM); } if (udata) { struct iwch_mm_entry *mm1, *mm2; mm1 = kmalloc(sizeof *mm1, M_NOWAIT); if (!mm1) { iwch_destroy_qp(&qhp->ibqp); return ERR_PTR(-ENOMEM); } mm2 = kmalloc(sizeof *mm2, M_NOWAIT); if (!mm2) { cxfree(mm1); iwch_destroy_qp(&qhp->ibqp); return ERR_PTR(-ENOMEM); } uresp.qpid = qhp->wq.qpid; uresp.size_log2 = qhp->wq.size_log2; uresp.sq_size_log2 = qhp->wq.sq_size_log2; uresp.rq_size_log2 = qhp->wq.rq_size_log2; mtx_lock(&ucontext->mmap_lock); uresp.key = ucontext->key; ucontext->key += PAGE_SIZE; uresp.db_key = ucontext->key; ucontext->key += PAGE_SIZE; mtx_unlock(&ucontext->mmap_lock); if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) { cxfree(mm1); cxfree(mm2); iwch_destroy_qp(&qhp->ibqp); return ERR_PTR(-EFAULT); } mm1->key = uresp.key; mm1->addr = vtophys(qhp->wq.queue); mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr)); insert_mmap(ucontext, mm1); mm2->key = uresp.db_key; mm2->addr = qhp->wq.udb & PAGE_MASK; mm2->len = PAGE_SIZE; insert_mmap(ucontext, mm2); } qhp->ibqp.qp_num = qhp->wq.qpid; callout_init(&(qhp->timer), TRUE); CTR6(KTR_IW_CXGB, "sq_num_entries %d, rq_num_entries %d " "qpid 0x%0x qhp %p dma_addr 0x%llx size %d", qhp->attr.sq_num_entries, qhp->attr.rq_num_entries, qhp->wq.qpid, qhp, (unsigned long long) qhp->wq.dma_addr, 1 << qhp->wq.size_log2); return &qhp->ibqp; } static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { struct iwch_dev *rhp; struct iwch_qp *qhp; enum iwch_qp_attr_mask mask = 0; struct iwch_qp_attributes attrs; CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, ibqp); /* iwarp does not support the RTR state */ if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR)) attr_mask &= ~IB_QP_STATE; /* Make sure we still have something left to do */ if (!attr_mask) return 0; memset(&attrs, 0, sizeof attrs); qhp = to_iwch_qp(ibqp); rhp = qhp->rhp; attrs.next_state = iwch_convert_state(attr->qp_state); attrs.enable_rdma_read = (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) ? 1 : 0; attrs.enable_rdma_write = (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0; mask |= (attr_mask & IB_QP_STATE) ? IWCH_QP_ATTR_NEXT_STATE : 0; mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ? (IWCH_QP_ATTR_ENABLE_RDMA_READ | IWCH_QP_ATTR_ENABLE_RDMA_WRITE | IWCH_QP_ATTR_ENABLE_RDMA_BIND) : 0; return iwch_modify_qp(rhp, qhp, mask, &attrs, 0); } void iwch_qp_add_ref(struct ib_qp *qp) { CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, qp); mtx_lock(&to_iwch_qp(qp)->lock); to_iwch_qp(qp)->refcnt++; mtx_unlock(&to_iwch_qp(qp)->lock); } void iwch_qp_rem_ref(struct ib_qp *qp) { CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, qp); mtx_lock(&to_iwch_qp(qp)->lock); if (--to_iwch_qp(qp)->refcnt == 0) wakeup(to_iwch_qp(qp)); mtx_unlock(&to_iwch_qp(qp)->lock); } static struct ib_qp *iwch_get_qp(struct ib_device *dev, int qpn) { CTR3(KTR_IW_CXGB, "%s ib_dev %p qpn 0x%x", __FUNCTION__, dev, qpn); return (struct ib_qp *)get_qhp(to_iwch_dev(dev), qpn); } static int iwch_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 * pkey) { CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev); *pkey = 0; return 0; } static int iwch_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid) { struct iwch_dev *dev; struct port_info *pi; struct adapter *sc; CTR5(KTR_IW_CXGB, "%s ibdev %p, port %d, index %d, gid %p", __FUNCTION__, ibdev, port, index, gid); dev = to_iwch_dev(ibdev); sc = dev->rdev.adap; PANIC_IF(port == 0 || port > 2); pi = &sc->port[port - 1]; memset(&(gid->raw[0]), 0, sizeof(gid->raw)); memcpy(&(gid->raw[0]), pi->hw_addr, 6); return 0; } static int iwch_query_device(struct ib_device *ibdev, struct ib_device_attr *props) { struct iwch_dev *dev; struct adapter *sc; CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev); dev = to_iwch_dev(ibdev); sc = dev->rdev.adap; memset(props, 0, sizeof *props); memcpy(&props->sys_image_guid, sc->port[0].hw_addr, 6); props->device_cap_flags = dev->device_cap_flags; props->page_size_cap = dev->attr.mem_pgsizes_bitmask; props->vendor_id = pci_get_vendor(sc->dev); props->vendor_part_id = pci_get_device(sc->dev); props->max_mr_size = dev->attr.max_mr_size; props->max_qp = dev->attr.max_qps; props->max_qp_wr = dev->attr.max_wrs; props->max_sge = dev->attr.max_sge_per_wr; props->max_sge_rd = 1; props->max_qp_rd_atom = dev->attr.max_rdma_reads_per_qp; props->max_qp_init_rd_atom = dev->attr.max_rdma_reads_per_qp; props->max_cq = dev->attr.max_cqs; props->max_cqe = dev->attr.max_cqes_per_cq; props->max_mr = dev->attr.max_mem_regs; props->max_pd = dev->attr.max_pds; props->local_ca_ack_delay = 0; return 0; } static int iwch_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) { CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev); memset(props, 0, sizeof(struct ib_port_attr)); props->max_mtu = IB_MTU_4096; props->active_mtu = IB_MTU_2048; props->state = IB_PORT_ACTIVE; props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_SNMP_TUNNEL_SUP | IB_PORT_REINIT_SUP | IB_PORT_DEVICE_MGMT_SUP | IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP; props->gid_tbl_len = 1; props->pkey_tbl_len = 1; props->active_width = 2; props->active_speed = 2; props->max_msg_sz = -1; return 0; } int iwch_register_device(struct iwch_dev *dev) { int ret; struct adapter *sc = dev->rdev.adap; CTR2(KTR_IW_CXGB, "%s iwch_dev %p", __FUNCTION__, dev); strlcpy(dev->ibdev.name, "cxgb3_%d", IB_DEVICE_NAME_MAX); memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid)); memcpy(&dev->ibdev.node_guid, sc->port[0].hw_addr, 6); dev->device_cap_flags = (IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW); dev->ibdev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | (1ull << IB_USER_VERBS_CMD_REG_MR) | (1ull << IB_USER_VERBS_CMD_DEREG_MR) | (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | (1ull << IB_USER_VERBS_CMD_CREATE_QP) | (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | (1ull << IB_USER_VERBS_CMD_POLL_CQ) | (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | (1ull << IB_USER_VERBS_CMD_POST_SEND) | (1ull << IB_USER_VERBS_CMD_POST_RECV); dev->ibdev.node_type = RDMA_NODE_RNIC; memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC)); dev->ibdev.phys_port_cnt = sc->params.nports; dev->ibdev.num_comp_vectors = 1; dev->ibdev.dma_device = dev->rdev.adap->dev; dev->ibdev.query_device = iwch_query_device; dev->ibdev.query_port = iwch_query_port; dev->ibdev.modify_port = iwch_modify_port; dev->ibdev.query_pkey = iwch_query_pkey; dev->ibdev.query_gid = iwch_query_gid; dev->ibdev.alloc_ucontext = iwch_alloc_ucontext; dev->ibdev.dealloc_ucontext = iwch_dealloc_ucontext; dev->ibdev.mmap = iwch_mmap; dev->ibdev.alloc_pd = iwch_allocate_pd; dev->ibdev.dealloc_pd = iwch_deallocate_pd; dev->ibdev.create_ah = iwch_ah_create; dev->ibdev.destroy_ah = iwch_ah_destroy; dev->ibdev.create_qp = iwch_create_qp; dev->ibdev.modify_qp = iwch_ib_modify_qp; dev->ibdev.destroy_qp = iwch_destroy_qp; dev->ibdev.create_cq = iwch_create_cq; dev->ibdev.destroy_cq = iwch_destroy_cq; dev->ibdev.resize_cq = iwch_resize_cq; dev->ibdev.poll_cq = iwch_poll_cq; dev->ibdev.get_dma_mr = iwch_get_dma_mr; dev->ibdev.reg_phys_mr = iwch_register_phys_mem; dev->ibdev.rereg_phys_mr = iwch_reregister_phys_mem; dev->ibdev.reg_user_mr = iwch_reg_user_mr; dev->ibdev.dereg_mr = iwch_dereg_mr; dev->ibdev.alloc_mw = iwch_alloc_mw; dev->ibdev.bind_mw = iwch_bind_mw; dev->ibdev.dealloc_mw = iwch_dealloc_mw; dev->ibdev.attach_mcast = iwch_multicast_attach; dev->ibdev.detach_mcast = iwch_multicast_detach; dev->ibdev.process_mad = iwch_process_mad; dev->ibdev.req_notify_cq = iwch_arm_cq; dev->ibdev.post_send = iwch_post_send; dev->ibdev.post_recv = iwch_post_receive; dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION; dev->ibdev.iwcm = kmalloc(sizeof(struct iw_cm_verbs), M_NOWAIT); if (!dev->ibdev.iwcm) return (ENOMEM); dev->ibdev.iwcm->connect = iwch_connect; dev->ibdev.iwcm->accept = iwch_accept_cr; dev->ibdev.iwcm->reject = iwch_reject_cr; dev->ibdev.iwcm->create_listen = iwch_create_listen; dev->ibdev.iwcm->destroy_listen = iwch_destroy_listen; dev->ibdev.iwcm->add_ref = iwch_qp_add_ref; dev->ibdev.iwcm->rem_ref = iwch_qp_rem_ref; dev->ibdev.iwcm->get_qp = iwch_get_qp; - ret = ib_register_device(&dev->ibdev); + ret = ib_register_device(&dev->ibdev, NULL); if (ret) goto bail1; return (0); bail1: cxfree(dev->ibdev.iwcm); return (ret); } void iwch_unregister_device(struct iwch_dev *dev) { ib_unregister_device(&dev->ibdev); cxfree(dev->ibdev.iwcm); return; } #endif Index: stable/9/sys/dev/vxge/include/vxgehal-ll.h =================================================================== --- stable/9/sys/dev/vxge/include/vxgehal-ll.h (revision 273863) +++ stable/9/sys/dev/vxge/include/vxgehal-ll.h (revision 273864) @@ -1,6121 +1,6121 @@ /*- * Copyright(c) 2002-2011 Exar Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification are permitted provided the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * 3. Neither the name of the Exar Corporation nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*$FreeBSD$*/ #ifndef VXGE_HAL_LL_H #define VXGE_HAL_LL_H #include #include #include #include "vxgehal-status.h" #include #include #include #include #include #include #include #include __EXTERN_BEGIN_DECLS /* * Driver */ /* * enum vxge_hal_xpak_alarm_type_e - XPAK Alarm types * @VXGE_HAL_XPAK_ALARM_EXCESS_TEMP: Excess temparature alarm * @VXGE_HAL_XPAK_ALARM_EXCESS_BIAS_CURRENT: Excess bias current alarm * @VXGE_HAL_XPAK_ALARM_EXCESS_LASER_OUTPUT: Excess laser output alarm * * XPAK alarm types */ typedef enum vxge_hal_xpak_alarm_type_e { VXGE_HAL_XPAK_ALARM_EXCESS_TEMP = 1, VXGE_HAL_XPAK_ALARM_EXCESS_BIAS_CURRENT = 2, VXGE_HAL_XPAK_ALARM_EXCESS_LASER_OUTPUT = 3, } vxge_hal_xpak_alarm_type_e; /* * function vxge_uld_sched_timer_cb_f - Per-device periodic timer * callback. * @devh: HAL device handle. * @userdata: Per-device user data (a.k.a. context) specified via * vxge_hal_device_initialize(). * * Periodic or one-shot timer callback. If specified (that is, not NULL) * HAL invokes this callback periodically. The call is performed in the * interrupt context, or more exactly, in the context of HAL's ISR * vxge_hal_device_continue_irq(). * * See also: vxge_hal_device_initialize {} */ typedef void (*vxge_uld_sched_timer_cb_f) ( vxge_hal_device_h devh, void *userdata); /* * function vxge_uld_link_up_f - Link-Up callback provided by upper-layer * driver. * @devh: HAL device handle. * @userdata: Opaque context set by the ULD via * vxge_hal_device_private_set() * (typically - at HAL device iinitialization time). * * Link-up notification callback provided by the ULD. * This is one of the per-driver callbacks, see vxge_hal_uld_cbs_t {}. * * See also: vxge_hal_uld_cbs_t {}, vxge_uld_link_down_f {}, * vxge_hal_driver_initialize(), vxge_hal_device_private_set(). */ typedef void (*vxge_uld_link_up_f) ( vxge_hal_device_h devh, void *userdata); /* * function vxge_uld_link_down_f - Link-Down callback provided by * upper-layer driver. * @devh: HAL device handle. * @userdata: Opaque context set by the ULD via * vxge_hal_device_private_set() * (typically - at HAL device iinitialization time). * * Link-Down notification callback provided by the upper-layer driver. * This is one of the per-driver callbacks, see vxge_hal_uld_cbs_t {}. * * See also: vxge_hal_uld_cbs_t {}, vxge_uld_link_up_f {}, * vxge_hal_driver_initialize(), vxge_hal_device_private_set(). */ typedef void (*vxge_uld_link_down_f) ( vxge_hal_device_h devh, void *userdata); /* * function vxge_uld_crit_err_f - Critical Error notification callback. * @devh: HAL device handle. * @userdata: Opaque context set by the ULD via * vxge_hal_device_private_set() * (typically - at HAL device iinitialization time). * @type: Enumerated hw error, e.g.: double ECC. * @serr_data: X3100 status. * @ext_data: Extended data. The contents depends on the @type. * * Critical error notification callback provided by the upper-layer driver. * This is one of the per-driver callbacks, see vxge_hal_uld_cbs_t {}. * * See also: vxge_hal_uld_cbs_t {}, vxge_hal_event_e {}, * vxge_hal_device_private_set(), vxge_hal_driver_initialize(). */ typedef void (*vxge_uld_crit_err_f) ( vxge_hal_device_h devh, void *userdata, vxge_hal_event_e type, u64 ext_data); /* * function vxge_uld_xpak_alarm_log_f - ULD "XPAK alarm log" callback. * @devh: HAL device handle. * @port: Port number * @type: XPAK Alarm type * * Unless NULL is specified, HAL invokes the callback after checking XPAK * counters */ typedef void (*vxge_uld_xpak_alarm_log_f) ( vxge_hal_device_h devh, u32 port, vxge_hal_xpak_alarm_type_e type); /* * struct vxge_hal_uld_cbs_t - Upper-layer driver "slow-path" callbacks. * @link_up: See vxge_uld_link_up_f {}. * @link_down: See vxge_uld_link_down_f {}. * @crit_err: See vxge_uld_crit_err_f {}. * @sched_timer: See vxge_uld_sched_timer_cb_f {}. * @xpak_alarm_log: * * Upper layer driver slow-path (per-driver) callbacks. * Implemented by ULD and provided to HAL via * vxge_hal_driver_initialize(). * Note that these callbacks are not mandatory: HAL will not invoke * a callback if NULL is specified. * * See also: vxge_hal_driver_initialize(). */ typedef struct vxge_hal_uld_cbs_t { vxge_uld_link_up_f link_up; vxge_uld_link_down_f link_down; vxge_uld_crit_err_f crit_err; vxge_uld_sched_timer_cb_f sched_timer; vxge_uld_xpak_alarm_log_f xpak_alarm_log; } vxge_hal_uld_cbs_t; /* * vxge_hal_driver_initialize - Initialize HAL. * @config: HAL configuration, see vxge_hal_driver_config_t {}. * @uld_callbacks: Upper-layer driver callbacks, e.g. link-up. * * HAL initialization entry point. Not to confuse with device initialization * (note that HAL "contains" zero or more X3100 devices). * * Returns: VXGE_HAL_OK - success; * VXGE_HAL_ERR_BAD_DRIVER_CONFIG - Driver configuration params invalid. * */ vxge_hal_status_e vxge_hal_driver_initialize( vxge_hal_driver_config_t *config, vxge_hal_uld_cbs_t *uld_callbacks); /* * vxge_hal_driver_debug_set - Set the debug module, level and timestamp * @level: Debug level as defined in enum vxge_debug_level_e * * This routine is used to dynamically change the debug output */ void vxge_hal_driver_debug_set( vxge_debug_level_e level); /* * vxge_hal_driver_debug_get - Get the debug level * * This routine returns the current debug level set */ u32 vxge_hal_driver_debug_get(void); /* * vxge_hal_driver_terminate - Terminate HAL. * * HAL termination entry point. */ void vxge_hal_driver_terminate(void); void * vxge_hal_device_get_legacy_reg(pci_dev_h pdev, pci_reg_h regh, u8 *bar0); /* * RX Descriptor */ /* * enum vxge_hal_rxd_state_e - Descriptor (RXD) state. * @VXGE_HAL_RXD_STATE_NONE: Invalid state. * @VXGE_HAL_RXD_STATE_AVAIL: Descriptor is available for reservation. * @VXGE_HAL_RXD_STATE_POSTED: Descriptor is posted for processing by the * device. * @VXGE_HAL_RXD_STATE_FREED: Descriptor is free and can be reused for * filling-in and posting later. * * X3100/HAL descriptor states. * */ typedef enum vxge_hal_rxd_state_e { VXGE_HAL_RXD_STATE_NONE = 0, VXGE_HAL_RXD_STATE_AVAIL = 1, VXGE_HAL_RXD_STATE_POSTED = 2, VXGE_HAL_RXD_STATE_FREED = 3 } vxge_hal_rxd_state_e; /* * Ring */ /* * struct vxge_hal_ring_rxd_info_t - Extended information associated with a * completed ring descriptor. * @syn_flag: SYN flag * @is_icmp: Is ICMP * @fast_path_eligible: Fast Path Eligible flag * @l3_cksum_valid: in L3 checksum is valid * @l3_cksum: Result of IP checksum check (by X3100 hardware). * This field containing VXGE_HAL_L3_CKSUM_OK would mean that * the checksum is correct, otherwise - the datagram is * corrupted. * @l4_cksum_valid: in L4 checksum is valid * @l4_cksum: Result of TCP/UDP checksum check (by X3100 hardware). * This field containing VXGE_HAL_L4_CKSUM_OK would mean that * the checksum is correct. Otherwise - the packet is * corrupted. * @frame: Zero or more of vxge_hal_frame_type_e flags. * See vxge_hal_frame_type_e {}. * @proto: zero or more of vxge_hal_frame_proto_e flags. Reporting bits for * various higher-layer protocols, including (but note restricted to) * TCP and UDP. See vxge_hal_frame_proto_e {}. * @is_vlan: If vlan tag is valid * @vlan: VLAN tag extracted from the received frame. * @rth_bucket: RTH bucket * @rth_it_hit: Set, If RTH hash value calculated by the X3100 hardware * has a matching entry in the Indirection table. * @rth_spdm_hit: Set, If RTH hash value calculated by the X3100 hardware * has a matching entry in the Socket Pair Direct Match table. * @rth_hash_type: RTH hash code of the function used to calculate the hash. * @rth_value: Receive Traffic Hashing(RTH) hash value. Produced by X3100 * hardware if RTH is enabled. */ typedef struct vxge_hal_ring_rxd_info_t { u32 syn_flag; u32 is_icmp; u32 fast_path_eligible; u32 l3_cksum_valid; u32 l3_cksum; u32 l4_cksum_valid; u32 l4_cksum; u32 frame; u32 proto; u32 is_vlan; #define VXGE_HAL_VLAN_VID_MASK 0xfff u32 vlan; u32 rth_bucket; u32 rth_it_hit; u32 rth_spdm_hit; u32 rth_hash_type; u32 rth_value; } vxge_hal_ring_rxd_info_t; /* * enum vxge_hal_frame_type_e - Ethernet frame format. * @VXGE_HAL_FRAME_TYPE_DIX: DIX (Ethernet II) format. * @VXGE_HAL_FRAME_TYPE_LLC: LLC format. * @VXGE_HAL_FRAME_TYPE_SNAP: SNAP format. * @VXGE_HAL_FRAME_TYPE_IPX: IPX format. * * Ethernet frame format. */ typedef enum vxge_hal_frame_type_e { VXGE_HAL_FRAME_TYPE_DIX = 0x0, VXGE_HAL_FRAME_TYPE_LLC = 0x1, VXGE_HAL_FRAME_TYPE_SNAP = 0x2, VXGE_HAL_FRAME_TYPE_IPX = 0x3, } vxge_hal_frame_type_e; typedef enum vxge_hal_tcp_option_e { VXGE_HAL_TCPOPT_NOP = 1, /* Padding */ VXGE_HAL_TCPOPT_EOL = 0, /* End of options */ VXGE_HAL_TCPOPT_MSS = 2, /* Segment size negotiating */ VXGE_HAL_TCPOPT_WINDOW = 3, /* Window scaling */ VXGE_HAL_TCPOPT_SACK_PERM = 4, /* SACK Permitted */ VXGE_HAL_TCPOPT_SACK = 5, /* SACK Block */ VXGE_HAL_TCPOPT_TIMESTAMP = 8, /* Better RTT estimations/PAWS */ VXGE_HAL_TCPOPT_MD5SIG = 19, /* MD5 Signature (RFC2385) */ VXGE_HAL_TCPOLEN_TIMESTAMP = 10, VXGE_HAL_TCPOLEN_TSTAMP_ALIGNED = 12 } vxge_hal_tcp_option_e; /* * enum vxge_hal_frame_proto_e - Higher-layer ethernet protocols. * @VXGE_HAL_FRAME_PROTO_VLAN_TAGGED: VLAN. * @VXGE_HAL_FRAME_PROTO_IPV4: IPv4. * @VXGE_HAL_FRAME_PROTO_IPV6: IPv6. * @VXGE_HAL_FRAME_PROTO_IP_FRAG: IP fragmented. * @VXGE_HAL_FRAME_PROTO_TCP: TCP. * @VXGE_HAL_FRAME_PROTO_UDP: UDP. * @VXGE_HAL_FRAME_PROTO_TCP_OR_UDP: TCP or UDP. * * Higher layer ethernet protocols and options. */ typedef enum vxge_hal_frame_proto_e { VXGE_HAL_FRAME_PROTO_VLAN_TAGGED = 0x80, VXGE_HAL_FRAME_PROTO_IPV4 = 0x10, VXGE_HAL_FRAME_PROTO_IPV6 = 0x08, VXGE_HAL_FRAME_PROTO_IP_FRAG = 0x04, VXGE_HAL_FRAME_PROTO_TCP = 0x02, VXGE_HAL_FRAME_PROTO_UDP = 0x01, VXGE_HAL_FRAME_PROTO_TCP_OR_UDP = (VXGE_HAL_FRAME_PROTO_TCP | \ VXGE_HAL_FRAME_PROTO_UDP) } vxge_hal_frame_proto_e; /* * enum vxge_hal_ring_tcode_e - Transfer codes returned by adapter * @VXGE_HAL_RING_T_CODE_OK: Transfer ok. * @VXGE_HAL_RING_T_CODE_L3_CKSUM_MISMATCH: Layer 3 checksum presentation * configuration mismatch. * @VXGE_HAL_RING_T_CODE_L4_CKSUM_MISMATCH: Layer 4 checksum presentation * configuration mismatch. * @VXGE_HAL_RING_T_CODE_L3_L4_CKSUM_MISMATCH: Layer 3 and Layer 4 checksum * presentation configuration mismatch. - * @VXGE_HAL_RING_T_CODE_L3_PKT_ERR: Layer 3 error¸unparseable packet, + * @VXGE_HAL_RING_T_CODE_L3_PKT_ERR: Layer 3 error: unparseable packet, * such as unknown IPv6 header. - * @VXGE_HAL_RING_T_CODE_L2_FRM_ERR: Layer 2 error¸frame integrity + * @VXGE_HAL_RING_T_CODE_L2_FRM_ERR: Layer 2 error: frame integrity * error, such as FCS or ECC). - * @VXGE_HAL_RING_T_CODE_BUF_SIZE_ERR: Buffer size error¸the RxD buffer( - * s) were not appropriately sized and data loss occurred. - * @VXGE_HAL_RING_T_CODE_INT_ECC_ERR: Internal ECC error¸RxD corrupted. - * @VXGE_HAL_RING_T_CODE_BENIGN_OVFLOW: Benign overflow¸the contents of + * @VXGE_HAL_RING_T_CODE_BUF_SIZE_ERR: Buffer size error: the RxD buffer(s) + * were not appropriately sized and data loss occurred. + * @VXGE_HAL_RING_T_CODE_INT_ECC_ERR: Internal ECC error: RxD corrupted. + * @VXGE_HAL_RING_T_CODE_BENIGN_OVFLOW: Benign overflow: the contents of * Segment1 exceeded the capacity of Buffer1 and the remainder * was placed in Buffer2. Segment2 now starts in Buffer3. * No data loss or errors occurred. - * @VXGE_HAL_RING_T_CODE_ZERO_LEN_BUFF: Buffer size 0¸one of the RxDs + * @VXGE_HAL_RING_T_CODE_ZERO_LEN_BUFF: Buffer size 0: one of the RxDs * assigned buffers has a size of 0 bytes. - * @VXGE_HAL_RING_T_CODE_FRM_DROP: Frame dropped¸either due to + * @VXGE_HAL_RING_T_CODE_FRM_DROP: Frame dropped: either due to * VPath Reset or because of a VPIN mismatch. * @VXGE_HAL_RING_T_CODE_UNUSED: Unused - * @VXGE_HAL_RING_T_CODE_MULTI_ERR: Multiple errors¸more than one + * @VXGE_HAL_RING_T_CODE_MULTI_ERR: Multiple errors: more than one * transfer code condition occurred. * * Transfer codes returned by adapter. */ typedef enum vxge_hal_ring_tcode_e { VXGE_HAL_RING_T_CODE_OK = 0x0, VXGE_HAL_RING_T_CODE_L3_CKSUM_MISMATCH = 0x1, VXGE_HAL_RING_T_CODE_L4_CKSUM_MISMATCH = 0x2, VXGE_HAL_RING_T_CODE_L3_L4_CKSUM_MISMATCH = 0x3, VXGE_HAL_RING_T_CODE_L3_PKT_ERR = 0x5, VXGE_HAL_RING_T_CODE_L2_FRM_ERR = 0x6, VXGE_HAL_RING_T_CODE_BUF_SIZE_ERR = 0x7, VXGE_HAL_RING_T_CODE_INT_ECC_ERR = 0x8, VXGE_HAL_RING_T_CODE_BENIGN_OVFLOW = 0x9, VXGE_HAL_RING_T_CODE_ZERO_LEN_BUFF = 0xA, VXGE_HAL_RING_T_CODE_FRM_DROP = 0xC, VXGE_HAL_RING_T_CODE_UNUSED = 0xE, VXGE_HAL_RING_T_CODE_MULTI_ERR = 0xF } vxge_hal_ring_tcode_e; /* * enum vxge_hal_ring_hash_type_e - RTH hash types * @VXGE_HAL_RING_HASH_TYPE_NONE: No Hash * @VXGE_HAL_RING_HASH_TYPE_TCP_IPV4: TCP IPv4 * @VXGE_HAL_RING_HASH_TYPE_UDP_IPV4: UDP IPv4 * @VXGE_HAL_RING_HASH_TYPE_IPV4: IPv4 * @VXGE_HAL_RING_HASH_TYPE_TCP_IPV6: TCP IPv6 * @VXGE_HAL_RING_HASH_TYPE_UDP_IPV6: UDP IPv6 * @VXGE_HAL_RING_HASH_TYPE_IPV6: IPv6 * @VXGE_HAL_RING_HASH_TYPE_TCP_IPV6_EX: TCP IPv6 extension * @VXGE_HAL_RING_HASH_TYPE_UDP_IPV6_EX: UDP IPv6 extension * @VXGE_HAL_RING_HASH_TYPE_IPV6_EX: IPv6 extension * * RTH hash types */ typedef enum vxge_hal_ring_hash_type_e { VXGE_HAL_RING_HASH_TYPE_NONE = 0x0, VXGE_HAL_RING_HASH_TYPE_TCP_IPV4 = 0x1, VXGE_HAL_RING_HASH_TYPE_UDP_IPV4 = 0x2, VXGE_HAL_RING_HASH_TYPE_IPV4 = 0x3, VXGE_HAL_RING_HASH_TYPE_TCP_IPV6 = 0x4, VXGE_HAL_RING_HASH_TYPE_UDP_IPV6 = 0x5, VXGE_HAL_RING_HASH_TYPE_IPV6 = 0x6, VXGE_HAL_RING_HASH_TYPE_TCP_IPV6_EX = 0x7, VXGE_HAL_RING_HASH_TYPE_UDP_IPV6_EX = 0x8, VXGE_HAL_RING_HASH_TYPE_IPV6_EX = 0x9 } vxge_hal_ring_hash_type_e; /* * struct vxge_hal_ring_rxd_1_t - One buffer mode RxD for ring * @host_control: This field is exclusively for host use and is "readonly" * from the adapter's perspective. * @control_0:Bits 0 to 6 - RTH_Bucket get * Bit 7 - Own Descriptor ownership bit. This bit is set to 1 by the * host, and is set to 0 by the adapter. * 0 - Host owns RxD and buffer. * 1 - The adapter owns RxD and buffer. * Bit 8 - Fast_Path_Eligible When set, indicates that the received * frame meets all of the criteria for fast path processing. * The required criteria are as follows: * !SYN & * (Transfer_Code == "Transfer OK") & * (!Is_IP_Fragment) & * ((Is_IPv4 & computed_L3_checksum == 0xFFFF) | * (Is_IPv6)) & * ((Is_TCP & computed_L4_checksum == 0xFFFF) | * (Is_UDP & (computed_L4_checksum == 0xFFFF | * computed _L4_checksum == 0x0000))) * (same meaning for all RxD buffer modes) * Bit 9 - L3 Checksum Correct * Bit 10 - L4 Checksum Correct * Bit 11 - Reserved * Bit 12 to 15 - This field is written by the adapter. It is used * to report the status of the frame transfer to the host. * 0x0 - Transfer OK * 0x4 - RDA Failure During Transfer * 0x5 - Unparseable Packet, such as unknown IPv6 header. * 0x6 - Frame integrity error (FCS or ECC). * 0x7 - Buffer Size Error. The provided buffer(s) were not * appropriately sized and data loss occurred. * 0x8 - Internal ECC Error. RxD corrupted. * 0x9 - IPv4 Checksum error * 0xA - TCP/UDP Checksum error * 0xF - Unknown Error or Multiple Error. Indicates an unknown * problem or that more than one of transfer codes is set. * Bit 16 - SYN The adapter sets this field to indicate that the * incoming frame contained a TCP segment with its SYN bit set * and its ACK bit NOT set. (same meaning for all RxD buffer modes) * Bit 17 - Is ICMP * Bit 18 - RTH_SPDM_HIT Set to 1 if there was a match in the * Socket * Pair Direct Match Table and the frame was steered based on SPDM. * Bit 19 - RTH_IT_HIT Set to 1 if there was a match in the * Indirection Table and the frame was steered based on hash * indirection. * Bit 20 to 23 - RTH_HASH_TYPE Indicates the function (hash type) * that was used to calculate the hash. * Bit 19 - IS_VLAN Set to '1' if the frame was/is VLAN tagged. * Bit 25 to 26 - ETHER_ENCAP Reflects the Ethernet encapsulation * of the received frame. * 0x0 - Ethernet DIX * 0x1 - LLC * 0x2 - SNAP (includes Jumbo-SNAP) * 0x3 - IPX * Bit 27 - IS_IPV4 Set to '1' if the frame contains IPv4 packet. * Bit 28 - IS_IPV6 Set to '1' if the frame contains IPv6 packet. * Bit 29 - IS_IP_FRAG Set to '1' if the frame contains a * fragmented IP packet. * Bit 30 - IS_TCP Set to '1' if the frame contains a TCP segment. * Bit 31 - IS_UDP Set to '1' if the frame contains a UDP message. * Bit 32 to 47 - L3_Checksum[0:15] The IPv4 checksum value that * arrived with the frame. If the resulting computed IPv4 header * checksum for the frame did not produce the expected 0xFFFF value, * then the transfer code would be set to 0x9. * Bit 48 to 63 - L4_Checksum[0:15] TCP/UDP checksum value that * arrived with the frame. If the resulting computed TCP/UDP checksum * for the frame did not produce the expected 0xFFFF value, then the * transfer code would be set to 0xA. * @control_1:Bits 0 to 1 - Reserved * Bits 2 to 15 - Buffer0_Size.This field is set by the host and * eventually overwritten by the adapter. The host writes the * available buffer size in bytes when it passes the descriptor to * the adapter. When a frame is delivered the host, the adapter * populates this field with the number of bytes written into the * buffer. The largest supported buffer is 16, 383 bytes. * Bit 16 to 47 - RTH Hash Value 32-bit RTH hash value. Only valid * if RTH_HASH_TYPE (Control_0, bits 20:23) is nonzero. * Bit 48 to 63 - VLAN_Tag[0:15] The contents of the variable * portion of the VLAN tag, if one was detected by the adapter. * This field is populated even if VLAN-tag stripping is enabled. * @buffer0_ptr: Pointer to buffer. This field is populated by the driver. * * One buffer mode RxD for ring structure */ typedef struct vxge_hal_ring_rxd_1_t { u64 host_control; u64 control_0; #define VXGE_HAL_RING_RXD_RTH_BUCKET_GET(ctrl0) bVAL7(ctrl0, 0) #define VXGE_HAL_RING_RXD_RTH_BUCKET_ADAPTER vBIT(val, 0, 7) #define VXGE_HAL_RING_RXD_LIST_OWN_GET(ctrl0) bVAL1(ctrl0, 7) #define VXGE_HAL_RING_RXD_LIST_OWN_ADAPTER mBIT(7) #define VXGE_HAL_RING_RXD_FAST_PATH_ELIGIBLE_GET(ctrl0) bVAL1(ctrl0, 8) #define VXGE_HAL_RING_RXD_FAST_PATH_ELIGIBLE mBIT(8) #define VXGE_HAL_RING_RXD_L3_CKSUM_CORRECT_GET(ctrl0) bVAL1(ctrl0, 9) #define VXGE_HAL_RING_RXD_L3_CKSUM_CORRECT mBIT(9) #define VXGE_HAL_RING_RXD_L4_CKSUM_CORRECT_GET(ctrl0) bVAL1(ctrl0, 10) #define VXGE_HAL_RING_RXD_L4_CKSUM_CORRECT mBIT(10) #define VXGE_HAL_RING_RXD_T_CODE_GET(ctrl0) bVAL4(ctrl0, 12) #define VXGE_HAL_RING_RXD_T_CODE(val) vBIT(val, 12, 4) #define VXGE_HAL_RING_RXD_T_CODE_OK VXGE_HAL_RING_T_CODE_OK #define VXGE_HAL_RING_RXD_T_CODE_L3_CKSUM_MISMATCH \ VXGE_HAL_RING_T_CODE_L3_CKSUM_MISMATCH #define VXGE_HAL_RING_RXD_T_CODE_L4_CKSUM_MISMATCH \ VXGE_HAL_RING_T_CODE_L4_CKSUM_MISMATCH #define VXGE_HAL_RING_RXD_T_CODE_L3_L4_CKSUM_MISMATCH \ VXGE_HAL_RING_T_CODE_L3_L4_CKSUM_MISMATCH #define VXGE_HAL_RING_RXD_T_CODE_L3_PKT_ERR VXGE_HAL_RING_T_CODE_L3_PKT_ERR #define VXGE_HAL_RING_RXD_T_CODE_L2_FRM_ERR VXGE_HAL_RING_T_CODE_L2_FRM_ERR #define VXGE_HAL_RING_RXD_T_CODE_BUF_SIZE_ERR \ VXGE_HAL_RING_T_CODE_BUF_SIZE_ERR #define VXGE_HAL_RING_RXD_T_CODE_INT_ECC_ERR VXGE_HAL_RING_T_CODE_INT_ECC_ERR #define VXGE_HAL_RING_RXD_T_CODE_BENIGN_OVFLOW \ VXGE_HAL_RING_T_CODE_BENIGN_OVFLOW #define VXGE_HAL_RING_RXD_T_CODE_ZERO_LEN_BUFF \ VXGE_HAL_RING_T_CODE_ZERO_LEN_BUFF #define VXGE_HAL_RING_RXD_T_CODE_FRM_DROP VXGE_HAL_RING_T_CODE_FRM_DROP #define VXGE_HAL_RING_RXD_T_CODE_UNUSED VXGE_HAL_RING_T_CODE_UNUSED #define VXGE_HAL_RING_RXD_T_CODE_MULTI_ERR VXGE_HAL_RING_T_CODE_MULTI_ERR #define VXGE_HAL_RING_RXD_SYN_GET(ctrl0) bVAL1(ctrl0, 16) #define VXGE_HAL_RING_RXD_SYN mBIT(16) #define VXGE_HAL_RING_RXD_IS_ICMP_GET(ctrl0) bVAL1(ctrl0, 17) #define VXGE_HAL_RING_RXD_IS_ICMP mBIT(17) #define VXGE_HAL_RING_RXD_RTH_SPDM_HIT_GET(ctrl0) bVAL1(ctrl0, 18) #define VXGE_HAL_RING_RXD_RTH_SPDM_HIT mBIT(18) #define VXGE_HAL_RING_RXD_RTH_IT_HIT_GET(ctrl0) bVAL1(ctrl0, 19) #define VXGE_HAL_RING_RXD_RTH_IT_HIT mBIT(19) #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_GET(ctrl0) bVAL4(ctrl0, 20) #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE(val) vBIT(val, 20, 4) #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_NONE VXGE_HAL_RING_HASH_TYPE_NONE #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_TCP_IPV4 \ VXGE_HAL_RING_HASH_TYPE_TCP_IPV4 #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_UDP_IPV4 \ VXGE_HAL_RING_HASH_TYPE_UDP_IPV4 #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_IPV4 VXGE_HAL_RING_HASH_TYPE_IPV4 #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_TCP_IPV6 \ VXGE_HAL_RING_HASH_TYPE_TCP_IPV6 #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_UDP_IPV6 \ VXGE_HAL_RING_HASH_TYPE_UDP_IPV6 #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_IPV6 VXGE_HAL_RING_HASH_TYPE_IPV6 #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_TCP_IPV6_EX \ VXGE_HAL_RING_HASH_TYPE_TCP_IPV6_EX #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_UDP_IPV6_EX \ VXGE_HAL_RING_HASH_TYPE_UDP_IPV6_EX #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_IPV6_EX VXGE_HAL_RING_HASH_TYPE_IPV6_EX #define VXGE_HAL_RING_RXD_IS_VLAN_GET(ctrl0) bVAL1(ctrl0, 24) #define VXGE_HAL_RING_RXD_IS_VLAN mBIT(24) #define VXGE_HAL_RING_RXD_ETHER_ENCAP_GET(ctrl0) bVAL2(ctrl0, 25) #define VXGE_HAL_RING_RXD_ETHER_ENCAP(val) vBIT(val, 25, 2) #define VXGE_HAL_RING_RXD_ETHER_ENCAP_DIX VXGE_HAL_FRAME_TYPE_DIX #define VXGE_HAL_RING_RXD_ETHER_ENCAP_LLC VXGE_HAL_FRAME_TYPE_LLC #define VXGE_HAL_RING_RXD_ETHER_ENCAP_SNAP VXGE_HAL_FRAME_TYPE_SNAP #define VXGE_HAL_RING_RXD_ETHER_ENCAP_IPX VXGE_HAL_FRAME_TYPE_IPX #define VXGE_HAL_RING_RXD_IS_IPV4_GET(ctrl0) bVAL1(ctrl0, 27) #define VXGE_HAL_RING_RXD_IS_IPV4 mBIT(27) #define VXGE_HAL_RING_RXD_IS_IPV6_GET(ctrl0) bVAL1(ctrl0, 28) #define VXGE_HAL_RING_RXD_IS_IPV6 mBIT(28) #define VXGE_HAL_RING_RXD_IS_IPV_FRAG_GET(ctrl0) bVAL1(ctrl0, 29) #define VXGE_HAL_RING_RXD_IS_IPV_FRAG mBIT(29) #define VXGE_HAL_RING_RXD_IS_TCP_GET(ctrl0) bVAL1(ctrl0, 30) #define VXGE_HAL_RING_RXD_IS_TCP mBIT(30) #define VXGE_HAL_RING_RXD_IS_UDP_GET(ctrl0) bVAL1(ctrl0, 31) #define VXGE_HAL_RING_RXD_IS_UDP mBIT(31) #define VXGE_HAL_RING_RXD_FRAME_PROTO_GET(ctrl0) bVAL5(ctrl0, 27) #define VXGE_HAL_RING_RXD_FRAME_PROTO(val) vBIT(val, 27, 5) #define VXGE_HAL_RING_RXD_FRAME_PROTO_IPV4 VXGE_HAL_FRAME_PROTO_IPV4 #define VXGE_HAL_RING_RXD_FRAME_PROTO_IPV6 VXGE_HAL_FRAME_PROTO_IPV6 #define VXGE_HAL_RING_RXD_FRAME_PROTO_IP_FRAG VXGE_HAL_FRAME_PROTO_IP_FRAG #define VXGE_HAL_RING_RXD_FRAME_PROTO_TCP VXGE_HAL_FRAME_PROTO_TCP #define VXGE_HAL_RING_RXD_FRAME_PROTO_UDP VXGE_HAL_FRAME_PROTO_UDP #define VXGE_HAL_RING_RXD_FRAME_PROTO_TCP_OR_UDP (VXGE_HAL_FRAME_PROTO_TCP |\ VXGE_HAL_FRAME_PROTO_UDP) #define VXGE_HAL_RING_RXD_L3_CKSUM_GET(ctrl0) bVAL16(ctrl0, 32) #define VXGE_HAL_RING_RXD_L3_CKSUM(val) vBIT(val, 32, 16) #define VXGE_HAL_RING_RXD_L4_CKSUM_GET(ctrl0) bVAL16(ctrl0, 48) #define VXGE_HAL_RING_RXD_L4_CKSUM(val) vBIT(val, 48, 16) u64 control_1; #define VXGE_HAL_RING_RXD_LIST_TAIL_OWN_ADAPTER mBIT(0) #define VXGE_HAL_RING_RXD_1_BUFFER0_SIZE_GET(ctrl1) bVAL14(ctrl1, 2) #define VXGE_HAL_RING_RXD_1_BUFFER0_SIZE(val) vBIT(val, 2, 14) #define VXGE_HAL_RING_RXD_1_BUFFER0_SIZE_MASK vBIT(0x3FFF, 2, 14) #define VXGE_HAL_RING_RXD_1_RTH_HASH_VAL_GET(ctrl1) bVAL32(ctrl1, 16) #define VXGE_HAL_RING_RXD_1_RTH_HASH_VAL(val) vBIT(val, 16, 32) #define VXGE_HAL_RING_RXD_VLAN_TAG_GET(ctrl1) bVAL16(ctrl1, 48) #define VXGE_HAL_RING_RXD_VLAN_TAG(val) vBIT(val, 48, 16) u64 buffer0_ptr; } vxge_hal_ring_rxd_1_t; /* * struct vxge_hal_ring_rxd_3_t - Three buffer mode RxD for ring * @host_control: This field is exclusively for host use and is "readonly" * from the adapter's perspective. * @control_0:Bits 0 to 6 - RTH_Bucket get * Bit 7 - Own Descriptor ownership bit. This bit is set to 1 * by the host, and is set to 0 by the adapter. * 0 - Host owns RxD and buffer. * 1 - The adapter owns RxD and buffer. * Bit 8 - Fast_Path_Eligible When set, indicates that the * received frame meets all of the criteria for fast path processing. * The required criteria are as follows: * !SYN & * (Transfer_Code == "Transfer OK") & * (!Is_IP_Fragment) & * ((Is_IPv4 & computed_L3_checksum == 0xFFFF) | * (Is_IPv6)) & * ((Is_TCP & computed_L4_checksum == 0xFFFF) | * (Is_UDP & (computed_L4_checksum == 0xFFFF | * computed _L4_checksum == 0x0000))) * (same meaning for all RxD buffer modes) * Bit 9 - L3 Checksum Correct * Bit 10 - L4 Checksum Correct * Bit 11 - Reserved * Bit 12 to 15 - This field is written by the adapter. It is used * to report the status of the frame transfer to the host. * 0x0 - Transfer OK * 0x4 - RDA Failure During Transfer * 0x5 - Unparseable Packet, such as unknown IPv6 header. * 0x6 - Frame integrity error (FCS or ECC). * 0x7 - Buffer Size Error. The provided buffer(s) were not * appropriately sized and data loss occurred. * 0x8 - Internal ECC Error. RxD corrupted. * 0x9 - IPv4 Checksum error * 0xA - TCP/UDP Checksum error * 0xF - Unknown Error or Multiple Error. Indicates an unknown * problem or that more than one of transfer codes is set. * Bit 16 - SYN The adapter sets this field to indicate that the * incoming frame contained a TCP segment with its SYN bit set * and its ACK bit NOT set. (same meaning for all RxD buffer modes) * Bit 17 - Is ICMP * Bit 18 - RTH_SPDM_HIT Set to 1 if there was a match in the * Socket * Pair Direct Match Table and the frame was steered based on SPDM. * Bit 19 - RTH_IT_HIT Set to 1 if there was a match in the * Indirection Table and the frame was steered based on hash * indirection. * Bit 20 to 23 - RTH_HASH_TYPE Indicates the function (hash type) * that was used to calculate the hash. * Bit 19 - IS_VLAN Set to '1' if the frame was/is VLAN tagged. * Bit 25 to 26 - ETHER_ENCAP Reflects the Ethernet encapsulation * of the received frame. * 0x0 - Ethernet DIX * 0x1 - LLC * 0x2 - SNAP (includes Jumbo-SNAP) * 0x3 - IPX * Bit 27 - IS_IPV4 Set to '1' if the frame contains IPv4 packet. * Bit 28 - IS_IPV6 Set to '1' if the frame contains IPv6 packet. * Bit 29 - IS_IP_FRAG Set to '1' if the frame contains a * fragmented IP packet. * Bit 30 - IS_TCP Set to '1' if the frame contains a TCP segment. * Bit 31 - IS_UDP Set to '1' if the frame contains a UDP message. * Bit 32 to 47 - L3_Checksum[0:15] The IPv4 checksum value that * arrived with the frame. If the resulting computed IPv4 header * checksum for the frame did not produce the expected 0xFFFF value, * then the transfer code would be set to 0x9. * Bit 48 to 63 - L4_Checksum[0:15] TCP/UDP checksum value that * arrived with the frame. If the resulting computed TCP/UDP checksum * for the frame did not produce the expected 0xFFFF value, then the * transfer code would be set to 0xA. * @control_1:Bit 0 - This field must be used in conjunction with the Ownership * field (above). * 1 - Set by the host to indicate that the RxD points to fresh * buffers. * 0 - Cleared by the adapter to indicate that frame data has been * placed into the assigned buffers, and that the host once again * owns the descriptor. * (Note: Please observe the usage guidelines outlined in the * Ownership field (above)). * Bit 1 - Unused. Ignored by Adapter on RxD read. Set to 0 * by Adapter on RxD write. * Bits 2 to 15 - This field is written by the host and by X3100. * The host writes the available buffer 0 size in bytes when it * passes the descriptor to the X3100. The X3100 writes the number * of bytes written to the buffer when it passes the descriptor back * to the host. * Bits 16 to 17 - Reserved * Bits 18 to 31 - This field is set by the host and eventually * overwritten by the adapter. The host writes the available * buffer 1size in bytes when it passes the descriptor to the adapter. * When a frame is delivered the host, the adapter populates this field * with the number of bytes written into the buffer 1. The largest * supported buffer is 16, 383 bytes. * Bits 32 to 33 - Reserved * Bits 34 to 47 - This field is set by the host and eventually * overwritten by the adapter. The host writes the available * buffer 2 size in bytes when it passes the descriptor to the adapter. * When a frame is delivered the host, the adapter populates this field * with the number of bytes written into the buffer 2. The largest * supported buffer is 16, 383 bytes. * Bit 48 to 63 - VLAN_Tag[0:15] The contents of the variable * portion of the VLAN tag, if one was detected by the adapter. This * field is populated even if VLAN-tag stripping is enabled. * @buffer0_ptr: Pointer to buffer 0. This field is populated by the driver. * In 3-buffer mode, when the RxD is returned to the host, * buffer0_ptr field will be overwritten if the following conditions * are met: * 1 - RTH_Disable in the PRC_CTRL register is not set. * 2 - RTH is enabled and a valid hash value was calculated for the * frame. This will be indicated by a non-zero value in the * RTH_HASH_TYPE field (Control_0, bits 20:23). In the event that the * pointer is overwritten on return to the host, bits 0:31 will be * all zeroes while bits 32:63 will contain the calculated hash value. * @buffer1_ptr: Pointer to buffer 1. This field is populated by the driver. * @buffer2_ptr: Pointer to buffer 2. This field is populated by the driver. * * Three buffer mode RxD for ring structure */ typedef struct vxge_hal_ring_rxd_3_t { u64 host_control; u64 control_0; /* * The following bit fields are common in all the three buffer modes and are * defined in vxge_hal_ring_rxd_1_t * #define VXGE_HAL_RING_RXD_RTH_BUCKET_GET(ctrl0) bVAL7(ctrl0, 0) * #define VXGE_HAL_RING_RXD_RTH_BUCKET_ADAPTER vBIT(val, 0, 7) * * #define VXGE_HAL_RING_RXD_LIST_OWN_GET(ctrl0) bVAL1(ctrl0, 7) * #define VXGE_HAL_RING_RXD_LIST_OWN_ADAPTER mBIT(7) * * #define VXGE_HAL_RING_RXD_FAST_PATH_ELIGIBLE_GET(ctrl0) \ * bVAL1(ctrl0, 8) * #define VXGE_HAL_RING_RXD_FAST_PATH_ELIGIBLE mBIT(8) * * #define VXGE_HAL_RING_RXD_L3_CKSUM_CORRECT_GET(ctrl0) \ * bVAL1(ctrl0, 9) * #define VXGE_HAL_RING_RXD_L3_CKSUM_CORRECT mBIT(9) * * #define VXGE_HAL_RING_RXD_L4_CKSUM_CORRECT_GET(ctrl0) \ * bVAL1(ctrl0, 10) * #define VXGE_HAL_RING_RXD_L4_CKSUM_CORRECT mBIT(10) * * #define VXGE_HAL_RING_RXD_T_CODE_GET(ctrl0) bVAL4(ctrl0, 12) * #define VXGE_HAL_RING_RXD_T_CODE(val) vBIT(val, 12, 4) * #define VXGE_HAL_RING_RXD_T_CODE_OK VXGE_HAL_RING_T_CODE_OK * #define VXGE_HAL_RING_RXD_T_CODE_L3_CKSUM_MISMATCH \ * VXGE_HAL_RING_T_CODE_L3_CKSUM_MISMATCH * #define VXGE_HAL_RING_RXD_T_CODE_L4_CKSUM_MISMATCH \ * VXGE_HAL_RING_T_CODE_L4_CKSUM_MISMATCH * #define VXGE_HAL_RING_RXD_T_CODE_L3_L4_CKSUM_MISMATCH \ * VXGE_HAL_RING_T_CODE_L3_L4_CKSUM_MISMATCH * #define VXGE_HAL_RING_RXD_T_CODE_L3_PKT_ERR \ * VXGE_HAL_RING_T_CODE_L3_PKT_ERR * #define VXGE_HAL_RING_RXD_T_CODE_L2_FRM_ERR \ * VXGE_HAL_RING_T_CODE_L2_FRM_ERR * #define VXGE_HAL_RING_RXD_T_CODE_BUF_SIZE_ERR \ * VXGE_HAL_RING_T_CODE_BUF_SIZE_ERR * #define VXGE_HAL_RING_RXD_T_CODE_INT_ECC_ERR \ * VXGE_HAL_RING_T_CODE_INT_ECC_ERR * #define VXGE_HAL_RING_RXD_T_CODE_BENIGN_OVFLOW \ * VXGE_HAL_RING_T_CODE_BENIGN_OVFLOW * #define VXGE_HAL_RING_RXD_T_CODE_ZERO_LEN_BUFF \ * VXGE_HAL_RING_T_CODE_ZERO_LEN_BUFF * #define VXGE_HAL_RING_RXD_T_CODE_FRM_DROP VXGE_HAL_RING_T_CODE_FRM_DROP * #define VXGE_HAL_RING_RXD_T_CODE_UNUSED VXGE_HAL_RING_T_CODE_UNUSED * #define VXGE_HAL_RING_RXD_T_CODE_MULTI_ERR \ * VXGE_HAL_RING_T_CODE_MULTI_ERR * * #define VXGE_HAL_RING_RXD_SYN_GET(ctrl0) bVAL1(ctrl0, 16) * #define VXGE_HAL_RING_RXD_SYN mBIT(16) * * #define VXGE_HAL_RING_RXD_IS_ICMP_GET(ctrl0) bVAL1(ctrl0, 17) * #define VXGE_HAL_RING_RXD_IS_ICMP mBIT(17) * * #define VXGE_HAL_RING_RXD_RTH_SPDM_HIT_GET(ctrl0) bVAL1(ctrl0, 18) * #define VXGE_HAL_RING_RXD_RTH_SPDM_HIT mBIT(18) * * #define VXGE_HAL_RING_RXD_RTH_IT_HIT_GET(ctrl0) bVAL1(ctrl0, 19) * #define VXGE_HAL_RING_RXD_RTH_IT_HIT mBIT(19) * * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_GET(ctrl0) bVAL4(ctrl0, 20) * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE(val) vBIT(val, 20, 4) * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_NONE \ * VXGE_HAL_RING_HASH_TYPE_NONE * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_TCP_IPV4 \ * VXGE_HAL_RING_HASH_TYPE_TCP_IPV4 * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_UDP_IPV4 \ * VXGE_HAL_RING_HASH_TYPE_UDP_IPV4 * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_IPV4 \ * VXGE_HAL_RING_HASH_TYPE_IPV4 * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_TCP_IPV6 \ * VXGE_HAL_RING_HASH_TYPE_TCP_IPV6 * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_UDP_IPV6 \ * VXGE_HAL_RING_HASH_TYPE_UDP_IPV6 * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_IPV6 \ * VXGE_HAL_RING_HASH_TYPE_IPV6 * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_TCP_IPV6_EX \ * VXGE_HAL_RING_HASH_TYPE_TCP_IPV6_EX * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_UDP_IPV6_EX \ * VXGE_HAL_RING_HASH_TYPE_UDP_IPV6_EX * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_IPV6_EX \ * VXGE_HAL_RING_HASH_TYPE_IPV6_EX * * #define VXGE_HAL_RING_RXD_IS_VLAN_GET(ctrl0) bVAL1(ctrl0, 24) * #define VXGE_HAL_RING_RXD_IS_VLAN mBIT(24) * * #define VXGE_HAL_RING_RXD_ETHER_ENCAP_GET(ctrl0) bVAL2(ctrl0, 25) * #define VXGE_HAL_RING_RXD_ETHER_ENCAP(val) vBIT(val, 25, 2) * #define VXGE_HAL_RING_RXD_ETHER_ENCAP_DIX VXGE_HAL_FRAME_TYPE_DIX * #define VXGE_HAL_RING_RXD_ETHER_ENCAP_LLC VXGE_HAL_FRAME_TYPE_LLC * #define VXGE_HAL_RING_RXD_ETHER_ENCAP_SNAP VXGE_HAL_FRAME_TYPE_SNAP * #define VXGE_HAL_RING_RXD_ETHER_ENCAP_IPX VXGE_HAL_FRAME_TYPE_IPX * * #define VXGE_HAL_RING_RXD_IS_IPV4_GET(ctrl0) bVAL1(ctrl0, 27) * #define VXGE_HAL_RING_RXD_IS_IPV4 mBIT(27) * * #define VXGE_HAL_RING_RXD_IS_IPV6_GET(ctrl0) bVAL1(ctrl0, 28) * #define VXGE_HAL_RING_RXD_IS_IPV6 mBIT(28) * * #define VXGE_HAL_RING_RXD_IS_IPV_FRAG_GET(ctrl0) bVAL1(ctrl0, 29) * #define VXGE_HAL_RING_RXD_IS_IPV_FRAG mBIT(29) * * #define VXGE_HAL_RING_RXD_IS_TCP_GET(ctrl0) bVAL1(ctrl0, 30) * #define VXGE_HAL_RING_RXD_IS_TCP mBIT(30) * * #define VXGE_HAL_RING_RXD_IS_UDP_GET(ctrl0) bVAL1(ctrl0, 31) * #define VXGE_HAL_RING_RXD_IS_UDP mBIT(31) * * #define VXGE_HAL_RING_RXD_FRAME_PROTO_GET(ctrl0) bVAL5(ctrl0, 27) * #define VXGE_HAL_RING_RXD_FRAME_PROTO(val) vBIT(val, 27, 5) * #define VXGE_HAL_RING_RXD_FRAME_PROTO_IPV4 \ * VXGE_HAL_FRAME_PROTO_IPV4 * #define VXGE_HAL_RING_RXD_FRAME_PROTO_IPV6 \ * VXGE_HAL_FRAME_PROTO_IPV6 * #define VXGE_HAL_RING_RXD_FRAME_PROTO_IP_FRAG \ * VXGE_HAL_FRAME_PROTO_IP_FRAG * #define VXGE_HAL_RING_RXD_FRAME_PROTO_TCP \ * VXGE_HAL_FRAME_PROTO_TCP * #define VXGE_HAL_RING_RXD_FRAME_PROTO_UDP \ * VXGE_HAL_FRAME_PROTO_UDP * #define VXGE_HAL_RING_RXD_FRAME_PROTO_TCP_OR_UDP \ * (VXGE_HAL_FRAME_PROTO_TCP | VXGE_HAL_FRAME_PROTO_UDP) * * #define VXGE_HAL_RING_RXD_L3_CKSUM_GET(ctrl0) bVAL16(ctrl0, 32) * #define VXGE_HAL_RING_RXD_L3_CKSUM(val) vBIT(val, 32, 16) * * #define VXGE_HAL_RING_RXD_L4_CKSUM_GET(ctrl0) bVAL16(ctrl0, 48) * #define VXGE_HAL_RING_RXD_L4_CKSUM(val) vBIT(val, 48, 16) */ u64 control_1; #define VXGE_HAL_RING_RXD_3_BUFFER_EMPTY_GET(ctrl1) bVAL1(ctrl1, 0) #define VXGE_HAL_RING_RXD_3_BUFFER_EMPTY mBIT(0) #define VXGE_HAL_RING_RXD_3_BUFFER0_SIZE_GET(ctrl1) bVAL14(ctrl1, 2) #define VXGE_HAL_RING_RXD_3_BUFFER0_SIZE(val) vBIT(val, 2, 14) #define VXGE_HAL_RING_RXD_3_BUFFER0_SIZE_MASK vBIT(0x3FFc, 2, 14) #define VXGE_HAL_RING_RXD_3_BUFFER1_SIZE_GET(ctrl1) bVAL14(ctrl1, 18) #define VXGE_HAL_RING_RXD_3_BUFFER1_SIZE(val) vBIT(val, 18, 14) #define VXGE_HAL_RING_RXD_3_BUFFER1_SIZE_MASK vBIT(0x3FFc, 18, 14) #define VXGE_HAL_RING_RXD_3_BUFFER2_SIZE_GET(ctrl1) bVAL14(ctrl1, 34) #define VXGE_HAL_RING_RXD_3_BUFFER2_SIZE(val) vBIT(val, 34, 14) #define VXGE_HAL_RING_RXD_3_BUFFER2_SIZE_MASK vBIT(0x3FFc, 34, 14) /* * The following bit fields are common in all the three buffer modes and are * defined in vxge_hal_ring_rxd_1_t * #define VXGE_HAL_RING_RXD_VLAN_TAG_GET(ctrl1) bVAL16(ctrl1, 48) * #define VXGE_HAL_RING_RXD_VLAN_TAG(val) vBIT(val, 48, 16) */ u64 buffer0_ptr; #define VXGE_HAL_RING_RXD_3_RTH_HASH_VALUE_GET(b0_ptr) bVAL32(b0_ptr, 32) #define VXGE_HAL_RING_RXD_3_RTH_HASH_VALUE(val) vBIT(val, 32, 32) u64 buffer1_ptr; u64 buffer2_ptr; } vxge_hal_ring_rxd_3_t; /* * struct vxge_hal_ring_rxd_5_t - Five buffer mode RxD for ring * @host_control: This 32 bitfield is exclusively for host use and is "readonly" * from the adapter's perspective. * @control_2: Bits 0 to 1 - Reserved * Bits 2 to 15 - This field is set by the host and eventually * overwritten by the adapter. The host writes the available buffer * 3 size in bytes when it pas ses the descriptor to the adapter. * When a frame is delivered the host, the adapter populates this * field with the number of bytes written into the buffer. The * largest supported buffer is 16, 383 bytes. * Bits 16 to 17 - Reserved * Bits 18 to 31 - This field is set by the host and eventually * overwritten by the adapter. The host writes the available buffer * 4 size in bytes when it passes the descriptor to the adapter. * When a frame is delivered the host, the adapter populates this * field with the number of bytes written into the buffer. * The largest supported buffer is 16, 383 bytes. * @control_0: Bits 0 to 6 - RTH_Bucket get * Bit 7 - Own Descriptor ownership bit. This bit is set to 1 by * the host, and is set to 0 by the adapter. * 0 - Host owns RxD and buffer. * 1 - The adapter owns RxD and buffer. * Bit 8 - Fast_Path_Eligible When set,indicates that the received * frame meets all of the criteria for fast path processing. * The required criteria are as follows: * !SYN & * (Transfer_Code == "Transfer OK") & * (!Is_IP_Fragment) & * ((Is_IPv4 & computed_L3_checksum == 0xFFFF) | * (Is_IPv6)) & * ((Is_TCP & computed_L4_checksum == 0xFFFF) | * (Is_UDP & (computed_L4_checksum == 0xFFFF | * computed _L4_checksum == 0x0000))) * (same meaning for all RxD buffer modes) * Bit 9 - L3 Checksum Correct * Bit 10 - L4 Checksum Correct * Bit 11 - Reserved * Bit 12 to 15 - This field is written by the adapter. It is used * to report the status of the frame transfer to the host. * 0x0 - Transfer OK * 0x4 - RDA Failure During Transfer * 0x5 - Unparseable Packet, such as unknown IPv6 header. * 0x6 - Frame integrity error (FCS or ECC). * 0x7 - Buffer Size Error. The provided buffer(s) were not * appropriately sized and data loss occurred. * 0x8 - Internal ECC Error. RxD corrupted. * 0x9 - IPv4 Checksum error * 0xA - TCP/UDP Checksum error * 0xF - Unknown Error or Multiple Error. Indicates an unknown * problem or that more than one of transfer codes is set. * Bit 16 - SYN The adapter sets this field to indicate that the * incoming frame contained a TCP segment with its SYN bit set * and its ACK bit NOT set. (same meaning for all RxD buffer modes) * Bit 17 - Is ICMP * Bit 18 - RTH_SPDM_HIT Set to 1 if there was a match in the * Socket Pair Direct Match Table and the frame was steered based on * SPDM. * Bit 19 - RTH_IT_HIT Set to 1 if there was a match in the * Indirection Table and the frame was steered based on hash * indirection. * Bit 20 to 23 - RTH_HASH_TYPE Indicates the function (hash type) * that was used to calculate the hash. * Bit 19 - IS_VLAN Set to '1' if the frame was/is VLAN tagged. * Bit 25 to 26 - ETHER_ENCAP Reflects the Ethernet encapsulation * of the received frame. * 0x0 - Ethernet DIX * 0x1 - LLC * 0x2 - SNAP (includes Jumbo-SNAP) * 0x3 - IPX * Bit 27 - IS_IPV4 Set to '1' if the frame contains IPv4 packet. * Bit 28 - IS_IPV6 Set to '1' if the frame contains IPv6 packet. * Bit 29 - IS_IP_FRAG Set to '1' if the frame contains a * fragmented IP packet. * Bit 30 - IS_TCP Set to '1' if the frame contains a TCP segment. * Bit 31 - IS_UDP Set to '1' if the frame contains a UDP message. * Bit 32 to 47 - L3_Checksum[0:15] The IPv4 checksum value that * arrived with the frame. If the resulting computed IPv4 header * checksum for the frame did not produce the expected 0xFFFF value, * then the transfer code would be set to 0x9. * Bit 48 to 63 - L4_Checksum[0:15] TCP/UDP checksum value that * arrived with the frame. If the resulting computed TCP/UDP checksum * for the frame did not produce the expected 0xFFFF value, then the * transfer code would be set to 0xA. * @control_1: Bits 0 to 1 - Reserved. * Bits 2 to 15 - This field is written by the host and by X3100. * The host writes the available buffer 0 size in bytes when it * passes the descriptor to the X3100. The X3100 writes the number * of bytes written to the buffer when it passes the descriptor back * to the host. * Bits 16 to 17 - Reserved * Bits 18 to 31 - This field is set by the host and eventually * overwritten by the adapter. The host writes the available * buffer 1 size in bytes when it passes the descriptor to the adapter. * When a frame is delivered the host, the adapter populates this field * with the number of bytes written into the buffer 1. The largest * supported buffer is 16, 383 bytes. * Bits 32 to 33 - Reserved * Bits 34 to 47 - This field is set by the host and eventually * overwritten by the adapter. The host writes the available * buffer 2 size in bytes when it passes the descriptor to the adapter. * When a frame is delivered the host, the adapter populates this field * with the number of bytes written into the buffer 2. The largest * supported buffer is 16, 383 bytes. * Bit 48 to 63 - VLAN_Tag[0:15] The contents of the variable * portion of the VLAN tag, if one was detected by the adapter. This * field is populated even if VLAN-tag stripping is enabled. * @buffer0_ptr: Pointer to buffer 0. This field is populated by the driver. * In 5-buffer mode, when the RxD is returned to the host, * buffer0_ptr field will be overwritten if the following conditions * are met: * 1 - RTH_Disable in the PRC_CTRL register is not set. * 2 - RTH is enabled and a valid hash value was calculated for the * frame. This will be indicated by a non-zero value in the * RTH_HASH_TYPE field (Control_0, bits 20:23). In the event that the * pointer is overwritten on return to the host, bits 0:31 will be * all zeroes while bits 32:63 will contain the calculated hash value. * @buffer1_ptr: Pointer to buffer 1. This field is populated by the driver. * @buffer2_ptr: Pointer to buffer 2. This field is populated by the driver. * @buffer3_ptr: Pointer to buffer 3. This field is populated by the driver. * @buffer4_ptr: Pointer to buffer 4. This field is populated by the driver. * @pad: Pad to align at cache line boundary * * Three buffer mode RxD for ring structure */ typedef struct vxge_hal_ring_rxd_5_t { #if defined(VXGE_OS_HOST_BIG_ENDIAN) u32 host_control; u32 control_2; #else u32 control_2; u32 host_control; #endif #define VXGE_HAL_RING_RXD_5_BUFFER3_SIZE_GET(ctrl2) bVAL14(ctrl2, 34) #define VXGE_HAL_RING_RXD_5_BUFFER3_SIZE(val) vBIT(val, 34, 14) #define VXGE_HAL_RING_RXD_5_BUFFER3_SIZE_MASK vBIT(0x3FFF, 34, 14) #define VXGE_HAL_RING_RXD_5_BUFFER4_SIZE_GET(ctrl2) bVAL14(ctrl2, 50) #define VXGE_HAL_RING_RXD_5_BUFFER4_SIZE(val) vBIT(val, 50, 14) #define VXGE_HAL_RING_RXD_5_BUFFER4_SIZE_MASK vBIT(0x3FFF, 50, 14) u64 control_0; /* * The following bit fields are common in all the three buffer modes and are * defined in vxge_hal_ring_rxd_1_t * #define VXGE_HAL_RING_RXD_RTH_BUCKET_GET(ctrl0) bVAL7(ctrl0, 0) * #define VXGE_HAL_RING_RXD_RTH_BUCKET_ADAPTER vBIT(val, 0, 7) * * #define VXGE_HAL_RING_RXD_LIST_OWN_GET(ctrl0) bVAL1(ctrl0, 7) * #define VXGE_HAL_RING_RXD_LIST_OWN_ADAPTER mBIT(7) * * #define VXGE_HAL_RING_RXD_FAST_PATH_ELIGIBLE_GET(ctrl0 \ * bVAL1(ctrl0, 8) * #define VXGE_HAL_RING_RXD_FAST_PATH_ELIGIBLE mBIT(8) * * #define VXGE_HAL_RING_RXD_L3_CKSUM_CORRECT_GET(ctrl0) \ * bVAL1(ctrl0, 9) * #define VXGE_HAL_RING_RXD_L3_CKSUM_CORRECT mBIT(9) * * #define VXGE_HAL_RING_RXD_L4_CKSUM_CORRECT_GET(ctrl0) \ * bVAL1(ctrl0, 10) * #define VXGE_HAL_RING_RXD_L4_CKSUM_CORRECT mBIT(10) * * #define VXGE_HAL_RING_RXD_T_CODE_GET(ctrl0) bVAL4(ctrl0, 12) * #define VXGE_HAL_RING_RXD_T_CODE(val) vBIT(val, 12, 4) * #define VXGE_HAL_RING_RXD_T_CODE_OK \ * VXGE_HAL_RING_T_CODE_OK * #define VXGE_HAL_RING_RXD_T_CODE_L3_CKSUM_MISMATCH \ * VXGE_HAL_RING_T_CODE_L3_CKSUM_MISMATCH * #define VXGE_HAL_RING_RXD_T_CODE_L4_CKSUM_MISMATCH \ * VXGE_HAL_RING_T_CODE_L4_CKSUM_MISMATCH * #define VXGE_HAL_RING_RXD_T_CODE_L3_L4_CKSUM_MISMATCH \ * VXGE_HAL_RING_T_CODE_L3_L4_CKSUM_MISMATCH * #define VXGE_HAL_RING_RXD_T_CODE_L3_PKT_ERR \ * VXGE_HAL_RING_T_CODE_L3_PKT_ERR * #define VXGE_HAL_RING_RXD_T_CODE_L2_FRM_ERR \ * VXGE_HAL_RING_T_CODE_L2_FRM_ERR * #define VXGE_HAL_RING_RXD_T_CODE_BUF_SIZE_ERR \ * VXGE_HAL_RING_T_CODE_BUF_SIZE_ERR * #define VXGE_HAL_RING_RXD_T_CODE_INT_ECC_ERR \ * VXGE_HAL_RING_T_CODE_INT_ECC_ERR * #define VXGE_HAL_RING_RXD_T_CODE_BENIGN_OVFLOW \ * VXGE_HAL_RING_T_CODE_BENIGN_OVFLOW * #define VXGE_HAL_RING_RXD_T_CODE_ZERO_LEN_BUFF \ * VXGE_HAL_RING_T_CODE_ZERO_LEN_BUFF * #define VXGE_HAL_RING_RXD_T_CODE_FRM_DROP \ * VXGE_HAL_RING_T_CODE_FRM_DROP * #define VXGE_HAL_RING_RXD_T_CODE_UNUSED \ * VXGE_HAL_RING_T_CODE_UNUSED * #define VXGE_HAL_RING_RXD_T_CODE_MULTI_ERR \ * VXGE_HAL_RING_T_CODE_MULTI_ERR * * #define VXGE_HAL_RING_RXD_SYN_GET(ctrl0) bVAL1(ctrl0, 16) * #define VXGE_HAL_RING_RXD_SYN mBIT(16) * * #define VXGE_HAL_RING_RXD_IS_ICMP_GET(ctrl0) bVAL1(ctrl0, 17) * #define VXGE_HAL_RING_RXD_IS_ICMP mBIT(17) * * #define VXGE_HAL_RING_RXD_RTH_SPDM_HIT_GET(ctrl0) bVAL1(ctrl0, 18) * #define VXGE_HAL_RING_RXD_RTH_SPDM_HIT mBIT(18) * * #define VXGE_HAL_RING_RXD_RTH_IT_HIT_GET(ctrl0) bVAL1(ctrl0, 19) * #define VXGE_HAL_RING_RXD_RTH_IT_HIT mBIT(19) * * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_GET(ctrl0) bVAL4(ctrl0, 20) * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE(val) vBIT(val, 20, 4) * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_NONE \ * VXGE_HAL_RING_HASH_TYPE_NONE * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_TCP_IPV4 \ * VXGE_HAL_RING_HASH_TYPE_TCP_IPV4 * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_UDP_IPV4 \ * VXGE_HAL_RING_HASH_TYPE_UDP_IPV4 * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_IPV4 \ * VXGE_HAL_RING_HASH_TYPE_IPV4 * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_TCP_IPV6 \ * VXGE_HAL_RING_HASH_TYPE_TCP_IPV6 * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_UDP_IPV6 \ * VXGE_HAL_RING_HASH_TYPE_UDP_IPV6 * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_IPV6 \ * VXGE_HAL_RING_HASH_TYPE_IPV6 * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_TCP_IPV6_EX \ * VXGE_HAL_RING_HASH_TYPE_TCP_IPV6_EX * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_UDP_IPV6_EX \ * VXGE_HAL_RING_HASH_TYPE_UDP_IPV6_EX * #define VXGE_HAL_RING_RXD_RTH_HASH_TYPE_IPV6_EX \ * VXGE_HAL_RING_HASH_TYPE_IPV6_EX * * #define VXGE_HAL_RING_RXD_IS_VLAN_GET(ctrl0) bVAL1(ctrl0, 24) * #define VXGE_HAL_RING_RXD_IS_VLAN mBIT(24) * * #define VXGE_HAL_RING_RXD_ETHER_ENCAP_GET(ctrl0) bVAL2(ctrl0, 25) * #define VXGE_HAL_RING_RXD_ETHER_ENCAP(val) vBIT(val, 25, 2) * #define VXGE_HAL_RING_RXD_ETHER_ENCAP_DIX VXGE_HAL_FRAME_TYPE_DIX * #define VXGE_HAL_RING_RXD_ETHER_ENCAP_LLC VXGE_HAL_FRAME_TYPE_LLC * #define VXGE_HAL_RING_RXD_ETHER_ENCAP_SNAP VXGE_HAL_FRAME_TYPE_SNAP * #define VXGE_HAL_RING_RXD_ETHER_ENCAP_IPX VXGE_HAL_FRAME_TYPE_IPX * * #define VXGE_HAL_RING_RXD_IS_IPV4_GET(ctrl0) bVAL1(ctrl0, 27) * #define VXGE_HAL_RING_RXD_IS_IPV4 mBIT(27) * * #define VXGE_HAL_RING_RXD_IS_IPV6_GET(ctrl0) bVAL1(ctrl0, 28) * #define VXGE_HAL_RING_RXD_IS_IPV6 mBIT(28) * * #define VXGE_HAL_RING_RXD_IS_IPV_FRAG_GET(ctrl0) bVAL1(ctrl0, 29) * #define VXGE_HAL_RING_RXD_IS_IPV_FRAG mBIT(29) * * #define VXGE_HAL_RING_RXD_IS_TCP_GET(ctrl0) bVAL1(ctrl0, 30) * #define VXGE_HAL_RING_RXD_IS_TCP mBIT(30) * * #define VXGE_HAL_RING_RXD_IS_UDP_GET(ctrl0) bVAL1(ctrl0, 31) * #define VXGE_HAL_RING_RXD_IS_UDP mBIT(31) * * #define VXGE_HAL_RING_RXD_FRAME_PROTO_GET(ctrl0) bVAL5(ctrl0, 27) * #define VXGE_HAL_RING_RXD_FRAME_PROTO(val) vBIT(val, 27, 5) * #define VXGE_HAL_RING_RXD_FRAME_PROTO_IPV4 VXGE_HAL_FRAME_PROTO_IPV4 * #define VXGE_HAL_RING_RXD_FRAME_PROTO_IPV6 VXGE_HAL_FRAME_PROTO_IPV6 * #define VXGE_HAL_RING_RXD_FRAME_PROTO_IP_FRAG \ * VXGE_HAL_FRAME_PROTO_IP_FRAG * #define VXGE_HAL_RING_RXD_FRAME_PROTO_TCP VXGE_HAL_FRAME_PROTO_TCP * #define VXGE_HAL_RING_RXD_FRAME_PROTO_UDP VXGE_HAL_FRAME_PROTO_UDP * #define VXGE_HAL_RING_RXD_FRAME_PROTO_TCP_OR_UDP \ * (VXGE_HAL_FRAME_PROTO_TCP | VXGE_HAL_FRAME_PROTO_UDP) * * #define VXGE_HAL_RING_RXD_L3_CKSUM_GET(ctrl0) bVAL16(ctrl0, 32) * #define VXGE_HAL_RING_RXD_L3_CKSUM(val) vBIT(val, 32, 16) * * #define VXGE_HAL_RING_RXD_L4_CKSUM_GET(ctrl0) bVAL16(ctrl0, 48) * #define VXGE_HAL_RING_RXD_L4_CKSUM(val) vBIT(val, 48, 16) */ u64 control_1; #define VXGE_HAL_RING_RXD_5_BUFFER0_SIZE_GET(ctrl1) bVAL14(ctrl1, 2) #define VXGE_HAL_RING_RXD_5_BUFFER0_SIZE(val) vBIT(val, 2, 14) #define VXGE_HAL_RING_RXD_5_BUFFER0_SIZE_MASK vBIT(0x3FFF, 2, 14) #define VXGE_HAL_RING_RXD_5_BUFFER1_SIZE_GET(ctrl1) bVAL14(ctrl1, 18) #define VXGE_HAL_RING_RXD_5_BUFFER1_SIZE(val) vBIT(val, 18, 14) #define VXGE_HAL_RING_RXD_5_BUFFER1_SIZE_MASK vBIT(0x3FFF, 18, 14) #define VXGE_HAL_RING_RXD_5_BUFFER2_SIZE_GET(ctrl1) bVAL14(ctrl1, 34) #define VXGE_HAL_RING_RXD_5_BUFFER2_SIZE(val) vBIT(val, 34, 14) #define VXGE_HAL_RING_RXD_5_BUFFER2_SIZE_MASK vBIT(0xFFFF, 34, 14) /* * The following bit fields are common in all the three buffer modes and are * defined in vxge_hal_ring_rxd_1_t * #define VXGE_HAL_RING_RXD_VLAN_TAG_GET(ctrl1) bVAL16(ctrl1, 48) * #define VXGE_HAL_RING_RXD_VLAN_TAG(val) vBIT(val, 48, 16) */ u64 buffer0_ptr; #define VXGE_HAL_RING_RXD_5_RTH_HASH_VALUE_GET(b0_ptr) bVAL32(b0_ptr, 32) #define VXGE_HAL_RING_RXD_5_RTH_HASH_VALUE(val) vBIT(val, 32, 32) u64 buffer1_ptr; u64 buffer2_ptr; u64 buffer3_ptr; u64 buffer4_ptr; } vxge_hal_ring_rxd_5_t; /* * function vxge_hal_ring_callback_f - Ring callback. * @vpath_handle: Virtual Path whose Ring "containing" 1 or more completed * descriptors. * @rxdh: First completed descriptor. * @rxd_priv: Pointer to per rxd space allocated * @t_code: Transfer code, as per X3100 User Guide. * Returned by HAL. * @userdata: Opaque per-ring data specified at ring open * time, via vxge_hal_vpath_open(). * * ring completion callback (type declaration). A single per-ring * callback is specified at virtual path open time, via * vxge_hal_vpath_open(). * Typically gets called as part of the processing of the Interrupt * Service Routine. * * ring callback gets called by HAL if, and only if, there is at least * one new completion on a given ring . Upon processing the first @rxdh * ULD is _supposed_ to continue consuming completions * using - vxge_hal_ring_rxd_next_completed(). * * Note that failure to process new completions in a timely fashion * leads to VXGE_HAL_INF_OUT_OF_DESCRIPTORS condition. * * Non-zero @t_code means failure to process receive descriptor. * * In the "transmit" case the failure could happen, for instance, when the * link is down, in which case X3100 completes the descriptor because it * is not able to send the data out. * * For details please refer to X3100 User Guide. * * See also: vxge_hal_ring_rxd_next_completed(), vxge_hal_ring_rxd_term_f {}. */ typedef vxge_hal_status_e(*vxge_hal_ring_callback_f) ( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh, void *rxd_priv, u8 t_code, void *userdata); /* * function vxge_hal_ring_rxd_init_f - Initialize descriptor callback. * @vpath_handle: Virtual path whose ring "containing" the @rxdh descriptor. * @rxdh: Descriptor. * @rxd_priv: Pointer to per rxd space allocated * @index: Index of the descriptor in the ring's set of descriptors. * @userdata: Per-ring user data (a.k.a. context) specified at * ring open time, via vxge_hal_vpath_open(). * @reopen: See vxge_hal_reopen_e {}. * * Initialize descriptor callback. Unless NULL is specified in the * vxge_hal_ring_attr_t {} structure passed to vxge_hal_vpath_open()), * HAL invokes the callback as part of the ring create in vxge_hal_vpath_open() * implementation. * The ULD is expected to fill in this descriptor with buffer(s) * and control information. * * See also: vxge_hal_ring_attr_t {}, vxge_hal_ring_rxd_term_f {}. */ typedef vxge_hal_status_e(*vxge_hal_ring_rxd_init_f) ( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh, void *rxd_priv, u32 index, void *userdata, vxge_hal_reopen_e reopen); /* * function vxge_hal_ring_rxd_term_f - Terminate descriptor callback. * @vpath_handle: Virtual path whose ring "containing" the @rxdh descriptor. * @rxdh: First completed descriptor. * @rxd_priv: Pointer to per rxd space allocated * @state: One of the vxge_hal_rxd_state_e {} enumerated states. * @userdata: Per-ring user data (a.k.a. context) specified at * ring open time, via vxge_hal_vpath_open(). * @reopen: See vxge_hal_reopen_e {}. * * Terminate descriptor callback. Unless NULL is specified in the * vxge_hal_ring_attr_t {} structure passed to vxge_hal_vpath_open()), * HAL invokes the callback as part of closing the corresponding * ring, prior to de-allocating the ring and associated data * structures (including descriptors). * ULD should utilize the callback to (for instance) unmap * and free DMA data buffers associated with the posted (state = * VXGE_HAL_RXD_STATE_POSTED) descriptors, * as well as other relevant cleanup functions. * * See also: vxge_hal_ring_attr_t {}, vxge_hal_ring_rxd_init_f {}. */ typedef void (*vxge_hal_ring_rxd_term_f) ( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh, void *rxd_priv, vxge_hal_rxd_state_e state, void *userdata, vxge_hal_reopen_e reopen); /* * struct vxge_hal_ring_attr_t - Ring open "template". * @callback: Ring completion callback. HAL invokes the callback when there * are new completions on that ring. In many implementations * the @callback executes in the hw interrupt context. * @rxd_init: Ring's descriptor-initialize callback. * See vxge_hal_ring_rxd_init_f {}. * If not NULL, HAL invokes the callback when opening * the ring. * @rxd_term: Ring's descriptor-terminate callback. If not NULL, * HAL invokes the callback when closing the corresponding ring. * See also vxge_hal_ring_rxd_term_f {}. * @userdata: User-defined "context" of _that_ ring. Passed back to the * user as one of the @callback, @rxd_init, and @rxd_term arguments. * @per_rxd_space: If specified (i.e., greater than zero): extra space * reserved by HAL per each receive descriptor. Can be used to store, * and retrieve on completion, information specific * to the upper-layer. * * Ring open "template". User fills the structure with ring * attributes and passes it to vxge_hal_vpath_open(). */ typedef struct vxge_hal_ring_attr_t { vxge_hal_ring_callback_f callback; vxge_hal_ring_rxd_init_f rxd_init; vxge_hal_ring_rxd_term_f rxd_term; void *userdata; u32 per_rxd_space; } vxge_hal_ring_attr_t; /* * vxge_hal_ring_rxd_size_get - Get the size of ring descriptor. * @buf_mode: Buffer mode (1, 3 or 5) * * This function returns the size of RxD for given buffer mode */ static inline u32 /* LINTED */ vxge_hal_ring_rxd_size_get( u32 buf_mode) { return ((u32) (buf_mode == 1 ? sizeof(vxge_hal_ring_rxd_1_t) : \ (buf_mode == 3 ? sizeof(vxge_hal_ring_rxd_3_t) : \ sizeof(vxge_hal_ring_rxd_5_t)))); } /* * vxge_hal_ring_rxds_per_block_get - Get the number of rxds per block. * @buf_mode: Buffer mode (1, 3 or 5) * * This function returns the number of RxD for RxD block for given buffer mode */ static inline u32 /* LINTED */ vxge_hal_ring_rxds_per_block_get( u32 buf_mode) { return ((u32) ((VXGE_OS_HOST_PAGE_SIZE - 16) / ((buf_mode == 1) ? sizeof(vxge_hal_ring_rxd_1_t) : ((buf_mode == 3) ? sizeof(vxge_hal_ring_rxd_3_t) : sizeof(vxge_hal_ring_rxd_5_t))))); } /* * vxge_hal_ring_rxd_reserve - Reserve ring descriptor. * @vpath_handle: virtual Path handle. * @rxdh: Reserved descriptor. On success HAL fills this "out" parameter * with a valid handle. * @rxd_priv: Buffer to return the pointer to per rxd space allocated * * Reserve Rx descriptor for the subsequent filling-in (by upper layer * driver (ULD)) and posting on the corresponding ring * via vxge_hal_ring_rxd_post(). * * Returns: VXGE_HAL_OK - success. * VXGE_HAL_INF_OUT_OF_DESCRIPTORS - Currently no descriptors available. * */ vxge_hal_status_e vxge_hal_ring_rxd_reserve( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h *rxdh, void **rxd_priv); /* * vxge_hal_ring_rxd_1b_set - Prepare 1-buffer-mode descriptor. * @rxdh: Descriptor handle. * @dma_pointer: DMA address of a single receive buffer this descriptor * should carry. Note that by the time * vxge_hal_ring_rxd_1b_set is called, the * receive buffer should be already mapped * to the corresponding X3100 device. * @size: Size of the receive @dma_pointer buffer. * * Prepare 1-buffer-mode Rx descriptor for posting * (via vxge_hal_ring_rxd_post()). * * This inline helper-function does not return any parameters and always * succeeds. * */ static inline /* LINTED */ void vxge_hal_ring_rxd_1b_set( vxge_hal_rxd_h rxdh, dma_addr_t dma_pointer, int size) { vxge_hal_ring_rxd_1_t *rxdp = (vxge_hal_ring_rxd_1_t *) rxdh; rxdp->buffer0_ptr = dma_pointer; rxdp->control_1 &= ~VXGE_HAL_RING_RXD_1_BUFFER0_SIZE_MASK; rxdp->control_1 |= VXGE_HAL_RING_RXD_1_BUFFER0_SIZE(size); } /* * vxge_hal_ring_rxd_3b_set - Prepare 3-buffer-mode descriptor. * @rxdh: Descriptor handle. * @dma_pointers: Array of DMA addresses. Contains exactly 3 receive buffers * _this_ descriptor should carry. Note that by the time * vxge_hal_ring_rxd_3b_set is called, the receive buffers should * be mapped to the corresponding X3100 device. * @sizes: Array of receive buffer sizes. Contains 3 sizes: one size per * buffer from @dma_pointers. * * Prepare 3-buffer-mode Rx descriptor for posting (via * vxge_hal_ring_rxd_post()). * This inline helper-function does not return any parameters and always * succeeds. * */ static inline /* LINTED */ void vxge_hal_ring_rxd_3b_set( vxge_hal_rxd_h rxdh, dma_addr_t dma_pointers[], u32 sizes[]) { vxge_hal_ring_rxd_3_t *rxdp = (vxge_hal_ring_rxd_3_t *) rxdh; rxdp->buffer0_ptr = dma_pointers[0]; rxdp->control_1 &= (~VXGE_HAL_RING_RXD_3_BUFFER0_SIZE_MASK); rxdp->control_1 |= VXGE_HAL_RING_RXD_3_BUFFER0_SIZE(sizes[0]); rxdp->buffer1_ptr = dma_pointers[1]; rxdp->control_1 &= (~VXGE_HAL_RING_RXD_3_BUFFER1_SIZE_MASK); rxdp->control_1 |= VXGE_HAL_RING_RXD_3_BUFFER1_SIZE(sizes[1]); rxdp->buffer2_ptr = dma_pointers[2]; rxdp->control_1 &= (~VXGE_HAL_RING_RXD_3_BUFFER2_SIZE_MASK); rxdp->control_1 |= VXGE_HAL_RING_RXD_3_BUFFER2_SIZE(sizes[2]); } /* * vxge_hal_ring_rxd_5b_set - Prepare 5-buffer-mode descriptor. * @rxdh: Descriptor handle. * @dma_pointers: Array of DMA addresses. Contains exactly 5 receive buffers * _this_ descriptor should carry. Note that by the time * vxge_hal_ring_rxd_5b_set is called, the receive buffers should * be mapped to the corresponding X3100 device. * @sizes: Array of receive buffer sizes. Contains 5 sizes: one size per buffer * from @dma_pointers. * * Prepare 5-buffer-mode Rx descriptor for posting * (via vxge_hal_ring_rxd_post()). * This inline helper-function does not return any * values and always succeeds. * * See also: vxge_hal_ring_rxd_1b_set(), vxge_hal_ring_rxd_3b_set(). */ static inline /* LINTED */ void vxge_hal_ring_rxd_5b_set( vxge_hal_rxd_h rxdh, dma_addr_t dma_pointers[], u32 sizes[]) { vxge_hal_ring_rxd_5_t *rxdp = (vxge_hal_ring_rxd_5_t *) rxdh; rxdp->buffer0_ptr = dma_pointers[0]; rxdp->control_1 &= (~VXGE_HAL_RING_RXD_5_BUFFER0_SIZE_MASK); rxdp->control_1 |= VXGE_HAL_RING_RXD_5_BUFFER0_SIZE(sizes[0]); rxdp->buffer1_ptr = dma_pointers[1]; rxdp->control_1 &= (~VXGE_HAL_RING_RXD_5_BUFFER1_SIZE_MASK); rxdp->control_1 |= VXGE_HAL_RING_RXD_5_BUFFER1_SIZE(sizes[1]); rxdp->buffer2_ptr = dma_pointers[2]; rxdp->control_1 &= (~VXGE_HAL_RING_RXD_5_BUFFER2_SIZE_MASK); rxdp->control_1 |= VXGE_HAL_RING_RXD_5_BUFFER2_SIZE(sizes[2]); rxdp->buffer3_ptr = dma_pointers[3]; rxdp->control_2 &= (~VXGE_HAL_RING_RXD_5_BUFFER3_SIZE_MASK); rxdp->control_2 |= VXGE_HAL_RING_RXD_5_BUFFER3_SIZE(sizes[3]); rxdp->buffer4_ptr = dma_pointers[4]; rxdp->control_2 &= (~VXGE_HAL_RING_RXD_5_BUFFER4_SIZE_MASK); rxdp->control_2 |= VXGE_HAL_RING_RXD_5_BUFFER4_SIZE(sizes[4]); } /* * vxge_hal_ring_rxd_pre_post - Prepare rxd and post * @vpath_handle: virtual Path handle. * @rxdh: Descriptor handle. * * This routine prepares a rxd and posts */ void vxge_hal_ring_rxd_pre_post( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh); /* * vxge_hal_ring_rxd_post_post - Process rxd after post. * @vpath_handle: virtual Path handle. * @rxdh: Descriptor handle. * * Processes rxd after post */ void vxge_hal_ring_rxd_post_post( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh); /* * vxge_hal_ring_rxd_post_post_db - Post Doorbell after posting the rxd(s). * @vpath_handle: virtual Path handle. * * Post Doorbell after posting the rxd(s). */ void vxge_hal_ring_rxd_post_post_db( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_ring_rxd_post_post_wmb - Process rxd after post with memory barrier * @vpath_handle: virtual Path handle. * @rxdh: Descriptor handle. * * Processes rxd after post with memory barrier. */ void vxge_hal_ring_rxd_post_post_wmb( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh); /* * vxge_hal_ring_rxd_post - Post descriptor on the ring. * @vpath_handle: virtual Path handle. * @rxdh: Descriptor obtained via vxge_hal_ring_rxd_reserve(). * * Post descriptor on the ring. * Prior to posting the descriptor should be filled in accordance with * Host/X3100 interface specification for a given service (LL, etc.). * */ void vxge_hal_ring_rxd_post( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh); /* * vxge_hal_ring_is_next_rxd_completed - Check if the next rxd is completed * @vpath_handle: Virtual Path handle. * * Checks if the _next_ completed descriptor is in host memory * * Returns: VXGE_HAL_OK - success. * VXGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed descriptors * are currently available for processing. */ vxge_hal_status_e vxge_hal_ring_is_next_rxd_completed( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_ring_rxd_next_completed - Get the _next_ completed descriptor. * @vpath_handle: Virtual path handle. * @rxdh: Descriptor handle. Returned by HAL. * @rxd_priv: Buffer to return a pointer to the per rxd space allocated * @t_code: Transfer code, as per X3100 User Guide, * Receive Descriptor Format. Returned by HAL. * * Retrieve the _next_ completed descriptor. * HAL uses ring callback (*vxge_hal_ring_callback_f) to notifiy * upper-layer driver (ULD) of new completed descriptors. After that * the ULD can use vxge_hal_ring_rxd_next_completed to retrieve the rest * completions (the very first completion is passed by HAL via * vxge_hal_ring_callback_f). * * Implementation-wise, the upper-layer driver is free to call * vxge_hal_ring_rxd_next_completed either immediately from inside the * ring callback, or in a deferred fashion and separate (from HAL) * context. * * Non-zero @t_code means failure to fill-in receive buffer(s) * of the descriptor. * For instance, parity error detected during the data transfer. * In this case X3100 will complete the descriptor and indicate * for the host that the received data is not to be used. * For details please refer to X3100 User Guide. * * Returns: VXGE_HAL_OK - success. * VXGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed descriptors * are currently available for processing. * * See also: vxge_hal_ring_callback_f {}, * vxge_hal_fifo_txdl_next_completed(), vxge_hal_status_e {}. */ vxge_hal_status_e vxge_hal_ring_rxd_next_completed( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h *rxdh, void **rxd_priv, u8 *t_code); /* * vxge_hal_ring_handle_tcode - Handle transfer code. * @vpath_handle: Virtual Path handle. * @rxdh: Descriptor handle. * @t_code: One of the enumerated (and documented in the X3100 user guide) * "transfer codes". * * Handle descriptor's transfer code. The latter comes with each completed * descriptor. * * Returns: one of the vxge_hal_status_e {} enumerated types. * VXGE_HAL_OK - for success. * VXGE_HAL_ERR_CRITICAL - when encounters critical error. */ vxge_hal_status_e vxge_hal_ring_handle_tcode( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh, u8 t_code); /* * vxge_hal_ring_rxd_1b_get - Get data from the completed 1-buf * descriptor. * @vpath_handle: Virtual Path handle. * @rxdh: Descriptor handle. * @dma_pointer: DMA address of a single receive buffer _this_ descriptor * carries. Returned by HAL. * @pkt_length: Length (in bytes) of the data in the buffer pointed by * @dma_pointer. Returned by HAL. * * Retrieve protocol data from the completed 1-buffer-mode Rx descriptor. * This inline helper-function uses completed descriptor to populate receive * buffer pointer and other "out" parameters. The function always succeeds. * */ static inline /* LINTED */ void vxge_hal_ring_rxd_1b_get( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh, dma_addr_t *dma_pointer, u32 *pkt_length) { vxge_hal_ring_rxd_1_t *rxdp = (vxge_hal_ring_rxd_1_t *) rxdh; *pkt_length = (u32) VXGE_HAL_RING_RXD_1_BUFFER0_SIZE_GET(rxdp->control_1); *dma_pointer = rxdp->buffer0_ptr; } /* * vxge_hal_ring_rxd_3b_get - Get data from the completed 3-buf * descriptor. * @vpath_handle: Virtual Path handle. * @rxdh: Descriptor handle. * @dma_pointers: DMA addresses of the 3 receive buffers _this_ descriptor * carries. The first two buffers contain ethernet and * (IP + transport) headers. The 3rd buffer contains packet * data. * @sizes: Array of receive buffer sizes. Contains 3 sizes: one size per * buffer from @dma_pointers. Returned by HAL. * * Retrieve protocol data from the completed 3-buffer-mode Rx descriptor. * This inline helper-function uses completed descriptor to populate receive * buffer pointer and other "out" parameters. The function always succeeds. * */ static inline /* LINTED */ void vxge_hal_ring_rxd_3b_get( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh, dma_addr_t dma_pointers[], u32 sizes[]) { vxge_hal_ring_rxd_3_t *rxdp = (vxge_hal_ring_rxd_3_t *) rxdh; dma_pointers[0] = rxdp->buffer0_ptr; sizes[0] = (u32) VXGE_HAL_RING_RXD_3_BUFFER0_SIZE_GET(rxdp->control_1); dma_pointers[1] = rxdp->buffer1_ptr; sizes[1] = (u32) VXGE_HAL_RING_RXD_3_BUFFER1_SIZE_GET(rxdp->control_1); dma_pointers[2] = rxdp->buffer2_ptr; sizes[2] = (u32) VXGE_HAL_RING_RXD_3_BUFFER2_SIZE_GET(rxdp->control_1); } /* * vxge_hal_ring_rxd_5b_get - Get data from the completed 5-buf descriptor. * @vpath_handle: Virtual Path handle. * @rxdh: Descriptor handle. * @dma_pointers: DMA addresses of the 5 receive buffers _this_ descriptor * carries. The first 4 buffers contains L2 (ethernet) through * L5 headers. The 5th buffer contain received (applicaion) * data. Returned by HAL. * @sizes: Array of receive buffer sizes. Contains 5 sizes: one size per * buffer from @dma_pointers. Returned by HAL. * * Retrieve protocol data from the completed 5-buffer-mode Rx descriptor. * This inline helper-function uses completed descriptor to populate receive * buffer pointer and other "out" parameters. The function always succeeds. * * See also: vxge_hal_ring_rxd_3b_get(), vxge_hal_ring_rxd_5b_get(). */ static inline /* LINTED */ void vxge_hal_ring_rxd_5b_get( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh, dma_addr_t dma_pointers[], int sizes[]) { vxge_hal_ring_rxd_5_t *rxdp = (vxge_hal_ring_rxd_5_t *) rxdh; dma_pointers[0] = rxdp->buffer0_ptr; sizes[0] = (u32) VXGE_HAL_RING_RXD_5_BUFFER0_SIZE_GET(rxdp->control_1); dma_pointers[1] = rxdp->buffer1_ptr; sizes[1] = (u32) VXGE_HAL_RING_RXD_5_BUFFER1_SIZE_GET(rxdp->control_1); dma_pointers[2] = rxdp->buffer2_ptr; sizes[2] = (u32) VXGE_HAL_RING_RXD_5_BUFFER2_SIZE_GET(rxdp->control_1); dma_pointers[3] = rxdp->buffer3_ptr; sizes[3] = (u32) VXGE_HAL_RING_RXD_5_BUFFER3_SIZE_GET(rxdp->control_2); dma_pointers[4] = rxdp->buffer4_ptr; sizes[4] = (u32) VXGE_HAL_RING_RXD_5_BUFFER3_SIZE_GET(rxdp->control_2); } /* * vxge_hal_ring_rxd_1b_info_get - Get extended information associated with * a completed receive descriptor for 1b mode. * @vpath_handle: Virtual Path handle. * @rxdh: Descriptor handle. * @rxd_info: Descriptor information * * Retrieve extended information associated with a completed receive descriptor. * */ static inline /* LINTED */ void vxge_hal_ring_rxd_1b_info_get( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh, vxge_hal_ring_rxd_info_t *rxd_info) { vxge_hal_ring_rxd_1_t *rxdp = (vxge_hal_ring_rxd_1_t *) rxdh; rxd_info->syn_flag = (u32) VXGE_HAL_RING_RXD_SYN_GET(rxdp->control_0); rxd_info->is_icmp = (u32) VXGE_HAL_RING_RXD_IS_ICMP_GET(rxdp->control_0); rxd_info->fast_path_eligible = (u32) VXGE_HAL_RING_RXD_FAST_PATH_ELIGIBLE_GET(rxdp->control_0); rxd_info->l3_cksum_valid = (u32) VXGE_HAL_RING_RXD_L3_CKSUM_CORRECT_GET(rxdp->control_0); rxd_info->l3_cksum = (u32) VXGE_HAL_RING_RXD_L3_CKSUM_GET(rxdp->control_0); rxd_info->l4_cksum_valid = (u32) VXGE_HAL_RING_RXD_L4_CKSUM_CORRECT_GET(rxdp->control_0); rxd_info->l4_cksum = (u32) VXGE_HAL_RING_RXD_L4_CKSUM_GET(rxdp->control_0); rxd_info->frame = (u32) VXGE_HAL_RING_RXD_ETHER_ENCAP_GET(rxdp->control_0); rxd_info->proto = (u32) VXGE_HAL_RING_RXD_FRAME_PROTO_GET(rxdp->control_0); rxd_info->is_vlan = (u32) VXGE_HAL_RING_RXD_IS_VLAN_GET(rxdp->control_0); rxd_info->vlan = (u32) VXGE_HAL_RING_RXD_VLAN_TAG_GET(rxdp->control_1); rxd_info->rth_bucket = (u32) VXGE_HAL_RING_RXD_RTH_BUCKET_GET(rxdp->control_0); rxd_info->rth_it_hit = (u32) VXGE_HAL_RING_RXD_RTH_IT_HIT_GET(rxdp->control_0); rxd_info->rth_spdm_hit = (u32) VXGE_HAL_RING_RXD_RTH_SPDM_HIT_GET(rxdp->control_0); rxd_info->rth_hash_type = (u32) VXGE_HAL_RING_RXD_RTH_HASH_TYPE_GET(rxdp->control_0); rxd_info->rth_value = (u32) VXGE_HAL_RING_RXD_1_RTH_HASH_VAL_GET(rxdp->control_1); } /* * vxge_hal_ring_rxd_3b_5b_info_get - Get extended information associated with * a completed receive descriptor for 3b & 5b mode. * @vpath_handle: Virtual Path handle. * @rxdh: Descriptor handle. * @rxd_info: Descriptor information * * Retrieve extended information associated with a completed receive descriptor. * */ static inline /* LINTED */ void vxge_hal_ring_rxd_3b_5b_info_get( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh, vxge_hal_ring_rxd_info_t *rxd_info) { vxge_hal_ring_rxd_3_t *rxdp = (vxge_hal_ring_rxd_3_t *) rxdh; rxd_info->syn_flag = (u32) VXGE_HAL_RING_RXD_SYN_GET(rxdp->control_0); rxd_info->is_icmp = (u32) VXGE_HAL_RING_RXD_IS_ICMP_GET(rxdp->control_0); rxd_info->fast_path_eligible = (u32) VXGE_HAL_RING_RXD_FAST_PATH_ELIGIBLE_GET(rxdp->control_0); rxd_info->l3_cksum_valid = (u32) VXGE_HAL_RING_RXD_L3_CKSUM_CORRECT_GET(rxdp->control_0); rxd_info->l3_cksum = (u32) VXGE_HAL_RING_RXD_L3_CKSUM_GET(rxdp->control_0); rxd_info->l4_cksum_valid = (u32) VXGE_HAL_RING_RXD_L4_CKSUM_CORRECT_GET(rxdp->control_0); rxd_info->l4_cksum = (u32) VXGE_HAL_RING_RXD_L4_CKSUM_GET(rxdp->control_0); rxd_info->frame = (u32) VXGE_HAL_RING_RXD_ETHER_ENCAP_GET(rxdp->control_0); rxd_info->proto = (u32) VXGE_HAL_RING_RXD_FRAME_PROTO_GET(rxdp->control_0); rxd_info->is_vlan = (u32) VXGE_HAL_RING_RXD_IS_VLAN_GET(rxdp->control_0); rxd_info->vlan = (u32) VXGE_HAL_RING_RXD_VLAN_TAG_GET(rxdp->control_1); rxd_info->rth_bucket = (u32) VXGE_HAL_RING_RXD_RTH_BUCKET_GET(rxdp->control_0); rxd_info->rth_it_hit = (u32) VXGE_HAL_RING_RXD_RTH_IT_HIT_GET(rxdp->control_0); rxd_info->rth_spdm_hit = (u32) VXGE_HAL_RING_RXD_RTH_SPDM_HIT_GET(rxdp->control_0); rxd_info->rth_hash_type = (u32) VXGE_HAL_RING_RXD_RTH_HASH_TYPE_GET(rxdp->control_0); rxd_info->rth_value = (u32) VXGE_HAL_RING_RXD_3_RTH_HASH_VALUE_GET( rxdp->buffer0_ptr); } /* * vxge_hal_device_is_privileged * @host_type: host type. * @func_id: function id. * */ vxge_hal_status_e vxge_hal_device_is_privileged( u32 host_type, u32 func_id); /* * vxge_hal_ring_rxd_private_get - Get ULD private per-descriptor data * @vpath_handle: Virtual Path handle. * @rxdh: Descriptor handle. * * Returns: private ULD info associated with the descriptor. * ULD requests per-descriptor space via vxge_hal_ring_attr. * */ void * vxge_hal_ring_rxd_private_get( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh); /* * vxge_hal_ring_rxd_free - Free descriptor. * @vpath_handle: Virtual Path handle. * @rxdh: Descriptor handle. * * Free the reserved descriptor. This operation is "symmetrical" to * vxge_hal_ring_rxd_reserve. The "free-ing" completes the descriptor's * lifecycle. * * After free-ing (see vxge_hal_ring_rxd_free()) the descriptor again can * be: * * - reserved (vxge_hal_ring_rxd_reserve); * * - posted (vxge_hal_ring_rxd_post); * * - completed (vxge_hal_ring_rxd_next_completed); * * - and recycled again (vxge_hal_ring_rxd_free). * * For alternative state transitions and more details please refer to * the design doc. * */ void vxge_hal_ring_rxd_free( vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh); /* * Fifo */ /* * TX Descriptor */ /* * enum vxge_hal_txdl_state_e - Descriptor (TXDL) state. * @VXGE_HAL_TXDL_STATE_NONE: Invalid state. * @VXGE_HAL_TXDL_STATE_AVAIL: Descriptor is available for reservation. * @VXGE_HAL_TXDL_STATE_POSTED: Descriptor is posted for processing by the * device. * @VXGE_HAL_TXDL_STATE_FREED: Descriptor is free and can be reused for * filling-in and posting later. * * X3100/HAL descriptor states. * */ typedef enum vxge_hal_txdl_state_e { VXGE_HAL_TXDL_STATE_NONE = 0, VXGE_HAL_TXDL_STATE_AVAIL = 1, VXGE_HAL_TXDL_STATE_POSTED = 2, VXGE_HAL_TXDL_STATE_FREED = 3 } vxge_hal_txdl_state_e; /* * enum vxge_hal_fifo_tcode_e - tcodes used in fifo * @VXGE_HAL_FIFO_T_CODE_OK: Transfer OK * @VXGE_HAL_FIFO_T_CODE_PCI_READ_CORRUPT: PCI read transaction (either TxD or * frame data) returned with corrupt data. * @VXGE_HAL_FIFO_T_CODE_PCI_READ_FAIL:PCI read transaction was returned * with no data. * @VXGE_HAL_FIFO_T_CODE_INVALID_MSS: The host attempted to send either a * frame or LSO MSS that was too long (>9800B). * @VXGE_HAL_FIFO_T_CODE_LSO_ERROR: Error detected during TCP/UDP Large Send * Offload operation, due to improper header template, * unsupported protocol, etc. * @VXGE_HAL_FIFO_T_CODE_UNUSED: Unused * @VXGE_HAL_FIFO_T_CODE_MULTI_ERROR: Set to 1 by the adapter if multiple * data buffer transfer errors are encountered (see below). * Otherwise it is set to 0. * * These tcodes are returned in various API for TxD status */ typedef enum vxge_hal_fifo_tcode_e { VXGE_HAL_FIFO_T_CODE_OK = 0x0, VXGE_HAL_FIFO_T_CODE_PCI_READ_CORRUPT = 0x1, VXGE_HAL_FIFO_T_CODE_PCI_READ_FAIL = 0x2, VXGE_HAL_FIFO_T_CODE_INVALID_MSS = 0x3, VXGE_HAL_FIFO_T_CODE_LSO_ERROR = 0x4, VXGE_HAL_FIFO_T_CODE_UNUSED = 0x7, VXGE_HAL_FIFO_T_CODE_MULTI_ERROR = 0x8 } vxge_hal_fifo_tcode_e; /* * enum vxge_hal_fifo_host_steer_e - Host steer type * @VXGE_HAL_FIFO_HOST_STEER_NORMAL: Normal. Use Destination/MAC Address * lookup to determine the transmit porte * @VXGE_HAL_FIFO_HOST_STEER_PORT1: Send on physical Port1 * @VXGE_HAL_FIFO_HOST_STEER_PORT0: Send on physical Port0 * @VXGE_HAL_FIFO_HOST_STEER_BOTH: Send on both ports. * * Host steer type */ typedef enum vxge_hal_fifo_host_steer_e { VXGE_HAL_FIFO_HOST_STEER_NORMAL = 0x0, VXGE_HAL_FIFO_HOST_STEER_PORT1 = 0x1, VXGE_HAL_FIFO_HOST_STEER_PORT0 = 0x2, VXGE_HAL_FIFO_HOST_STEER_BOTH = 0x3 } vxge_hal_fifo_host_steer_e; /* * enum vxge_hal_fifo_gather_code_e - Gather codes used in fifo TxD * @VXGE_HAL_FIFO_GATHER_CODE_FIRST: First TxDL * @VXGE_HAL_FIFO_GATHER_CODE_MIDDLE: Middle TxDL * @VXGE_HAL_FIFO_GATHER_CODE_LAST: Last TxDL * @VXGE_HAL_FIFO_GATHER_CODE_FIRST_LAST: First and Last TxDL. * * These gather codes are used to indicate the position of a TxD in a TxD list */ typedef enum vxge_hal_fifo_gather_code_e { VXGE_HAL_FIFO_GATHER_CODE_FIRST = 0x2, VXGE_HAL_FIFO_GATHER_CODE_MIDDLE = 0x0, VXGE_HAL_FIFO_GATHER_CODE_LAST = 0x1, VXGE_HAL_FIFO_GATHER_CODE_FIRST_LAST = 0x3 } vxge_hal_fifo_gather_code_e; /* * enum vxge_hal_fifo_lso_frm_encap_e - LSO Frame Encapsulation * @VXGE_HAL_FIFO_LSO_FRM_ENCAP_AUTO: auto mode (best guess) * @VXGE_HAL_FIFO_LSO_FRM_ENCAP_LLC: LLC * @VXGE_HAL_FIFO_LSO_FRM_ENCAP_SNAP: SNAP * @VXGE_HAL_FIFO_LSO_FRM_ENCAP_DIX: DIX * * LSO Frame Encapsulation type */ typedef enum vxge_hal_fifo_lso_frm_encap_e { VXGE_HAL_FIFO_LSO_FRM_ENCAP_AUTO = 0x0, VXGE_HAL_FIFO_LSO_FRM_ENCAP_LLC = 0x1, VXGE_HAL_FIFO_LSO_FRM_ENCAP_SNAP = 0x2, VXGE_HAL_FIFO_LSO_FRM_ENCAP_DIX = 0x3 } vxge_hal_fifo_lso_frm_encap_e; /* * struct vxge_hal_fifo_txd_t - Transmit Descriptor * @control_0: Bits 0 to 6 - Reserved. * Bit 7 - List Ownership. This field should be initialized * to '1' by the driver before the transmit list pointer is * written to the adapter. This field will be set to '0' by the * adapter once it has completed transmitting the frame or frames * in the list. Note - This field is only valid in TxD0. * Additionally, for multi-list sequences, the driver should not * release any buffers until the ownership of the last list in the * multi-list sequence has been returned to the host. * Bits 8 to 11 - Reserved * Bits 12 to 15 - Transfer_Code. This field is only valid in * TxD0. It is used to describe the status of the transmit data * buffer transfer. This field is always overwritten by the * adapter, so this field may be initialized to any value. * Bits 16 to 17 - Host steering. This field allows the host to * override the selection of the physical transmit port. * Attention: * Normal sounds as if learned from the switch rather than from * the aggregation algorythms. * 00: Normal. Use Destination/MAC Address * lookup to determine the transmit port. * 01: Send on physical Port1. * 10: Send on physical Port0. * 11: Send on both ports. * Bits 18 to 21 - Reserved * Bits 22 to 23 - Gather_Code. This field is set by the host and * is used to describe how individual buffers comprise a frame. * 10: First descriptor of a frame. * 00: Middle of a multi-descriptor frame. * 01: Last descriptor of a frame. * 11: First and last descriptor of a frame (the entire frame * resides in a single buffer). * For multi-descriptor frames, the only valid gather code sequence * is {10, [00], 01}. In other words,the descriptors must be placed * in the list in the correct order. * Bits 24 to 27 - Reserved * Bits 28 to 29 - LSO_Frm_Encap. LSO Frame Encapsulation * definition. Only valid in TxD0. This field allows the host to * indicate the Ethernet encapsulation of an outbound LSO packet. * 00 - classic mode (best guess) * 01 - LLC * 10 - SNAP * 11 - DIX * If "classic mode" is selected, the adapter will attempt to * decode the frame's Ethernet encapsulation by examining the L/T * field as follows: * <= 0x05DC LLC/SNAP encoding; must examine DSAP/SSAP to determine * if packet is IPv4 or IPv6. * 0x8870 Jumbo-SNAP encoding. * 0x0800 IPv4 DIX encoding * 0x86DD IPv6 DIX encoding * others illegal encapsulation * Bits 30 - LSO_ Flag. Large Send Offload (LSO) flag. * Set to 1 to perform segmentation offload for TCP/UDP. * This field is valid only in TxD0. * Bits 31 to 33 - Reserved. * Bits 34 to 47 - LSO_MSS. TCP/UDP LSO Maximum Segment Size * This field is meaningful only when LSO_Control is non-zero. * When LSO_Control is set to TCP_LSO, the single (possibly large) * TCP segment described by this TxDL will be sent as a series of * TCP segments each of which contains no more than LSO_MSS * payload bytes. * When LSO_Control is set to UDP_LSO, the single (possibly large) * UDP datagram described by this TxDL will be sent as a series of * UDP datagrams each of which contains no more than LSO_MSS * payload bytes. * All outgoing frames from this TxDL will have LSO_MSS bytes of * UDP or TCP payload, with the exception of the last, which will * have <= LSO_MSS bytes of payload. * Bits 48 to 63 - Buffer_Size. Number of valid bytes in the * buffer to be read by the adapter. This field is written by the * host. A value of 0 is illegal. * Bits 32 to 63 - This value is written by the adapter upon * completion of a UDP or TCP LSO operation and indicates the * number of UDP or TCP payload bytes that were transmitted. * 0x0000 will bereturned for any non-LSO operation. * @control_1: Bits 0 to 4 - Reserved. * Bit 5 - Tx_CKO_IPv4 Set to a '1' to enable IPv4 header checksum * offload. This field is only valid in the first TxD of a frame. * Bit 6 - Tx_CKO_TCP Set to a '1' to enable TCP checksum offload. * This field is only valid in the first TxD of a frame (the TxD's * gather code must be 10 or 11). The driver should only set this * Bit if it can guarantee that TCP is present. * Bit 7 - Tx_CKO_UDP Set to a '1' to enable UDP checksum offload. * This field is only valid in the first TxD of a frame (the TxD's * gather code must be 10 or 11). The driver should only set this * Bit if it can guarantee that UDP is present. * Bits 8 to 14 - Reserved. * Bit 15 - Tx_VLAN_Enable VLAN tag insertion flag. Set to a '1' to * instruct the adapter to insert the VLAN tag specified by the * Tx_VLAN_Tag field. This field is only valid in the first TxD of * a frame. * Bits 16 to 31 - Tx_VLAN_Tag. Variable portion of the VLAN tag * to be inserted into the frame by the adapter(the first two bytes * of a VLAN tag are always 0x8100).This field is only valid if the * Tx_VLAN_Enable field is set to '1'. * Bits 32 to 33 - Reserved. * Bits 34 to 39 - Tx_Int_Number. Indicates which Tx interrupt * number the frame associated with. This field is written by the * host. It is only valid in the first TxD of a frame. * Bits 40 to 42 - Reserved. * Bit 43 - Set to 1 to exclude the frame from bandwidth metering * functions. This field is valid only in the first TxD * of a frame. * Bits 44 to 45 - Reserved. * Bit 46 - Tx_Int_Per_List Set to a '1' to instruct the adapter to * generate an interrupt as soon as all of the frames in the list * have been transmitted. In order to have per-frame interrupts, * the driver should place a maximum of one frame per list. This * field is only valid in the first TxD of a frame. * Bit 47 - Tx_Int_Utilization Set to a '1' to instruct the adapter * to count the frame toward the utilization interrupt specified in * the Tx_Int_Number field. This field is only valid in the first * TxD of a frame. * Bits 48 to 63 - Reserved. * @buffer_pointer: Buffer start address. * @host_control: Host_Control.Opaque 64bit data stored by ULD inside the X3100 * descriptor prior to posting the latter on the fifo * via vxge_hal_fifo_txdl_post().The %host_control is returned as is to * the ULD with each completed descriptor. * * Transmit descriptor (TxD).Fifo descriptor contains configured number * (list) of TxDs. * For more details please refer to X3100 User Guide, * Section 5.4.2 "Transmit Descriptor (TxD) Format". */ typedef struct vxge_hal_fifo_txd_t { u64 control_0; #define VXGE_HAL_FIFO_TXD_LIST_OWN_GET(ctrl0) bVAL1(ctrl0, 7) #define VXGE_HAL_FIFO_TXD_LIST_OWN_ADAPTER mBIT(7) #define VXGE_HAL_FIFO_TXD_T_CODE_GET(ctrl0) bVAL4(ctrl0, 12) #define VXGE_HAL_FIFO_TXD_T_CODE(val) vBIT(val, 12, 4) #define VXGE_HAL_FIFO_TXD_T_CODE_OK VXGE_HAL_FIFO_T_CODE_OK #define VXGE_HAL_FIFO_TXD_T_CODE_PCI_READ_CORRUPT \ VXGE_HAL_FIFO_T_CODE_PCI_READ_CORRUPT #define VXGE_HAL_FIFO_TXD_T_CODE_PCI_READ_FAIL \ VXGE_HAL_FIFO_T_CODE_PCI_READ_FAIL #define VXGE_HAL_FIFO_TXD_T_CODE_INVALID_MSS VXGE_HAL_FIFO_T_CODE_INVALID_MSS #define VXGE_HAL_FIFO_TXD_T_CODE_LSO_ERROR VXGE_HAL_FIFO_T_CODE_LSO_ERROR #define VXGE_HAL_FIFO_TXD_T_CODE_UNUSED VXGE_HAL_FIFO_T_CODE_UNUSED #define VXGE_HAL_FIFO_TXD_T_CODE_MULTI_ERROR VXGE_HAL_FIFO_T_CODE_MULTI_ERROR #define VXGE_HAL_FIFO_TXD_HOST_STEER_GET(ctrl0) bVAL2(ctrl0, 16) #define VXGE_HAL_FIFO_TXD_HOST_STEER(val) vBIT(val, 16, 2) #define VXGE_HAL_FIFO_TXD_HOST_STEER_NORMAL VXGE_HAL_FIFO_HOST_STEER_NORMAL #define VXGE_HAL_FIFO_TXD_HOST_STEER_PORT1 VXGE_HAL_FIFO_HOST_STEER_PORT1 #define VXGE_HAL_FIFO_TXD_HOST_STEER_PORT0 VXGE_HAL_FIFO_HOST_STEER_PORT0 #define VXGE_HAL_FIFO_TXD_HOST_STEER_BOTH VXGE_HAL_FIFO_HOST_STEER_BOTH #define VXGE_HAL_FIFO_TXD_GATHER_CODE_GET(ctrl0) bVAL2(ctrl0, 22) #define VXGE_HAL_FIFO_TXD_GATHER_CODE(val) vBIT(val, 22, 2) #define VXGE_HAL_FIFO_TXD_GATHER_CODE_FIRST VXGE_HAL_FIFO_GATHER_CODE_FIRST #define VXGE_HAL_FIFO_TXD_GATHER_CODE_MIDDLE VXGE_HAL_FIFO_GATHER_CODE_MIDDLE #define VXGE_HAL_FIFO_TXD_GATHER_CODE_LAST VXGE_HAL_FIFO_GATHER_CODE_LAST #define VXGE_HAL_FIFO_TXD_GATHER_CODE_FIRST_LAST \ VXGE_HAL_FIFO_GATHER_CODE_FIRST_LAST #define VXGE_HAL_FIFO_TXD_LSO_FRM_ENCAP_GET(ctrl0) bVAL2(ctrl0, 28) #define VXGE_HAL_FIFO_TXD_LSO_FRM_ENCAP(val) vBIT(val, 28, 2) #define VXGE_HAL_FIFO_TXD_LSO_FRM_ENCAP_AUTO VXGE_HAL_FIFO_LSO_FRM_ENCAP_AUTO #define VXGE_HAL_FIFO_TXD_LSO_FRM_ENCAP_LLC VXGE_HAL_FIFO_LSO_FRM_ENCAP_LLC #define VXGE_HAL_FIFO_TXD_LSO_FRM_ENCAP_SNAP VXGE_HAL_FIFO_LSO_FRM_ENCAP_SNAP #define VXGE_HAL_FIFO_TXD_LSO_FRM_ENCAP_DIX VXGE_HAL_FIFO_LSO_FRM_ENCAP_DIX #define VXGE_HAL_FIFO_TXD_LSO_FLAG_GET(ctrl0) bVAL1(ctrl0, 30) #define VXGE_HAL_FIFO_TXD_LSO_FLAG mBIT(30) #define VXGE_HAL_FIFO_TXD_LSO_MSS_GET(ctrl0) bVAL14(ctrl0, 34) #define VXGE_HAL_FIFO_TXD_LSO_MSS(val) vBIT(val, 34, 14) #define VXGE_HAL_FIFO_TXD_BUFFER_SIZE_GET(ctrl0) bVAL16(ctrl0, 48) #define VXGE_HAL_FIFO_TXD_BUFFER_SIZE(val) vBIT(val, 48, 16) #define VXGE_HAL_FIFO_TXD_LSO_BYTES_SENT_GET(ctrl0) bVAL32(ctrl0, 32) #define VXGE_HAL_FIFO_TXD_LSO_BYTES_SENT(val) vBIT(val, 32, 32) u64 control_1; #define VXGE_HAL_FIFO_TXD_TX_CKO_IPV4_EN_GET(ctrl1) bVAL1(ctrl1, 5) #define VXGE_HAL_FIFO_TXD_TX_CKO_IPV4_EN mBIT(5) #define VXGE_HAL_FIFO_TXD_TX_CKO_TCP_EN_GET(ctrl1) bVAL1(ctrl1, 6) #define VXGE_HAL_FIFO_TXD_TX_CKO_TCP_EN mBIT(6) #define VXGE_HAL_FIFO_TXD_TX_CKO_UDP_EN_GET(ctrl1) bVAL1(ctrl1, 7) #define VXGE_HAL_FIFO_TXD_TX_CKO_UDP_EN mBIT(7) #define VXGE_HAL_FIFO_TXD_TX_CKO_CONTROL (mBIT(5)|mBIT(6)|mBIT(7)) #define VXGE_HAL_FIFO_TXD_VLAN_ENABLE_GET(ctrl1) bVAL1(ctrl1, 15) #define VXGE_HAL_FIFO_TXD_VLAN_ENABLE mBIT(15) #define VXGE_HAL_FIFO_TXD_VLAN_TAG_GET(ctrl1) bVAL16(ctrl1, 16) #define VXGE_HAL_FIFO_TXD_VLAN_TAG(val) vBIT(val, 16, 16) #define VXGE_HAL_FIFO_TXD_INT_NUMBER_GET(ctrl1) bVAL6(ctrl1, 34) #define VXGE_HAL_FIFO_TXD_INT_NUMBER(val) vBIT(val, 34, 6) #define VXGE_HAL_FIFO_TXD_NO_BW_LIMIT_GET(ctrl1) bVAL1(ctrl1, 43) #define VXGE_HAL_FIFO_TXD_NO_BW_LIMIT mBIT(43) #define VXGE_HAL_FIFO_TXD_INT_TYPE_PER_LIST_GET(ctrl1) bVAL1(ctrl1, 46) #define VXGE_HAL_FIFO_TXD_INT_TYPE_PER_LIST mBIT(46) #define VXGE_HAL_FIFO_TXD_INT_TYPE_UTILZ_GET(ctrl1) bVAL1(ctrl1, 47) #define VXGE_HAL_FIFO_TXD_INT_TYPE_UTILZ mBIT(47) u64 buffer_pointer; u64 host_control; } vxge_hal_fifo_txd_t; typedef vxge_hal_fifo_txd_t *vxge_hal_fifo_txdl_t; /* * function vxge_hal_fifo_callback_f - FIFO callback. * @vpath_handle: Virtual path whose Fifo "containing" 1 or more completed * descriptors. * @txdlh: First completed descriptor. * @txdl_priv: Pointer to per txdl space allocated * @t_code: Transfer code, as per X3100 User Guide. * Returned by HAL. * @host_control: Opaque 64bit data stored by ULD inside the X3100 * descriptor prior to posting the latter on the fifo * via vxge_hal_fifo_txdl_post(). The @host_control is returned * as is to the ULD with each completed descriptor. * @userdata: Opaque per-fifo data specified at fifo open * time, via vxge_hal_vpath_open(). * * Fifo completion callback (type declaration). A single per-fifo * callback is specified at fifo open time, via * vxge_hal_vpath_open(). Typically gets called as part of the processing * of the Interrupt Service Routine. * * Fifo callback gets called by HAL if, and only if, there is at least * one new completion on a given fifo. Upon processing the first @txdlh ULD * is _supposed_ to continue consuming completions using: * - vxge_hal_fifo_txdl_next_completed() * * Note that failure to process new completions in a timely fashion * leads to VXGE_HAL_INF_OUT_OF_DESCRIPTORS condition. * * Non-zero @t_code means failure to process transmit descriptor. * * In the "transmit" case the failure could happen, for instance, when the * link is down, in which case X3100 completes the descriptor because it * is not able to send the data out. * * For details please refer to X3100 User Guide. * * See also: vxge_hal_fifo_txdl_next_completed(), vxge_hal_fifo_txdl_term_f {}. */ typedef vxge_hal_status_e(*vxge_hal_fifo_callback_f) ( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h txdlh, void *txdl_priv, vxge_hal_fifo_tcode_e t_code, void *userdata); /* * function vxge_hal_fifo_txdl_init_f - Initialize descriptor callback. * @vpath_handle: Virtual path whose Fifo "containing" the @txdlh descriptor. * @txdlh: Descriptor. * @txdl_priv: Pointer to per txdl space allocated * @index: Index of the descriptor in the fifo's set of descriptors. * @userdata: Per-fifo user data (a.k.a. context) specified at * fifo open time, via vxge_hal_vpath_open(). * @reopen: See vxge_hal_reopen_e {}. * * Initialize descriptor callback. Unless NULL is specified in the * vxge_hal_fifo_attr_t {} structure passed to vxge_hal_vpath_open()), * HAL invokes the callback as part of the vxge_hal_vpath_open() * implementation. * The ULD could use the callback to pre-set DMA mappings and/or alignment * buffers. * * See also: vxge_hal_fifo_attr_t {}, vxge_hal_fifo_txdl_term_f {}. */ typedef vxge_hal_status_e(*vxge_hal_fifo_txdl_init_f) ( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h txdlh, void *txdl_priv, u32 index, void *userdata, vxge_hal_reopen_e reopen); /* * function vxge_hal_fifo_txdl_term_f - Terminate descriptor callback. * @vpath_handle: Virtual path whose Fifo "containing" the @txdlh descriptor. * @txdlh: First completed descriptor. * @txdl_priv: Pointer to per txdl space allocated * @state: One of the vxge_hal_txdl_state_e {} enumerated states. * @userdata: Per-fifo user data (a.k.a. context) specified at * fifo open time, via vxge_hal_vpath_open(). * @reopen: See vxge_hal_reopen_e {}. * * Terminate descriptor callback. Unless NULL is specified in the * vxge_hal_fifo_attr_t {} structure passed to vxge_hal_vpath_open()), * HAL invokes the callback as part of closing fifo, prior to * de-allocating the ring and associated data structures * (including descriptors). * ULD should utilize the callback to (for instance) unmap * and free DMA data buffers associated with the posted (state = * VXGE_HAL_TXDL_STATE_POSTED) descriptors, * as well as other relevant cleanup functions. * * See also: vxge_hal_fifo_attr_t {}, vxge_hal_fifo_txdl_init_f {}. */ typedef void (*vxge_hal_fifo_txdl_term_f) ( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h txdlh, void *txdl_priv, vxge_hal_txdl_state_e state, void *userdata, vxge_hal_reopen_e reopen); /* * struct vxge_hal_fifo_attr_t - Fifo open "template". * @callback: Fifo completion callback. HAL invokes the callback when there * are new completions on that fifo. In many implementations * the @callback executes in the hw interrupt context. * @txdl_init: Fifo's descriptor-initialize callback. * See vxge_hal_fifo_txdl_init_f {}. * If not NULL, HAL invokes the callback when opening * the fifo via vxge_hal_vpath_open(). * @txdl_term: Fifo's descriptor-terminate callback. If not NULL, * HAL invokes the callback when closing the corresponding fifo. * See also vxge_hal_fifo_txdl_term_f {}. * @userdata: User-defined "context" of _that_ fifo. Passed back to the * user as one of the @callback, @txdl_init, and @txdl_term arguments. * @per_txdl_space: If specified (i.e., greater than zero): extra space * reserved by HAL per each transmit descriptor. Can be used to * store, and retrieve on completion, information specific * to the upper-layer. * * Fifo open "template". User fills the structure with fifo * attributes and passes it to vxge_hal_vpath_open(). */ typedef struct vxge_hal_fifo_attr_t { vxge_hal_fifo_callback_f callback; vxge_hal_fifo_txdl_init_f txdl_init; vxge_hal_fifo_txdl_term_f txdl_term; void *userdata; u32 per_txdl_space; } vxge_hal_fifo_attr_t; /* * vxge_hal_fifo_doorbell_reset - Resets the doorbell fifo * @vpath_handle: Vpath Handle * * This function resets the doorbell fifo during if fifo error occurs */ vxge_hal_status_e vxge_hal_fifo_doorbell_reset( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_fifo_txdl_reserve - Reserve fifo descriptor. * @vpath_handle: virtual path handle. * @txdlh: Reserved descriptor. On success HAL fills this "out" parameter * with a valid handle. * @txdl_priv: Buffer to return the pointer to per txdl space allocated * * Reserve a single TxDL (that is, fifo descriptor) * for the subsequent filling-in by upper layerdriver (ULD)) * and posting on the corresponding fifo * via vxge_hal_fifo_txdl_post(). * * Note: it is the responsibility of ULD to reserve multiple descriptors * for lengthy (e.g., LSO) transmit operation. A single fifo descriptor * carries up to configured number (fifo.max_frags) of contiguous buffers. * * Returns: VXGE_HAL_OK - success; * VXGE_HAL_INF_OUT_OF_DESCRIPTORS - Currently no descriptors available * */ vxge_hal_status_e vxge_hal_fifo_txdl_reserve( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h *txdlh, void **txdl_priv); /* * vxge_hal_fifo_txdl_cksum_set_bits - Offload checksum. * @txdlh: Descriptor handle. * @cksum_bits: Specifies which checksums are to be offloaded: IPv4, * and/or TCP and/or UDP. * * Ask X3100 to calculate IPv4 & transport checksums for _this_ transmit * descriptor. * This API is part of the preparation of the transmit descriptor for posting * (via vxge_hal_fifo_txdl_post()). The related "preparation" APIs include * vxge_hal_fifo_txdl_mss_set(), vxge_hal_fifo_txdl_buffer_set_aligned(), * and vxge_hal_fifo_txdl_buffer_set(). * All these APIs fill in the fields of the fifo descriptor, * in accordance with the X3100 specification. * */ static inline /* LINTED */ void vxge_hal_fifo_txdl_cksum_set_bits( vxge_hal_txdl_h txdlh, u64 cksum_bits) { vxge_hal_fifo_txd_t *txdp = (vxge_hal_fifo_txd_t *) txdlh; txdp->control_1 |= cksum_bits; } /* * vxge_hal_fifo_txdl_interrupt_type_set - Set the interrupt type for the txdl * @txdlh: Descriptor handle. * @interrupt_type: utiliz based interupt or List interrupt * * vxge_hal_fifo_txdl_interrupt_type_set is used to set the interrupt type for * each xmit txdl dynamically */ static inline /* LINTED */ void vxge_hal_fifo_txdl_interrupt_type_set( vxge_hal_txdl_h txdlh, u64 interrupt_type) { vxge_hal_fifo_txd_t *txdp = (vxge_hal_fifo_txd_t *) txdlh; txdp->control_1 |= interrupt_type; } /* * vxge_hal_fifo_txdl_lso_set - Set LSO Parameters. * @txdlh: Descriptor handle. * @encap: LSO Encapsulation * @mss: MSS size for LSO. * * This API is part of the preparation of the transmit descriptor for posting * (via vxge_hal_fifo_txdl_post()). The related "preparation" APIs include * vxge_hal_fifo_txdl_buffer_set(), vxge_hal_fifo_txdl_buffer_set_aligned(), * and vxge_hal_fifo_txdl_cksum_set_bits(). * All these APIs fill in the fields of the fifo descriptor, * in accordance with the X3100 specification. * */ static inline /* LINTED */ void vxge_hal_fifo_txdl_lso_set( vxge_hal_txdl_h txdlh, u32 encap, u32 mss) { vxge_hal_fifo_txd_t *txdp = (vxge_hal_fifo_txd_t *) txdlh; txdp->control_0 |= VXGE_HAL_FIFO_TXD_LSO_FRM_ENCAP(encap) | VXGE_HAL_FIFO_TXD_LSO_FLAG | VXGE_HAL_FIFO_TXD_LSO_MSS(mss); } /* * vxge_hal_fifo_txdl_lso_bytes_sent - Get the lso bytes sent. * @txdlh: Descriptor handle. * * Returns the lso bytes sent */ static inline /* LINTED */ u32 vxge_hal_fifo_txdl_lso_bytes_sent( vxge_hal_txdl_h txdlh) { vxge_hal_fifo_txd_t *txdp = (vxge_hal_fifo_txd_t *) txdlh; return (u32) VXGE_HAL_FIFO_TXD_LSO_BYTES_SENT_GET(txdp->control_0); } /* * vxge_hal_fifo_txdl_vlan_set - Set VLAN tag. * @txdlh: Descriptor handle. * @vlan_tag: 16bit VLAN tag. * * Insert VLAN tag into specified transmit descriptor. * The actual insertion of the tag into outgoing frame is done by the hardware. */ static inline /* LINTED */ void vxge_hal_fifo_txdl_vlan_set( vxge_hal_txdl_h txdlh, u16 vlan_tag) { vxge_hal_fifo_txd_t *txdp = (vxge_hal_fifo_txd_t *) txdlh; txdp->control_1 |= VXGE_HAL_FIFO_TXD_VLAN_ENABLE; txdp->control_1 |= VXGE_HAL_FIFO_TXD_VLAN_TAG(vlan_tag); } /* * vxge_hal_fifo_txdl_buffer_set - Set transmit buffer pointer in the * descriptor. * @vpath_handle: virtual path handle. * @txdlh: Descriptor handle. - * @frag_idx: Index of the data buffer in the caller's scatter-gather list¤ + * @frag_idx: Index of the data buffer in the caller's scatter-gather list * (of buffers). * @dma_pointer: DMA address of the data buffer referenced by @frag_idx. * @size: Size of the data buffer (in bytes). * * This API is part of the preparation of the transmit descriptor for posting * (via vxge_hal_fifo_txdl_post()). The related "preparation" APIs include * vxge_hal_fifo_txdl_mss_set() and vxge_hal_fifo_txdl_cksum_set_bits(). * All three APIs fill in the fields of the fifo descriptor, * in accordance with the X3100 specification. * */ void vxge_hal_fifo_txdl_buffer_set( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h txdlh, u32 frag_idx, dma_addr_t dma_pointer, unsigned long size); /* * vxge_hal_fifo_txdl_buffer_set_aligned - Align transmit buffer and fill * in fifo descriptor. * @vpath_handle: Virtual path handle. * @txdlh: Descriptor handle. - * @frag_idx: Index of the data buffer in the caller's scatter-gather list¤ + * @frag_idx: Index of the data buffer in the caller's scatter-gather list * (of buffers). * @vaddr: Virtual address of the data buffer. * @dma_pointer: DMA address of the data buffer referenced by @frag_idx. * @size: Size of the data buffer (in bytes). * @misaligned_size: Size (in bytes) of the misaligned portion of the * data buffer. Calculated by the caller, based on the platform/OS/other * specific criteria, which is outside of HAL's domain. See notes below. * * This API is part of the transmit descriptor preparation for posting * (via vxge_hal_fifo_txdl_post()). The related "preparation" APIs include * vxge_hal_fifo_txdl_mss_set() and vxge_hal_fifo_txdl_cksum_set_bits(). * All three APIs fill in the fields of the fifo descriptor, * in accordance with the X3100 specification. * On the PCI-X based systems aligning transmit data typically provides better * transmit performance. The typical alignment granularity: L2 cacheline size. * However, HAL does not make assumptions in terms of the alignment granularity; * this is specified via additional @misaligned_size parameter described above. * Prior to calling vxge_hal_fifo_txdl_buffer_set_aligned(), * ULD is supposed to check alignment of a given fragment/buffer. For this HAL * provides a separate vxge_hal_check_alignment() API sufficient to cover * most (but not all) possible alignment criteria. * If the buffer appears to be aligned, the ULD calls * vxge_hal_fifo_txdl_buffer_set(). * Otherwise, ULD calls vxge_hal_fifo_txdl_buffer_set_aligned(). * * Note; This API is a "superset" of vxge_hal_fifo_txdl_buffer_set(). In * addition to filling in the specified descriptor it aligns transmit data on * the specified boundary. * Note: Decision on whether to align or not to align a given contiguous * transmit buffer is outside of HAL's domain. To this end ULD can use any * programmable criteria, which can help to 1) boost transmit performance, * and/or 2) provide a workaround for PCI bridge bugs, if any. * */ vxge_hal_status_e vxge_hal_fifo_txdl_buffer_set_aligned( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h txdlh, u32 frag_idx, void *vaddr, dma_addr_t dma_pointer, u32 size, u32 misaligned_size); /* * vxge_hal_fifo_txdl_buffer_append - Append the contents of virtually * contiguous data buffer to a single physically contiguous buffer. * @vpath_handle: Virtual path handle. * @txdlh: Descriptor handle. * @vaddr: Virtual address of the data buffer. * @size: Size of the data buffer (in bytes). * * This API is part of the transmit descriptor preparation for posting * (via vxge_hal_fifo_txdl_post()). * The main difference of this API wrt to the APIs * vxge_hal_fifo_txdl_buffer_set_aligned() is that this API appends the * contents of virtually contiguous data buffers received from * upper layer into a single physically contiguous data buffer and the * device will do a DMA from this buffer. * * See Also: vxge_hal_fifo_txdl_buffer_finalize(), * vxge_hal_fifo_txdl_buffer_set(), * vxge_hal_fifo_txdl_buffer_set_aligned(). */ vxge_hal_status_e vxge_hal_fifo_txdl_buffer_append( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h txdlh, void *vaddr, u32 size); /* * vxge_hal_fifo_txdl_buffer_finalize - Prepares a descriptor that contains the * single physically contiguous buffer. * * @vpath_handle: Virtual path handle. * @txdlh: Descriptor handle. * @frag_idx: Index of the data buffer in the Txdl list. * * This API in conjuction with vxge_hal_fifo_txdl_buffer_append() prepares * a descriptor that consists of a single physically contiguous buffer * which inturn contains the contents of one or more virtually contiguous * buffers received from the upper layer. * * See Also: vxge_hal_fifo_txdl_buffer_append(). */ void vxge_hal_fifo_txdl_buffer_finalize( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h txdlh, u32 frag_idx); /* * vxge_hal_fifo_txdl_new_frame_set - Start the new packet by setting TXDL flags * @vpath_handle: virtual path handle. * @txdlh: Descriptor handle. * * This API is part of the preparation of the transmit descriptor for posting * (via vxge_hal_fifo_txdl_post()). This api is used to mark the end of previous * frame and start of a new frame. * */ void vxge_hal_fifo_txdl_new_frame_set( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h txdlh, u32 tagged); /* * vxge_hal_fifo_txdl_post - Post descriptor on the fifo. * @vpath_handle: Virtual path handle. * @txdlh: Descriptor obtained via vxge_hal_fifo_txdl_reserve() * * Post descriptor on the fifo for transmission. * Prior to posting the descriptor should be filled in accordance with * Host/X3100 interface specification for a given service (LL, etc.). * */ void vxge_hal_fifo_txdl_post( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h txdlh, u32 tagged); /* * vxge_hal_fifo_is_next_txdl_completed - Checks if the next txdl is completed * @vpath_handle: Virtual path handle. */ vxge_hal_status_e vxge_hal_fifo_is_next_txdl_completed( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_fifo_free_txdl_count_get - returns the number of txdls available * in the fifo * @vpath_handle: Virtual path handle. */ u32 vxge_hal_fifo_free_txdl_count_get( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_fifo_txdl_next_completed - Retrieve next completed descriptor. * @vpath_handle: Virtual path handle. * @txdlh: Descriptor handle. Returned by HAL. * @txdl_priv: Buffer to return the pointer to per txdl space allocated * @t_code: Transfer code, as per X3100 User Guide, * Transmit Descriptor Format. * Returned by HAL. * * Retrieve the _next_ completed descriptor. * HAL uses fifo callback (*vxge_hal_fifo_callback_f) to notifiy * upper-layer driver (ULD) of new completed descriptors. After that * the ULD can use vxge_hal_fifo_txdl_next_completed to retrieve the rest * completions (the very first completion is passed by HAL via * vxge_hal_fifo_callback_f). * * Implementation-wise, the upper-layer driver is free to call * vxge_hal_fifo_txdl_next_completed either immediately from inside the * fifo callback, or in a deferred fashion and separate (from HAL) * context. * * Non-zero @t_code means failure to process the descriptor. * The failure could happen, for instance, when the link is * down, in which case X3100 completes the descriptor because it * is not able to send the data out. * * For details please refer to X3100 User Guide. * * Returns: VXGE_HAL_OK - success. * VXGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed descriptors * are currently available for processing. * */ vxge_hal_status_e vxge_hal_fifo_txdl_next_completed( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h *txdlh, void **txdl_priv, vxge_hal_fifo_tcode_e *t_code); /* * vxge_hal_fifo_handle_tcode - Handle transfer code. * @vpath_handle: Virtual Path handle. * @txdlh: Descriptor handle. * @t_code: One of the enumerated (and documented in the X3100 user guide) * "transfer codes". * * Handle descriptor's transfer code. The latter comes with each completed * descriptor. * * Returns: one of the vxge_hal_status_e {} enumerated types. * VXGE_HAL_OK - for success. * VXGE_HAL_ERR_CRITICAL - when encounters critical error. */ vxge_hal_status_e vxge_hal_fifo_handle_tcode( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h txdlh, vxge_hal_fifo_tcode_e t_code); /* * vxge_hal_fifo_txdl_private_get - Retrieve per-descriptor private data. * @vpath_handle: Virtual path handle. * @txdlh: Descriptor handle. * * Retrieve per-descriptor private data. * Note that ULD requests per-descriptor space via * vxge_hal_fifo_attr_t passed to * vxge_hal_vpath_open(). * * Returns: private ULD data associated with the descriptor. */ void * vxge_hal_fifo_txdl_private_get( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h txdlh); /* * vxge_hal_fifo_txdl_free - Free descriptor. * @vpath_handle: Virtual path handle. * @txdlh: Descriptor handle. * * Free the reserved descriptor. This operation is "symmetrical" to * vxge_hal_fifo_txdl_reserve. The "free-ing" completes the descriptor's * lifecycle. * * After free-ing (see vxge_hal_fifo_txdl_free()) the descriptor again can * be: * * - reserved (vxge_hal_fifo_txdl_reserve); * * - posted (vxge_hal_fifo_txdl_post); * * - completed (vxge_hal_fifo_txdl_next_completed); * * - and recycled again (vxge_hal_fifo_txdl_free). * * For alternative state transitions and more details please refer to * the design doc. * */ void vxge_hal_fifo_txdl_free( vxge_hal_vpath_h vpath_handle, vxge_hal_txdl_h txdlh); /* * Device */ /* * enum vxge_hal_card_e - X3100 adapter type. * @VXGE_HAL_CARD_UNKNOWN: Unknown device. * @VXGE_HAL_CARD_TITAN: X3100 device. * * Enumerates X3100 adapter types. * * See also: vxge_hal_device_check_id(). */ typedef enum vxge_hal_card_e { VXGE_HAL_CARD_UNKNOWN = 0, VXGE_HAL_CARD_TITAN_1 = 1, VXGE_HAL_CARD_TITAN_1A = 2, VXGE_HAL_CARD_TITAN_2 = 3 } vxge_hal_card_e; /* * struct vxge_hal_device_attr_t - Device memory spaces. * @regh0: BAR0 mapped memory handle (Solaris), or simply PCI device @pdev * (Linux and the rest.) * @regh1: BAR1 mapped memory handle. Same comment as above. * @regh2: BAR2 mapped memory handle. Same comment as above. * @bar0: BAR0 virtual address. * @bar1: BAR1 virtual address. * @bar2: BAR2 virtual address. * @irqh: IRQ handle (Solaris). * @cfgh: Configuration space handle (Solaris), or PCI device @pdev (Linux). * @pdev: PCI device object. * * Device memory spaces. Includes configuration, BAR0, BAR1, etc. per device * mapped memories. Also, includes a pointer to OS-specific PCI device object. */ typedef struct vxge_hal_device_attr_t { pci_reg_h regh0; pci_reg_h regh1; pci_reg_h regh2; u8 *bar0; u8 *bar1; u8 *bar2; pci_irq_h irqh; pci_cfg_h cfgh; pci_dev_h pdev; } vxge_hal_device_attr_t; /* * enum vxge_hal_device_link_state_e - Link state enumeration. * @VXGE_HAL_LINK_NONE: Invalid link state. * @VXGE_HAL_LINK_DOWN: Link is down. * @VXGE_HAL_LINK_UP: Link is up. * */ typedef enum vxge_hal_device_link_state_e { VXGE_HAL_LINK_NONE, VXGE_HAL_LINK_DOWN, VXGE_HAL_LINK_UP } vxge_hal_device_link_state_e; /* * enum vxge_hal_device_data_rate_e - Data rate enumeration. * @VXGE_HAL_DATA_RATE_UNKNOWN: Unknown . * @VXGE_HAL_DATA_RATE_1G: 1G. * @VXGE_HAL_DATA_RATE_10G: 10G. * */ typedef enum vxge_hal_device_data_rate_e { VXGE_HAL_DATA_RATE_UNKNOWN, VXGE_HAL_DATA_RATE_1G, VXGE_HAL_DATA_RATE_10G } vxge_hal_device_data_rate_e; /* * enum vxge_hal_device_lag_mode_e - X3100 adapter lag mode * @VXGE_HAL_DEVICE_LAG_MODE_UNKNOWN: Unknown mode. * @VXGE_HAL_DEVICE_LAG_MODE_HW_LACP: Hardware Link Aggregation. * @VXGE_HAL_DEVICE_LAG_MODE_ACTIVE_PASSIVE: Active Passive. * @VXGE_HAL_DEVICE_LAG_MODE_SINGLE_PORT: Single Port. * @VXGE_HAL_DEVICE_LAG_MODE_DUAL_PORT: Dual Port. * @VXGE_HAL_DEVICE_LAG_MODE_DISABLED: Disabled. * * Enumerates X3100 adapter lag modes. * */ typedef enum vxge_hal_device_lag_mode_e { VXGE_HAL_DEVICE_LAG_MODE_UNKNOWN = 0, VXGE_HAL_DEVICE_LAG_MODE_HW_LACP, VXGE_HAL_DEVICE_LAG_MODE_ACTIVE_PASSIVE, VXGE_HAL_DEVICE_LAG_MODE_SINGLE_PORT, VXGE_HAL_DEVICE_LAG_MODE_DUAL_PORT, VXGE_HAL_DEVICE_LAG_MODE_DISABLED } vxge_hal_device_lag_mode_e; /* * enum vxge_hal_pci_e_signalling_rate_e - PCI-E Lane signalling rate * @VXGE_HAL_PCI_E_SIGNALLING_RATE_2_5GB: PCI-E signalling rate 2.5 GB * @VXGE_HAL_PCI_E_SIGNALLING_RATE_5GB: PCI-E signalling rate 5 GB * @VXGE_HAL_PCI_E_SIGNALLING_RATE_UNKNOWN: Unrecognized PCI bus frequency * * PCI-E Lane signalling rate */ typedef enum vxge_hal_pci_e_signalling_rate_e { VXGE_HAL_PCI_E_SIGNALLING_RATE_2_5GB = 1, VXGE_HAL_PCI_E_SIGNALLING_RATE_5GB = 2, VXGE_HAL_PCI_E_SIGNALLING_RATE_UNKNOWN = 0 } vxge_hal_pci_e_signalling_rate_e; /* * enum vxge_hal_pci_e_link_width_e - PCI-E Link width enumeration. * @VXGE_HAL_PCI_E_LINK_WIDTH_X1: 1 Lane. * @VXGE_HAL_PCI_E_LINK_WIDTH_X2: 2 Lane. * @VXGE_HAL_PCI_E_LINK_WIDTH_X4: 4 Lane. * @VXGE_HAL_PCI_E_LINK_WIDTH_X8: 8 Lane. * @VXGE_HAL_PCI_E_LINK_WIDTH_X12: 12 Lane. * @VXGE_HAL_PCI_E_LINK_WIDTH_X16: 16 Lane. * @VXGE_HAL_PCI_E_LINK_WIDTH_X32: 32 Lane. * @VXGE_HAL_PCI_E_LINK_WIDTH_UNKNOWN: Unknown * * PCI-E Link width enumeration. */ typedef enum vxge_hal_pci_e_link_width_e { VXGE_HAL_PCI_E_LINK_WIDTH_X1 = 1, VXGE_HAL_PCI_E_LINK_WIDTH_X2 = 2, VXGE_HAL_PCI_E_LINK_WIDTH_X4 = 4, VXGE_HAL_PCI_E_LINK_WIDTH_X8 = 8, VXGE_HAL_PCI_E_LINK_WIDTH_X12 = 12, VXGE_HAL_PCI_E_LINK_WIDTH_X16 = 16, VXGE_HAL_PCI_E_LINK_WIDTH_X32 = 32, VXGE_HAL_PCI_E_LINK_WIDTH_UNKNOWN = 0 } vxge_hal_pci_e_link_width_e; #define VXGE_HAL_DEVICE_STATS_SW_INFO_NOT_TRAFFIC_INTR(hldev) \ ((vxge_hal_device_t *)hldev)->not_traffic_intr_cnt++ #define VXGE_HAL_DEVICE_STATS_SW_INFO_TRAFFIC_INTR(hldev) \ ((vxge_hal_device_t *)hldev)->traffic_intr_cnt++ /* * struct vxge_hal_device_t - Hal device object * @magic: Magic Number * @device_id: PCI Device Id of the adapter * @revision: PCI Device major revision * @upper_layer_data: Private data set by LL driver * @signalling_rate: PCI-E signalling rate * @link_width: see vxge_hal_pci_e_link_width_e {} * @regh0: BAR0 mapped memory handle (Solaris), or simply PCI device @pdev * (Linux and the rest.) * @regh1: BAR1 mapped memory handle. Same comment as above. * @regh2: BAR2 mapped memory handle. Same comment as above. * @bar0: BAR0 virtual address. * @bar1: BAR1 virtual address. * @bar2: BAR2 virtual address. * @irqh: IRQ handle * @cfgh: Configuration space handle * @pdev: Physical device handle * @config: Confguration passed by the LL driver at initialization * @is_initialized: Flag to specify if device is initialized * @msix_enabled: Flag to indicate if msix is enabled * @terminating: Flag to specify if the device is terminating * @link_state: Link state * @data_rate: Data rate * @not_traffic_intr_cnt: Number of times the host was interrupted * without new completions. * "Non-traffic interrupt counter". * @traffic_intr_cnt: Number of traffic interrupts for the device. * @debug_module_mask: Debug module mask * @debug_level: Debug Level * * HAL device object. Represents Titan adapter */ typedef struct vxge_hal_device_t { u32 magic; #define VXGE_HAL_DEVICE_MAGIC 0x12345678 #define VXGE_HAL_DEVICE_DEAD 0xDEADDEAD u16 device_id; u16 revision; void *upper_layer_data; vxge_hal_pci_e_signalling_rate_e signalling_rate; vxge_hal_pci_e_link_width_e link_width; pci_reg_h regh0; pci_reg_h regh1; pci_reg_h regh2; u8 *bar0; u8 *bar1; u8 *bar2; pci_irq_h irqh; pci_cfg_h cfgh; pci_dev_h pdev; vxge_hal_device_config_t config; volatile u32 is_initialized; volatile u32 msix_enabled; volatile u32 terminating; volatile vxge_hal_device_link_state_e link_state; volatile vxge_hal_device_data_rate_e data_rate; volatile u32 not_traffic_intr_cnt; volatile u32 traffic_intr_cnt; u32 debug_module_mask; u32 debug_level; } vxge_hal_device_t; /* * struct vxge_hal_device_date_t - Date Format * @day: Day * @month: Month * @year: Year * @date: Date in string format * * Structure for returning date */ typedef struct vxge_hal_device_date_t { u32 day; u32 month; u32 year; char date[16]; } vxge_hal_device_date_t; /* * struct vxge_hal_device_version_t - Version Format * @major: Major Version * @minor: Minor Version * @build: Build Number * @version: Version in string format * * Structure for returning version */ typedef struct vxge_hal_device_version_t { u32 major; u32 minor; u32 build; char version[32]; } vxge_hal_device_version_t; /* * struct vxge_hal_device_pmd_info_t - PMD Information * @type: PMD Type * @vendor: Vender name * @part_num: PMD Part Number * @ser_num: PMD Serial Number * * Structure for returning PMD info */ typedef struct vxge_hal_device_pmd_info_t { u32 type; #define VXGE_HAL_DEVICE_PMD_TYPE_UNKNOWN 0 #define VXGE_HAL_DEVICE_PMD_TYPE_10G_SR 1 #define VXGE_HAL_DEVICE_PMD_TYPE_10G_LR 2 #define VXGE_HAL_DEVICE_PMD_TYPE_10G_LRM 3 #define VXGE_HAL_DEVICE_PMD_TYPE_10G_DIRECT 4 #define VXGE_HAL_DEVICE_PMD_TYPE_10G_CX4 5 #define VXGE_HAL_DEVICE_PMD_TYPE_10G_BASE_T 6 #define VXGE_HAL_DEVICE_PMD_TYPE_10G_OTHER 7 #define VXGE_HAL_DEVICE_PMD_TYPE_1G_SX 8 #define VXGE_HAL_DEVICE_PMD_TYPE_1G_LX 9 #define VXGE_HAL_DEVICE_PMD_TYPE_1G_CX 10 #define VXGE_HAL_DEVICE_PMD_TYPE_1G_DIRECT 11 #define VXGE_HAL_DEVICE_PMD_TYPE_1G_CX4 12 #define VXGE_HAL_DEVICE_PMD_TYPE_1G_BASE_T 13 #define VXGE_HAL_DEVICE_PMD_TYPE_1G_OTHER 14 u32 unused; char vendor[24]; char part_num[24]; char ser_num[24]; } vxge_hal_device_pmd_info_t; /* * struct vxge_hal_device_hw_info_t - Device information * @host_type: Host Type * @function_mode: PCI Function Mode * @func_id: Function Id * @vpath_mask: vpath bit mask * @fw_version: Firmware version * @fw_date: Firmware Date * @flash_version: Firmware version * @flash_date: Firmware Date * @serial_number: Card Serial Number * @part_number: Card Part Number * @product_description: Card Product Description * @unused: For Solaris alignment purpose * @ports: Number of ports present * @pmd_port0: Port 0 PMD Info * @pmd_port1: Port 1 PMD Info * @mac_addrs: Mac addresses for each vpath * @mac_addr_masks: Mac address masks for each vpath * * Returns the vpath mask that has the bits set for each vpath allocated * for the driver and the first mac addresse for each vpath */ typedef struct vxge_hal_device_hw_info_t { u32 host_type; #define VXGE_HAL_NO_MR_NO_SR_NORMAL_FUNCTION 0 #define VXGE_HAL_MR_NO_SR_VH0_BASE_FUNCTION 1 #define VXGE_HAL_NO_MR_SR_VH0_FUNCTION0 2 #define VXGE_HAL_NO_MR_SR_VH0_VIRTUAL_FUNCTION 3 #define VXGE_HAL_MR_SR_VH0_INVALID_CONFIG 4 #define VXGE_HAL_SR_VH_FUNCTION0 5 #define VXGE_HAL_SR_VH_VIRTUAL_FUNCTION 6 #define VXGE_HAL_VH_NORMAL_FUNCTION 7 u64 function_mode; u32 func_id; u64 vpath_mask; vxge_hal_device_version_t fw_version; vxge_hal_device_date_t fw_date; vxge_hal_device_version_t flash_version; vxge_hal_device_date_t flash_date; u8 serial_number[24]; u8 part_number[24]; u8 product_description[72]; u32 unused; u32 ports; vxge_hal_device_pmd_info_t pmd_port0; vxge_hal_device_pmd_info_t pmd_port1; macaddr_t mac_addrs[VXGE_HAL_MAX_VIRTUAL_PATHS]; macaddr_t mac_addr_masks[VXGE_HAL_MAX_VIRTUAL_PATHS]; } vxge_hal_device_hw_info_t; /* * vxge_hal_device_hw_info_get - Get the hw information * @pdev: PCI device object. * @regh0: BAR0 mapped memory handle (Solaris), or simply PCI device @pdev * (Linux and the rest.) * @bar0: Address of BAR0 in PCI config * @hw_info: Buffer to return vxge_hal_device_hw_info_t {} structure * * Returns the vpath mask that has the bits set for each vpath allocated * for the driver, FW version information and the first mac addresse for * each vpath */ vxge_hal_status_e vxge_hal_device_hw_info_get( pci_dev_h pdev, pci_reg_h regh0, u8 *bar0, vxge_hal_device_hw_info_t *hw_info); /* * vxge_hal_device_config_default_get - Initialize device config with defaults. * @device_config: Configuration structure to be initialized, * For the X3100 configuration "knobs" please * refer to vxge_hal_device_config_t and X3100 * User Guide. * * Initialize X3100 device config with default values. * * See also: vxge_hal_device_initialize(), vxge_hal_device_terminate(), * vxge_hal_status_e {} vxge_hal_device_attr_t {}. */ vxge_hal_status_e vxge_hal_device_config_default_get( vxge_hal_device_config_t *device_config); /* * vxge_hal_device_initialize - Initialize X3100 device. * @devh: Buffer to return HAL device handle. * @attr: pointer to vxge_hal_device_attr_t structure * @device_config: Configuration to be _applied_ to the device, * For the X3100 configuration "knobs" please refer to * vxge_hal_device_config_t and X3100 User Guide. * * Initialize X3100 device. Note that all the arguments of this public API * are 'IN', except @hldev. Upper-layer driver (ULD) cooperates with * OS to find new X3100 device, locate its PCI and memory spaces. * * When done, the ULD allocates sizeof(vxge_hal_device_t) bytes for HAL * to enable the latter to perform X3100 hardware initialization. * * Returns: VXGE_HAL_OK - success. * VXGE_HAL_ERR_DRIVER_NOT_INITIALIZED - Driver is not initialized. * VXGE_HAL_ERR_BAD_DEVICE_CONFIG - Device configuration params are not * valid. * VXGE_HAL_ERR_OUT_OF_MEMORY - Memory allocation failed. * VXGE_HAL_ERR_BAD_SUBSYSTEM_ID - Device subsystem id is invalid. * VXGE_HAL_ERR_INVALID_MAC_ADDRESS - Device mac address in not valid. * VXGE_HAL_INF_MEM_STROBE_CMD_EXECUTING - Failed to retrieve the mac * address within the time(timeout) or TTI/RTI initialization failed. * VXGE_HAL_ERR_SWAPPER_CTRL - Failed to configure swapper control. * * See also: vxge_hal_device_terminate(), vxge_hal_status_e {} * vxge_hal_device_attr_t {}. */ vxge_hal_status_e vxge_hal_device_initialize( vxge_hal_device_h *devh, vxge_hal_device_attr_t *attr, vxge_hal_device_config_t *device_config); /* * vxge_hal_device_check_id - Verify device ID. * @devh: HAL device handle. * * Verify device ID. * Returns: one of the vxge_hal_card_e {} enumerated types. * See also: vxge_hal_card_e {}. */ static inline /* LINTED */ vxge_hal_card_e vxge_hal_device_check_id( vxge_hal_device_h devh) { vxge_hal_device_t *hldev = (vxge_hal_device_t *) devh; switch (hldev->device_id) { case VXGE_PCI_DEVICE_ID_TITAN_1: if (hldev->revision == VXGE_PCI_REVISION_TITAN_1) return (VXGE_HAL_CARD_TITAN_1); else if (hldev->revision == VXGE_PCI_REVISION_TITAN_1A) return (VXGE_HAL_CARD_TITAN_1A); else break; case VXGE_PCI_DEVICE_ID_TITAN_2: if (hldev->revision == VXGE_PCI_REVISION_TITAN_2) return (VXGE_HAL_CARD_TITAN_2); else break; default: break; } return (VXGE_HAL_CARD_UNKNOWN); } /* * vxge_hal_device_revision_get - Get Device revision number. * @devh: HAL device handle. * * Returns: Device revision number */ static inline /* LINTED */ u32 vxge_hal_device_revision_get( vxge_hal_device_h devh) { return (((vxge_hal_device_t *) devh)->revision); } /* * vxge_hal_device_pciconfig_get - Read the content of given address * in pci config space. * @devh: Device handle. * @offset: Configuration address(offset)to read from * @length: Length of the data (1, 2 or 4 bytes) * @val: Pointer to a buffer to return the content of the address * * Read from the pci config space. * */ vxge_hal_status_e vxge_hal_device_pciconfig_get( vxge_hal_device_h devh, u32 offset, u32 length, void *val); /* * vxge_hal_device_bar0_get - Get BAR0 mapped address. * @devh: HAL device handle. * * Returns: BAR0 address of the specified device. */ static inline /* LINTED */ u8 *vxge_hal_device_bar0_get( vxge_hal_device_h devh) { return (((vxge_hal_device_t *) devh)->bar0); } /* * vxge_hal_device_bar1_get - Get BAR1 mapped address. * @devh: HAL device handle. * * Returns: BAR1 address of the specified device. */ static inline /* LINTED */ u8 *vxge_hal_device_bar1_get( vxge_hal_device_h devh) { return (((vxge_hal_device_t *) devh)->bar1); } /* * vxge_hal_device_bar2_get - Get BAR2 mapped address. * @devh: HAL device handle. * * Returns: BAR2 address of the specified device. */ static inline /* LINTED */ u8 *vxge_hal_device_bar2_get( vxge_hal_device_h devh) { return (((vxge_hal_device_t *) devh)->bar2); } /* * vxge_hal_device_bar0_set - Set BAR0 mapped address. * @devh: HAL device handle. * @bar0: BAR0 mapped address. * * Set BAR0 address in the HAL device object. */ static inline /* LINTED */ void vxge_hal_device_bar0_set( vxge_hal_device_h devh, u8 *bar0) { ((vxge_hal_device_t *) devh)->bar0 = bar0; } /* * vxge_hal_device_bar1_set - Set BAR1 mapped address. * @devh: HAL device handle. * @bar1: BAR1 mapped address. * * Set BAR1 address in the HAL Device Object. */ static inline /* LINTED */ void vxge_hal_device_bar1_set( vxge_hal_device_h devh, u8 *bar1) { ((vxge_hal_device_t *) devh)->bar1 = bar1; } /* * vxge_hal_device_bar2_set - Set BAR2 mapped address. * @devh: HAL device handle. * @bar2: BAR2 mapped address. * * Set BAR2 address in the HAL Device Object. */ static inline /* LINTED */ void vxge_hal_device_bar2_set( vxge_hal_device_h devh, u8 *bar2) { ((vxge_hal_device_t *) devh)->bar2 = bar2; } /* * vxge_hal_device_enable - Enable device. * @devh: HAL device handle. * * Enable the specified device: bring up the link/interface. * */ vxge_hal_status_e vxge_hal_device_enable( vxge_hal_device_h devh); /* * vxge_hal_device_disable - Disable X3100 adapter. * @devh: HAL device handle. * * Disable this device. To gracefully reset the adapter, the host should: * * - call vxge_hal_device_disable(); * * - call vxge_hal_device_intr_disable(); * * - do some work (error recovery, change mtu, reset, etc); * * - call vxge_hal_device_enable(); * * - call vxge_hal_device_intr_enable(). * * Note: Disabling the device does _not_ include disabling of interrupts. * After disabling the device stops receiving new frames but those frames * that were already in the pipe will keep coming for some few milliseconds. * * */ vxge_hal_status_e vxge_hal_device_disable( vxge_hal_device_h devh); /* * vxge_hal_device_pci_info_get - Get PCI bus informations such as width, * frequency, and mode from previously stored values. * @devh: HAL device handle. * @signalling_rate: pointer to a variable of enumerated type * vxge_hal_pci_e_signalling_rate_e {}. * @link_width: pointer to a variable of enumerated type * vxge_hal_pci_e_link_width_e {}. * * Get pci-e signalling rate and link width. * * Returns: one of the vxge_hal_status_e {} enumerated types. * VXGE_HAL_OK - for success. * VXGE_HAL_ERR_INVALID_DEVICE - for invalid device handle. */ static inline /* LINTED */ vxge_hal_status_e vxge_hal_device_pci_info_get( vxge_hal_device_h devh, vxge_hal_pci_e_signalling_rate_e *signalling_rate, vxge_hal_pci_e_link_width_e *link_width) { vxge_hal_device_t *hldev = (vxge_hal_device_t *) devh; if (!hldev || !hldev->is_initialized || (hldev->magic != VXGE_HAL_DEVICE_MAGIC)) { return (VXGE_HAL_ERR_INVALID_DEVICE); } *signalling_rate = hldev->signalling_rate; *link_width = hldev->link_width; return (VXGE_HAL_OK); } /* * vxge_hal_device_link_state_test - Test the link state. * @devh: HAL device handle. * * Test link state. * Returns: link state. */ vxge_hal_device_link_state_e vxge_hal_device_link_state_test( vxge_hal_device_h devh); /* * vxge_hal_device_link_state_get - Get link state. * @devh: HAL device handle. * * Get link state. * Returns: link state. */ static inline /* LINTED */ vxge_hal_device_link_state_e vxge_hal_device_link_state_get( vxge_hal_device_h devh) { return (((vxge_hal_device_t *) devh)->link_state); } /* * vxge_hal_device_link_state_poll - Poll for the link state. * @devh: HAL device handle. * * Get link state. * Returns: link state. */ vxge_hal_device_link_state_e vxge_hal_device_link_state_poll( vxge_hal_device_h devh); /* * vxge_hal_device_data_rate_get - Get data rate. * @devh: HAL device handle. * * Get data rate. * Returns: data rate(1G or 10G). */ static inline /* LINTED */ vxge_hal_device_data_rate_e vxge_hal_device_data_rate_get( vxge_hal_device_h devh) { return (((vxge_hal_device_t *) devh)->data_rate); } /* * vxge_hal_device_data_rate_poll - Poll for the data rate. * @devh: HAL device handle. * * Get data rate. * Returns: data rate. */ vxge_hal_device_data_rate_e vxge_hal_device_data_rate_poll( vxge_hal_device_h devh); /* * vxge_hal_device_lag_mode_get - Get Current LAG Mode * @devh: HAL device handle. * * Get Current LAG Mode */ vxge_hal_device_lag_mode_e vxge_hal_device_lag_mode_get( vxge_hal_device_h devh); /* * vxge_hal_device_reset - Reset device. * @devh: HAL device handle. * * Soft-reset the device, reset the device stats except reset_cnt. * * After reset is done, will try to re-initialize HW. * * Returns: VXGE_HAL_PENDING - successfully sent reset to device. * VXGE_HAL_ERR_DEVICE_NOT_INITIALIZED - Device is not initialized. * VXGE_HAL_ERR_RESET_FAILED - Reset failed. * */ vxge_hal_status_e vxge_hal_device_reset( vxge_hal_device_h devh); /* * vxge_hal_device_reset_poll - Poll the device for reset complete. * @devh: HAL device handle. * * Poll the device for reset complete * * Returns: VXGE_HAL_OK - success. * VXGE_HAL_ERR_DEVICE_NOT_INITIALIZED - Device is not initialized. * VXGE_HAL_ERR_RESET_FAILED - Reset failed. * * See also: vxge_hal_status_e {}. */ vxge_hal_status_e vxge_hal_device_reset_poll(vxge_hal_device_h devh); /* * vxge_hal_device_mrpcim_reset_poll - Poll the device for mrpcim reset * complete. * @devh: HAL device handle. * * Poll the device for mrpcim reset complete * * Returns: VXGE_HAL_OK - success. * VXGE_HAL_ERR_DEVICE_NOT_INITIALIZED - Device is not initialized. * VXGE_HAL_ERR_RESET_FAILED - Reset failed. * VXGE_HAL_ERR_MANAGER_NOT_FOUND - MRPCIM/SRPCIM manager not found * VXGE_HAL_ERR_TIME_OUT - Device Reset timed out * * See also: vxge_hal_status_e {}. */ vxge_hal_status_e vxge_hal_device_mrpcim_reset_poll(vxge_hal_device_h devh); /* * vxge_hal_device_terminating - Mark the device as 'terminating'. * @devh: HAL device handle. * * Mark the device as 'terminating', going to terminate. Can be used * to serialize termination with other running processes/contexts. * * See also: vxge_hal_device_terminate(). */ static inline void /* LINTED */ vxge_hal_device_terminating(vxge_hal_device_h devh) { ((vxge_hal_device_t *) devh)->terminating = 1; } /* * vxge_hal_device_terminate - Terminate X3100 device. * @devh: HAL device handle. * * Terminate HAL device. * * See also: vxge_hal_device_initialize(). */ void vxge_hal_device_terminate( vxge_hal_device_h devh); /* * vxge_hal_device_private_set - Set ULD context. * @devh: HAL device handle. * @data: pointer to ULD context * * Use HAL device to set upper-layer driver (ULD) context. * * See also: vxge_hal_device_private_get() */ static inline /* LINTED */ void vxge_hal_device_private_set( vxge_hal_device_h devh, void *data) { ((vxge_hal_device_t *) devh)->upper_layer_data = data; } /* * vxge_hal_device_private_get - Get ULD context. * @devh: HAL device handle. * * Use HAL device to set upper-layer driver (ULD) context. * * See also: vxge_hal_device_private_get() */ static inline /* LINTED */ void *vxge_hal_device_private_get( vxge_hal_device_h devh) { return (((vxge_hal_device_t *) devh)->upper_layer_data); } /* * vxge_hal_device_status - Check whether X3100 hardware is ready for * operation. * @devh: HAL device handle. * @hw_status: X3100 status register. Returned by HAL. * * Check whether X3100 hardware is ready for operation. * The checking includes TDMA, RDMA, PFC, PIC, MC_DRAM, and the rest * hardware functional blocks. * * Returns: VXGE_HAL_OK if the device is ready for operation. Otherwise * returns VXGE_HAL_FAIL. Also, fills in adapter status (in @hw_status). * * See also: vxge_hal_status_e {}. * Usage: See ex_open {}. */ vxge_hal_status_e vxge_hal_device_status(vxge_hal_device_h devh, u64 *hw_status); /* * vxge_hal_device_is_slot_freeze * @devh: the device * * Returns non-zero if the slot is freezed. * The determination is made based on the adapter_status * register which will never give all FFs, unless PCI read * cannot go through. */ int vxge_hal_device_is_slot_freeze(vxge_hal_device_h devh); /* * vxge_hal_device_is_traffic_interrupt * @reason: The reason returned by the vxge)hal_device_begin_irq * @vp_id: Id of vpath for which to check the interrupt * * Returns non-zero if traffic interrupt raised, 0 otherwise */ static inline u64 /* LINTED */ vxge_hal_device_is_traffic_interrupt(u64 reason, u32 vp_id) { return (reason & mBIT(vp_id + 3)); } /* * vxge_hal_device_intr_enable - Enable X3100 interrupts. * @devh: HAL device handle. * @op: One of the vxge_hal_device_intr_e enumerated values specifying * the type(s) of interrupts to enable. * * Enable X3100 interrupts. The function is to be executed the last in * X3100 initialization sequence. * * See also: vxge_hal_device_intr_disable() */ void vxge_hal_device_intr_enable( vxge_hal_device_h devh); /* * vxge_hal_device_intr_disable - Disable X3100 interrupts. * @devh: HAL device handle. * @op: One of the vxge_hal_device_intr_e enumerated values specifying * the type(s) of interrupts to disable. * * Disable X3100 interrupts. * * See also: vxge_hal_device_intr_enable() */ void vxge_hal_device_intr_disable( vxge_hal_device_h devh); /* * vxge_hal_device_mask_all - Mask all device interrupts. * @devh: HAL device handle. * * Mask all device interrupts. * * See also: vxge_hal_device_unmask_all() */ void vxge_hal_device_mask_all( vxge_hal_device_h devh); /* * vxge_hal_device_unmask_all - Unmask all device interrupts. * @devh: HAL device handle. * * Unmask all device interrupts. * * See also: vxge_hal_device_mask_all() */ void vxge_hal_device_unmask_all( vxge_hal_device_h devh); /* * vxge_hal_device_begin_irq - Begin IRQ processing. * @devh: HAL device handle. * @skip_alarms: Do not clear the alarms * @reason: "Reason" for the interrupt, the value of X3100's * general_int_status register. * * The function performs two actions, It first checks whether (shared IRQ) the * interrupt was raised by the device. Next, it masks the device interrupts. * * Note: * vxge_hal_device_begin_irq() does not flush MMIO writes through the * bridge. Therefore, two back-to-back interrupts are potentially possible. * It is the responsibility of the ULD to make sure that only one * vxge_hal_device_continue_irq() runs at a time. * * Returns: 0, if the interrupt is not "ours" (note that in this case the * device remain enabled). * Otherwise, vxge_hal_device_begin_irq() returns 64bit general adapter * status. * See also: vxge_hal_device_handle_irq() */ vxge_hal_status_e vxge_hal_device_begin_irq( vxge_hal_device_h devh, u32 skip_alarms, u64 *reason); /* * vxge_hal_device_continue_irq - Continue handling IRQ: process all * completed descriptors. * @devh: HAL device handle. * * Process completed descriptors and unmask the device interrupts. * * The vxge_hal_device_continue_irq() walks all open virtual paths * and calls upper-layer driver (ULD) via supplied completion * callback. * * Note that the vxge_hal_device_continue_irq is part of the _fast_ path. * To optimize the processing, the function does _not_ check for * errors and alarms. * * Returns: VXGE_HAL_OK. * * See also: vxge_hal_device_handle_irq(), * vxge_hal_ring_rxd_next_completed(), * vxge_hal_fifo_txdl_next_completed(), vxge_hal_ring_callback_f {}, * vxge_hal_fifo_callback_f {}. */ vxge_hal_status_e vxge_hal_device_continue_irq( vxge_hal_device_h devh); /* * vxge_hal_device_handle_irq - Handle device IRQ. * @devh: HAL device handle. * @skip_alarms: Do not clear the alarms * * Perform the complete handling of the line interrupt. The function * performs two calls. * First it uses vxge_hal_device_begin_irq() to check the reason for * the interrupt and mask the device interrupts. * Second, it calls vxge_hal_device_continue_irq() to process all * completed descriptors and re-enable the interrupts. * * Returns: VXGE_HAL_OK - success; * VXGE_HAL_ERR_WRONG_IRQ - (shared) IRQ produced by other device. * * See also: vxge_hal_device_begin_irq(), vxge_hal_device_continue_irq(). */ vxge_hal_status_e vxge_hal_device_handle_irq( vxge_hal_device_h devh, u32 skip_alarms); /* * vxge_hal_device_mask_tx - Mask Tx interrupts. * @devh: HAL device. * * Mask Tx device interrupts. * * See also: vxge_hal_device_unmask_tx(), vxge_hal_device_mask_rx(), * vxge_hal_device_clear_tx(). */ void vxge_hal_device_mask_tx( vxge_hal_device_h devh); /* * vxge_hal_device_clear_tx - Acknowledge (that is, clear) the * condition that has caused the TX interrupt. * @devh: HAL device. * * Acknowledge (that is, clear) the condition that has caused * the Tx interrupt. * See also: vxge_hal_device_begin_irq(), vxge_hal_device_continue_irq(), * vxge_hal_device_clear_rx(), vxge_hal_device_mask_tx(). */ void vxge_hal_device_clear_tx( vxge_hal_device_h devh); /* * vxge_hal_device_unmask_tx - Unmask Tx interrupts. * @devh: HAL device. * * Unmask Tx device interrupts. * * See also: vxge_hal_device_mask_tx(), vxge_hal_device_clear_tx(). */ void vxge_hal_device_unmask_tx( vxge_hal_device_h devh); /* * vxge_hal_device_mask_rx - Mask Rx interrupts. * @devh: HAL device. * * Mask Rx device interrupts. * * See also: vxge_hal_device_unmask_rx(), vxge_hal_device_mask_tx(), * vxge_hal_device_clear_rx(). */ void vxge_hal_device_mask_rx( vxge_hal_device_h devh); /* * vxge_hal_device_clear_rx - Acknowledge (that is, clear) the * condition that has caused the RX interrupt. * @devh: HAL device. * * Acknowledge (that is, clear) the condition that has caused * the Rx interrupt. * See also: vxge_hal_device_begin_irq(), vxge_hal_device_continue_irq(), * vxge_hal_device_clear_tx(), vxge_hal_device_mask_rx(). */ void vxge_hal_device_clear_rx( vxge_hal_device_h devh); /* * vxge_hal_device_unmask_rx - Unmask Rx interrupts. * @devh: HAL device. * * Unmask Rx device interrupts. * * See also: vxge_hal_device_mask_rx(), vxge_hal_device_clear_rx(). */ void vxge_hal_device_unmask_rx( vxge_hal_device_h devh); /* * vxge_hal_device_mask_tx_rx - Mask Tx and Rx interrupts. * @devh: HAL device. * * Mask Tx and Rx device interrupts. * * See also: vxge_hal_device_unmask_tx_rx(), vxge_hal_device_clear_tx_rx(). */ void vxge_hal_device_mask_tx_rx( vxge_hal_device_h devh); /* * vxge_hal_device_clear_tx_rx - Acknowledge (that is, clear) the * condition that has caused the Tx and RX interrupt. * @devh: HAL device. * * Acknowledge (that is, clear) the condition that has caused * the Tx and Rx interrupt. * See also: vxge_hal_device_begin_irq(), vxge_hal_device_continue_irq(), * vxge_hal_device_mask_tx_rx(), vxge_hal_device_unmask_tx_rx(). */ void vxge_hal_device_clear_tx_rx( vxge_hal_device_h devh); /* * vxge_hal_device_unmask_tx_rx - Unmask Tx and Rx interrupts. * @devh: HAL device. * * Unmask Tx and Rx device interrupts. * * See also: vxge_hal_device_mask_tx_rx(), vxge_hal_device_clear_tx_rx(). */ void vxge_hal_device_unmask_tx_rx( vxge_hal_device_h devh); /* * vxge_hal_device_msix_mode - Is MSIX enabled? * @devh: HAL device handle. * * Returns 0 if MSIX is enabled for the specified device, * non-zero otherwise. */ static inline int /* LINTED */ vxge_hal_device_msix_mode(vxge_hal_device_h devh) { return (((vxge_hal_device_t *) devh)->msix_enabled); } #if defined(VXGE_TRACE_INTO_CIRCULAR_ARR) /* * vxge_hal_device_trace_write - Write the trace from the given buffer into * circular trace buffer * @devh: HAL device handle. * @trace_buf: Buffer containing the trace. * @trace_len: Length of the trace in the buffer * * Writes the trace from the given buffer into the circular trace buffer * */ void vxge_hal_device_trace_write(vxge_hal_device_h devh, u8 *trace_buf, u32 trace_len); /* * vxge_hal_device_trace_dump - Dump the trace buffer. * @devh: HAL device handle. * * Dump the trace buffer contents. */ void vxge_hal_device_trace_dump(vxge_hal_device_h devh); /* * vxge_hal_device_trace_read - Read trace buffer contents. * @devh: HAL device handle. * @buffer: Buffer to store the trace buffer contents. * @buf_size: Size of the buffer. * @read_length: Size of the valid data in the buffer. * * Read HAL trace buffer contents starting from the offset * upto the size of the buffer or till EOF is reached. * * Returns: VXGE_HAL_OK - success. * VXGE_HAL_EOF_TRACE_BUF - No more data in the trace buffer. * */ vxge_hal_status_e vxge_hal_device_trace_read(vxge_hal_device_h devh, char *buffer, unsigned buf_size, unsigned *read_length); #endif /* * vxge_hal_device_debug_set - Set the debug module, level and timestamp * @devh: Hal device object * @level: Debug level as defined in enum vxge_debug_level_e * @mask: An or value of component masks as defined in vxge_debug.h * * This routine is used to dynamically change the debug output */ void vxge_hal_device_debug_set( vxge_hal_device_h devh, vxge_debug_level_e level, u32 mask); /* * vxge_hal_device_debug_level_get - Get the debug level * @devh: Hal device object * * This routine returns the current debug level set */ static inline u32 /* LINTED */ vxge_hal_device_debug_level_get(vxge_hal_device_h devh) { return (((vxge_hal_device_t *) devh)->debug_level); } /* * vxge_hal_device_debug_mask_get - Get the debug mask * @devh: Hal device object * * This routine returns the current debug mask set */ static inline u32 /* LINTED */ vxge_hal_device_debug_mask_get(vxge_hal_device_h devh) { return (((vxge_hal_device_t *) devh)->debug_module_mask); } /* * vxge_hal_device_flick_link_led - Flick (blink) link LED. * @devh: HAL device handle. * @port : Port number 0, or 1 * @on_off: TRUE if flickering to be on, FALSE to be off * * Flicker the link LED. */ vxge_hal_status_e vxge_hal_device_flick_link_led( vxge_hal_device_h devh, u32 port, u32 on_off); /* * vxge_hal_device_getpause_data -Pause frame frame generation and reception. * @devh: HAL device handle. * @port : Port number 0, 1, or 2 * @tx : A field to return the pause generation capability of the NIC. * @rx : A field to return the pause reception capability of the NIC. * * Returns the Pause frame generation and reception capability of the NIC. * Return value: * status */ vxge_hal_status_e vxge_hal_device_getpause_data( vxge_hal_device_h devh, u32 port, u32 *tx, u32 *rx); /* * Privileged operations */ /* * enum vxge_hal_pcie_function_mode_e - PCIE Function modes * @VXGE_HAL_PCIE_FUNC_MODE_SF1_VP17: Single Function * - 1 function with 17 VPATHs * @VXGE_HAL_PCIE_FUNC_MODE_MF8_VP2: Multi Function * - 8 functions with 2 VPATHs per function * @VXGE_HAL_PCIE_FUNC_MODE_SR17_VP1: SRIOV * - 17 VFs with 1 VPATH per VF * @VXGE_HAL_PCIE_FUNC_MODE_MR17_VP1: MRIOV * - 17 Virtual Hierarchies, 1 Path/Function/Hierarchy * @VXGE_HAL_PCIE_FUNC_MODE_MR8_VP2: MRIOV * - 8 Virtual Hierarchies, 2 Path/Function/Hierarchy * @VXGE_HAL_PCIE_FUNC_MODE_MF17_VP1: Multi Function * - 17 functions, 1 Path/Function (PCIe ARI) * @VXGE_HAL_PCIE_FUNC_MODE_SR8_VP2: SRIOV * - 1 PF, 7 VF, 2 Paths/Function * @VXGE_HAL_PCIE_FUNC_MODE_SR4_VP4: SRIOV * - 1 PF, 3 VF, 4 Paths/Function * @VXGE_HAL_PCIE_FUNC_MODE_MF2_VP8: Multi Function * - 2 functions, 8 Paths/Function (funcs 2-7 have no resources) */ typedef enum vxge_hal_pcie_function_mode_e { VXGE_HAL_PCIE_FUNC_MODE_SF1_VP17, VXGE_HAL_PCIE_FUNC_MODE_MF8_VP2, VXGE_HAL_PCIE_FUNC_MODE_SR17_VP1, VXGE_HAL_PCIE_FUNC_MODE_MR17_VP1, VXGE_HAL_PCIE_FUNC_MODE_MR8_VP2, VXGE_HAL_PCIE_FUNC_MODE_MF17_VP1, VXGE_HAL_PCIE_FUNC_MODE_SR8_VP2, VXGE_HAL_PCIE_FUNC_MODE_SR4_VP4, VXGE_HAL_PCIE_FUNC_MODE_MF2_VP8, VXGE_HAL_PCIE_FUNC_MODE_MF4_VP4, VXGE_HAL_PCIE_FUNC_MODE_MR4_VP4, VXGE_HAL_PCIE_FUNC_MODE_MF8P_VP2 } vxge_hal_pcie_function_mode_e; /* Behavior on failure */ typedef enum vxge_hal_xmac_nwif_behavior_on_failure { VXGE_HAL_XMAC_NWIF_OnFailure_NoMove, VXGE_HAL_XMAC_NWIF_OnFailure_OtherPort, VXGE_HAL_XMAC_NWIF_OnFailure_OtherPortBackOnRestore } vxge_hal_xmac_nwif_behavior_on_failure; /* * Network Port configuration cmds */ typedef enum vxge_hal_nwif_cmds { VXGE_HAL_XMAC_NWIF_Cmd_Version = 0x0, VXGE_HAL_XMAC_NWIF_Cmd_SetMode = 0x1, VXGE_HAL_XMAC_NWIF_Cmd_CfgSnglPort = 0x4, VXGE_HAL_XMAC_NWIF_Cmd_Avail = 0x6, VXGE_HAL_XMAC_NWIF_Cmd_CfgSetActPassPreferredPort = 0x7, VXGE_HAL_XMAC_NWIF_Cmd_CfgSetBehaviourOnFailure = 0x8, VXGE_HAL_XMAC_NWIF_Cmd_CfgDualPort_L2SwitchEnable = 0x9, VXGE_HAL_XMAC_NWIF_Cmd_CfgDualPort_VPathVector = 0xa, VXGE_HAL_XMAC_NWIF_Cmd_Get_Active_Config = 0xb } vxge_hal_nwif_cmds; /* Network port get active config options */ typedef enum vxge_hal_xmac_nwif_actconfig { VXGE_HAL_XMAC_NWIF_ActConfig_Avail = 0, VXGE_HAL_XMAC_NWIF_ActConfig_NWPortMode = 1, VXGE_HAL_XMAC_NWIF_ActConfig_PreferredPort = 2, VXGE_HAL_XMAC_NWIF_ActConfig_BehaviourOnFail = 3, VXGE_HAL_XMAC_NWIF_ActConfig_ActivePort = 4, VXGE_HAL_XMAC_NWIF_ActConfig_L2SwitchEnabled = 5, VXGE_HAL_XMAC_NWIF_ActConfig_DualPortPath = 6 } vxge_hal_xmac_nwif_actconfig; /* Dual port modes */ typedef enum vxge_hal_xmac_nwif_dp_mode { VXGE_HAL_DP_NP_MODE_DEFAULT, VXGE_HAL_DP_NP_MODE_LINK_AGGR, VXGE_HAL_DP_NP_MODE_ACTIVE_PASSIVE, VXGE_HAL_DP_NP_MODE_SINGLE_PORT, VXGE_HAL_DP_NP_MODE_DUAL_PORT, VXGE_HAL_DP_NP_MODE_DISABLE_PORT_MGMT } vxge_hal_xmac_nwif_dp_mode; /* L2 switch status */ typedef enum vxge_hal_xmac_nwif_l2_switch_status { VXGE_HAL_XMAC_NWIF_L2_SWITCH_DISABLE, VXGE_HAL_XMAC_NWIF_L2_SWITCH_ENABLE } vxge_hal_xmac_nwif_l2_switch_status; /* * vxge_hal_srpcim_alarm_process - Process srpcim Alarms. * @devh: Device Handle. * @skip_alarms: Flasg to indicate not to clear alarms * * Process srpcim alarms. * */ vxge_hal_status_e vxge_hal_srpcim_alarm_process(vxge_hal_device_h devh, u32 skip_alarms); /* * vxge_hal_srpcim_intr_enable - Enable srpcim interrupts. * @devh: Device Handle. * * Enable srpcim interrupts. * * See also: vxge_hal_srpcim_intr_disable() */ vxge_hal_status_e vxge_hal_srpcim_intr_enable(vxge_hal_device_h devh); /* * vxge_hal_srpcim_intr_disable - Disable srpcim interrupts. * @devh: Device Handle. * * Disable srpcim interrupts. * * See also: vxge_hal_srpcim_intr_enable() */ vxge_hal_status_e vxge_hal_srpcim_intr_disable(vxge_hal_device_h devh); /* * vxge_hal_srpcim_msix_set - Associate MSIX vector with srpcim alarm * @devh: Device Handle. * @alarm_msix_id: MSIX vector for alarm. * * This API will associate a given MSIX vector numbers with srpcim alarm */ vxge_hal_status_e vxge_hal_srpcim_msix_set(vxge_hal_device_h devh, int alarm_msix_id); /* * vxge_hal_srpcim_msix_mask - Mask MSIX Vector. * @devh: Device Handle. * * The function masks the srpcim msix interrupt * */ void vxge_hal_srpcim_msix_mask(vxge_hal_device_h devh); /* * vxge_hal_srpcim_msix_clear - Clear MSIX Vector. * @devh: Device Handle. * * The function clears the srpcim msix interrupt * */ void vxge_hal_srpcim_msix_clear(vxge_hal_device_h devh); /* * vxge_hal_srpcim_msix_unmask - Unmask MSIX Vector. * @devh: Device Handle. * * The function unmasks the srpcim msix interrupt * */ void vxge_hal_srpcim_msix_unmask(vxge_hal_device_h devh); vxge_hal_status_e vxge_hal_func_mode_count(vxge_hal_device_h devh, u32 func_mode, u32 *num_funcs); vxge_hal_status_e vxge_hal_send_message(vxge_hal_device_h devh, u64 vp_id, u8 msg_type, u8 msg_dst, u32 msg_data, u64 *msg_sent_to_vpaths); /* * vxge_hal_func_mode_set - Set PCI-E function mode * @devh: Device Handle. * @func_mode: PCI-E func mode. Please see vxge_hal_pcie_function_mode_e{} * * Set PCI-E function mode. * */ vxge_hal_status_e vxge_hal_func_mode_get(vxge_hal_device_h devh, u32 *func_mode); /* * vxge_hal_func_mode_set - Set PCI-E function mode * @devh: Device Handle. * @func_mode: PCI-E func mode. Please see vxge_hal_pcie_function_mode_e{} * * Set PCI-E function mode. * */ vxge_hal_status_e vxge_hal_mrpcim_pcie_func_mode_set(vxge_hal_device_h devh, vxge_hal_pcie_function_mode_e func_mode); /* * vxge_hal_get_active_config - Get active configuration * @devh: Device Handle. * */ vxge_hal_status_e vxge_hal_get_active_config(vxge_hal_device_h devh, vxge_hal_xmac_nwif_actconfig req_config, u64 *cur_config); /* * vxge_hw_set_port_mode - Set dual port mode * override the default dual port mode * @devh: Device Handle. * */ vxge_hal_status_e vxge_hal_set_port_mode(vxge_hal_device_h devh, vxge_hal_xmac_nwif_dp_mode port_mode); /* * vxge_hal_set_behavior_on_failure - Set port behaviour * change port behavior on failure * @devh: Device Handle. */ vxge_hal_status_e vxge_hal_set_behavior_on_failure(vxge_hal_device_h devh, vxge_hal_xmac_nwif_behavior_on_failure behave_on_failure); /* * vxge_hal_set_l2switch_mode - Set port behaviour * set l2switch mode * @devh: Device Handle. */ vxge_hal_status_e vxge_hal_set_l2switch_mode(vxge_hal_device_h devh, enum vxge_hal_xmac_nwif_l2_switch_status l2_switch); /* * vxge_hal_set_fw_api - Setup FW api * @devh: Device Handle. * */ vxge_hal_status_e vxge_hal_set_fw_api(vxge_hal_device_h devh, u64 vp_id, u32 action, u32 offset, u64 data0, u64 data1); vxge_hal_status_e vxge_hal_config_vpath_map(vxge_hal_device_h devh, u64 port_map); vxge_hal_status_e vxge_hal_get_vpath_mask(vxge_hal_device_h devh, u32 vf_id, u32 *no_of_vpath, u64 *vpath_mask); vxge_hal_status_e vxge_hal_get_vpath_list(vxge_hal_device_h devh, u32 vf_id, u64 *vpath_list, u32 *vpath_count); vxge_hal_status_e vxge_hal_rx_bw_priority_set(vxge_hal_device_h devh, u64 vp_id); vxge_hal_status_e vxge_hal_tx_bw_priority_set(vxge_hal_device_h devh, u64 vp_id); vxge_hal_status_e vxge_hal_bw_priority_get(vxge_hal_device_h devh, u64 vp_id, u32 *bandwidth, u32 *priority); vxge_hal_status_e vxge_hal_vf_rx_bw_get(vxge_hal_device_h devh, u64 func_id, u32 *bandwidth, u32 *priority); /* * vxge_hal_mrpcim_serial_number_get - Returns the serial number * @devh: Device Handle. * * Return the serial number */ const u8 * vxge_hal_mrpcim_serial_number_get(vxge_hal_device_h devh); /* * vxge_hal_mrpcim_fw_upgrade - Upgrade firmware * @pdev: PCI device object. * @regh0: BAR0 mapped memory handle (Solaris), or simply PCI device @pdev * (Linux and the rest.) * @bar0: Address of BAR0 in PCI config * @buffer: Buffer containing F/W image * @length: F/W image length * * Upgrade firmware */ vxge_hal_status_e vxge_hal_mrpcim_fw_upgrade( pci_dev_h pdev, pci_reg_h regh0, u8 *bar0, u8 *buffer, u32 length); /* * vxge_hal_mrpcim_vpath_map_get - Returns the assigned vpaths map * @pdev: PCI device object. * @regh0: BAR0 mapped memory handle (Solaris), or simply PCI device @pdev * (Linux and the rest.) * @bar0: Address of BAR0 in PCI config * @func: Function Number * * Returns the assigned vpaths map */ u64 vxge_hal_mrpcim_vpath_map_get( pci_dev_h pdev, pci_reg_h regh0, u8 *bar0, u32 func); /* * vxge_hal_mrpcim_vpath_qos_set - Set the priority, Guaranteed and maximum * bandwidth for a vpath. * @devh: HAL device handle. * @vp_id: Vpath Id. * @priority: Priority * @min_bandwidth: Minimum Bandwidth * @max_bandwidth: Maximum Bandwidth * * Set the Priority, Guaranteed and maximum bandwidth for a given vpath * */ vxge_hal_status_e vxge_hal_mrpcim_vpath_qos_set( vxge_hal_device_h devh, u32 vp_id, u32 priority, u32 min_bandwidth, u32 max_bandwidth); /* * vxge_hal_mrpcim_vpath_qos_get - Get the priority, Guaranteed and maximum * bandwidth for a vpath. * @devh: HAL device handle. * @vp_id: Vpath Id. * @priority: Buffer to return Priority * @min_bandwidth: Buffer to return Minimum Bandwidth * @max_bandwidth: Buffer to return Maximum Bandwidth * * Get the Priority, Guaranteed and maximum bandwidth for a given vpath * */ vxge_hal_status_e vxge_hal_mrpcim_vpath_qos_get( vxge_hal_device_h devh, u32 vp_id, u32 *priority, u32 *min_bandwidth, u32 *max_bandwidth); /* * vxge_hal_mrpcim_mac_addr_add - Add the mac address entry * into MAC address table. * @devh: Device handle. * @offset: Index into the DA table to add the mac address. * @macaddr: MAC address to be added for this vpath into the list * @macaddr_mask: MAC address mask for macaddr * @vpath_vector: Bit mask specifying the vpaths to which the mac address * applies * @duplicate_mode: Duplicate MAC address add mode. Please see * vxge_hal_vpath_mac_addr_add_mode_e {} * * Adds the given mac address, mac address mask and vpath vector into the list * * see also: vxge_hal_vpath_mac_addr_delete, vxge_hal_vpath_mac_addr_get and * vxge_hal_vpath_mac_addr_get_next * */ vxge_hal_status_e vxge_hal_mrpcim_mac_addr_add( vxge_hal_device_h devh, u32 offset, macaddr_t macaddr, macaddr_t macaddr_mask, u64 vpath_vector, u32 duplicate_mode); /* * vxge_hal_mrpcim_mac_addr_get - Read the mac address entry * into MAC address table. * @devh: Device handle. * @offset: Index into the DA table to execute the command on. * @macaddr: Buffer to return MAC address to be added for this vpath * @macaddr_mask: Buffer to return MAC address mask for macaddr * @vpath_vector: Buffer to return Bit mask specifying the vpaths to which * the mac address applies * * Reads the mac address,mac address mask and vpath vector from the given offset * * see also: vxge_hal_mrpcim_mac_addr_add * */ vxge_hal_status_e vxge_hal_mrpcim_mac_addr_get( vxge_hal_device_h devh, u32 offset, macaddr_t macaddr, macaddr_t macaddr_mask, u64 *vpath_vector); /* * vxge_hal_mrpcim_reset - Reset the entire device. * @devh: HAL device handle. * * Soft-reset the device, reset the device stats except reset_cnt. * * * Returns: VXGE_HAL_OK - success. * VXGE_HAL_ERR_DEVICE_NOT_INITIALIZED - Device is not initialized. * VXGE_HAL_ERR_RESET_FAILED - Reset failed. * * See also: vxge_hal_status_e {}. */ vxge_hal_status_e vxge_hal_mrpcim_reset(vxge_hal_device_h devh); /* * vxge_hal_mrpcim_reset_poll - Poll the device for reset complete. * @devh: HAL device handle. * * Soft-reset the device, reset the device stats except reset_cnt. * * After reset is done, will try to re-initialize HW. * * Returns: VXGE_HAL_OK - success. * VXGE_HAL_ERR_DEVICE_NOT_INITIALIZED - Device is not initialized. * VXGE_HAL_ERR_RESET_FAILED - Reset failed. * * See also: vxge_hal_status_e {}. */ vxge_hal_status_e vxge_hal_mrpcim_reset_poll(vxge_hal_device_h devh); /* * vxge_hal_mrpcim_strip_repl_vlan_tag_enable - Enable strip Repl vlan tag. * @devh: Device handle. * * Enable X3100 strip Repl vlan tag. * Returns: VXGE_HAL_OK on success. * */ vxge_hal_status_e vxge_hal_mrpcim_strip_repl_vlan_tag_enable( vxge_hal_device_h devh); /* * vxge_hal_mrpcim_strip_repl_vlan_tag_disable - Disable strip Repl vlan tag. * @devh: Device handle. * * Disable X3100 strip Repl vlan tag. * Returns: VXGE_HAL_OK on success. * */ vxge_hal_status_e vxge_hal_mrpcim_strip_repl_vlan_tag_disable( vxge_hal_device_h devh); /* * vxge_hal_mrpcim_intr_enable - Enable the interrupts on mrpcim. * @devh: HAL device handle. * * Enable mrpcim interrupts * * See also: vxge_hal_mrpcim_intr_disable(). */ vxge_hal_status_e vxge_hal_mrpcim_intr_enable(vxge_hal_device_h devh); /* * vxge_hal_mrpcim_intr_disable - Disable the interrupts on mrpcim. * @devh: HAL device handle. * * Disable mrpcim interrupts * * See also: vxge_hal_mrpcim_intr_enable(). */ vxge_hal_status_e vxge_hal_mrpcim_intr_disable(vxge_hal_device_h devh); /* * vxge_hal_mrpcim_lag_config_get - Get the LAG config. * @devh: Device handle. * @lconfig: LAG Configuration * * Returns the current LAG configuration. * Returns: VXGE_HAL_OK on success. * */ vxge_hal_status_e vxge_hal_mrpcim_lag_config_get( vxge_hal_device_h devh, vxge_hal_lag_config_t *lconfig); /* * vxge_hal_mrpcim_lag_config_set - Set the LAG config. * @devh: Device handle. * @lconfig: LAG Configuration * * Sets the LAG configuration. * Returns: VXGE_HAL_OK on success. * */ vxge_hal_status_e vxge_hal_mrpcim_lag_config_set( vxge_hal_device_h devh, vxge_hal_lag_config_t *lconfig); /* * vxge_hal_mrpcim_getpause_data -Pause frame frame generation and reception. * @devh: HAL device handle. * @port : Port number 0, 1, or 2 * @tx : A field to return the pause generation capability of the NIC. * @rx : A field to return the pause reception capability of the NIC. * * Returns the Pause frame generation and reception capability of the NIC. * Return value: * status */ vxge_hal_status_e vxge_hal_mrpcim_getpause_data(vxge_hal_device_h devh, u32 port, u32 *tx, u32 *rx); /* * vxge_hal_mrpcim_setpause_data - set/reset pause frame generation. * @devh: HAL device handle. * @port : Port number 0, 1, or 2 * @tx: A field that indicates the pause generation capability to be * set on the NIC. * @rx: A field that indicates the pause reception capability to be * set on the NIC. * * It can be used to set or reset Pause frame generation or reception * support of the NIC. * Return value: * int, returns 0 on Success */ vxge_hal_status_e vxge_hal_mrpcim_setpause_data( vxge_hal_device_h devh, u32 port, u32 tx, u32 rx); /* * vxge_hal_mrpcim_bist_test - invokes the MemBist test of the card . * @devh: HAL device handle. * @data:variable that returns the result of each of the test conducted by * the driver. * * This invokes the MemBist test of the card. We give around * 2 secs time for the Test to complete. If it's still not complete * within this peiod, we consider that the test failed. * Return value: * 0 on success and -1 on failure. */ vxge_hal_status_e vxge_hal_mrpcim_bist_test(vxge_hal_device_h devh, u64 *data); /* * vxge_hal_mrpcim_udp_rth_enable - Enable UDP/RTH. * @devh: HAL device handle. * * enable udp rth * */ vxge_hal_status_e vxge_hal_mrpcim_udp_rth_enable( vxge_hal_device_h devh); /* * Virtual Paths */ /* * struct vxge_hal_vpath_attr_t - Attributes of virtual path * @vp_id: Identifier of Virtual Path * @ring_attr: Attributes of ring for non-offload receive * @fifo_attr: Attributes of fifo for non-offload transmit * * Attributes of virtual path. This structure is passed as parameter * to the vxge_hal_vpath_open() routine to set the attributes of DMQ, UMQ, * ring and fifo. After virtual path is open, iWARP/RDMA module can attach * to virtual path. */ typedef struct vxge_hal_vpath_attr_t { u32 vp_id; vxge_hal_ring_attr_t ring_attr; vxge_hal_fifo_attr_t fifo_attr; } vxge_hal_vpath_attr_t; /* * vxge_hal_vpath_open - Open a virtual path on a given adapter * @devh: handle to device object * @attr: Virtual path attributes * @cb_fn: Call back to be called to complete an asynchronous function call * @client_handle: handle to be returned in the callback * @vpath_handle: Buffer to return a handle to the vpath * * This function is used to open access to virtual path of an * adapter for offload, LRO and SPDM operations. This function returns * synchronously. */ vxge_hal_status_e vxge_hal_vpath_open( vxge_hal_device_h devh, vxge_hal_vpath_attr_t *attr, vxge_hal_vpath_callback_f cb_fn, vxge_hal_client_h client_handle, vxge_hal_vpath_h *vpath_handle); /* * vxge_hal_vpath_enable * @vpath_handle: Handle to the vpath object * * This routine clears the vpath reset and puts vpath in service */ vxge_hal_status_e vxge_hal_vpath_enable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_id - Get virtual path ID * @vpath_handle: Handle got from previous vpath open * * This function returns virtual path id */ u32 vxge_hal_vpath_id(vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_close - Close the handle got from previous vpath (vpath) open * @vpath_handle: Handle got from previous vpath open * * This function is used to close access to virtual path opened * earlier. This function returns synchronously. */ vxge_hal_status_e vxge_hal_vpath_close(vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_reset - Resets vpath * @vpath_handle: Handle got from previous vpath open * * This function is used to request a reset of vpath */ vxge_hal_status_e vxge_hal_vpath_reset(vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_reset_poll - Poll for reset complete * @vpath_handle: Handle got from previous vpath open * * This function is used to poll for the vpath reset completion */ vxge_hal_status_e vxge_hal_vpath_reset_poll(vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_obj_count_get - Get the Object usage count for a given * virtual path * @vpath_handle: Virtal path handle * @obj_counts: Buffer to return object counts * * This function returns the object counts for virtual path. This function * returns synchronously. */ vxge_hal_status_e vxge_hal_vpath_obj_count_get(vxge_hal_vpath_h vpath_handle, vxge_hal_vpath_sw_obj_count_t *obj_counts); /* * vxge_hal_vpath_mtu_check - check MTU value for ranges * @vpath_handle: Virtal path handle * @new_mtu: new MTU value to check * * Will do sanity check for new MTU value. * * Returns: VXGE_HAL_OK - success. * VXGE_HAL_ERR_INVALID_MTU_SIZE - MTU is invalid. * * See also: vxge_hal_vpath_mtu_set() */ vxge_hal_status_e vxge_hal_device_mtu_check(vxge_hal_vpath_h vpath_handle, unsigned long new_mtu); /* * vxge_hal_vpath_mtu_set - Set MTU. * @vpath_handle: Virtal path handle * @new_mtu: New MTU size to configure. * * Set new MTU value. Example, to use jumbo frames: * vxge_hal_vpath_mtu_set(my_device, 9600); * */ vxge_hal_status_e vxge_hal_vpath_mtu_set(vxge_hal_vpath_h vpath_handle, unsigned long new_mtu); typedef enum vxge_hal_vpath_mac_addr_add_mode_e { VXGE_HAL_VPATH_MAC_ADDR_ADD_DUPLICATE = 0, VXGE_HAL_VPATH_MAC_ADDR_DISCARD_DUPLICATE = 1, VXGE_HAL_VPATH_MAC_ADDR_REPLACE_DUPLICATE = 2 } vxge_hal_vpath_mac_addr_add_mode_e; /* * vxge_hal_vpath_mac_addr_add - Add the mac address entry for this vpath * to MAC address table. * @vpath_handle: Vpath handle. * @macaddr: MAC address to be added for this vpath into the list * @macaddr_mask: MAC address mask for mac_addr * @duplicate_mode: Duplicate MAC address add mode. Please see * vxge_hal_vpath_mac_addr_add_mode_e {} * * Adds the given mac address and mac address mask into the list for this * vpath. * see also: vxge_hal_vpath_mac_addr_delete, vxge_hal_vpath_mac_addr_get and * vxge_hal_vpath_mac_addr_get_next * */ vxge_hal_status_e vxge_hal_vpath_mac_addr_add( vxge_hal_vpath_h vpath_handle, macaddr_t macaddr, macaddr_t macaddr_mask, vxge_hal_vpath_mac_addr_add_mode_e duplicate_mode); /* * vxge_hal_vpath_mac_addr_get - Get the first mac address entry for this vpath * from MAC address table. * @vpath_handle: Vpath handle. * @macaddr: First MAC address entry for this vpath in the list * @macaddr_mask: MAC address mask for mac_addr * * Returns the first mac address and mac address mask in the list for this * vpath. * see also: vxge_hal_vpath_mac_addr_get_next * */ vxge_hal_status_e vxge_hal_vpath_mac_addr_get( vxge_hal_vpath_h vpath_handle, macaddr_t macaddr, macaddr_t macaddr_mask); /* * vxge_hal_vpath_mac_addr_get_next - Get the next mac address entry for this * vpath from MAC address table. * @vpath_handle: Vpath handle. * @macaddr: Next MAC address entry for this vpath in the list * @macaddr_mask: MAC address mask for mac_addr * * Returns the next mac address and mac address mask in the list for this * vpath. * see also: vxge_hal_vpath_mac_addr_get * */ vxge_hal_status_e vxge_hal_vpath_mac_addr_get_next( vxge_hal_vpath_h vpath_handle, macaddr_t macaddr, macaddr_t macaddr_mask); /* * vxge_hal_vpath_mac_addr_delete - Delete the mac address entry for this vpath * to MAC address table. * @vpath_handle: Vpath handle. * @macaddr: MAC address to be added for this vpath into the list * @macaddr_mask: MAC address mask for macaddr * * Delete the given mac address and mac address mask into the list for this * vpath. * see also: vxge_hal_vpath_mac_addr_add, vxge_hal_vpath_mac_addr_get and * vxge_hal_vpath_mac_addr_get_next * */ vxge_hal_status_e vxge_hal_vpath_mac_addr_delete( vxge_hal_vpath_h vpath_handle, macaddr_t macaddr, macaddr_t macaddr_mask); /* * vxge_hal_vpath_vid_add - Add the vlan id entry for this vpath * to vlan id table. * @vpath_handle: Vpath handle. * @vid: vlan id to be added for this vpath into the list * * Adds the given vlan id into the list for this vpath. * see also: vxge_hal_vpath_vid_delete, vxge_hal_vpath_vid_get and * vxge_hal_vpath_vid_get_next * */ vxge_hal_status_e vxge_hal_vpath_vid_add( vxge_hal_vpath_h vpath_handle, u64 vid); /* * vxge_hal_vpath_vid_get - Get the first vid entry for this vpath * from vlan id table. * @vpath_handle: Vpath handle. * @vid: Buffer to return vlan id * * Returns the first vlan id in the list for this vpath. * see also: vxge_hal_vpath_vid_get_next * */ vxge_hal_status_e vxge_hal_vpath_vid_get( vxge_hal_vpath_h vpath_handle, u64 *vid); /* * vxge_hal_vpath_vid_get_next - Get the next vid entry for this vpath * from vlan id table. * @vpath_handle: Vpath handle. * @vid: Buffer to return vlan id * * Returns the next vlan id in the list for this vpath. * see also: vxge_hal_vpath_vid_get * */ vxge_hal_status_e vxge_hal_vpath_vid_get_next( vxge_hal_vpath_h vpath_handle, u64 *vid); /* * vxge_hal_vpath_vid_delete - Delete the vlan id entry for this vpath * to vlan id table. * @vpath_handle: Vpath handle. * @vid: vlan id to be added for this vpath into the list * * Adds the given vlan id into the list for this vpath. * see also: vxge_hal_vpath_vid_add, vxge_hal_vpath_vid_get and * vxge_hal_vpath_vid_get_next * */ vxge_hal_status_e vxge_hal_vpath_vid_delete( vxge_hal_vpath_h vpath_handle, u64 vid); /* * vxge_hal_vpath_etype_add - Add the Ethertype entry for this vpath * to Ethertype table. * @vpath_handle: Vpath handle. * @etype: ethertype to be added for this vpath into the list * * Adds the given Ethertype into the list for this vpath. * see also: vxge_hal_vpath_etype_delete, vxge_hal_vpath_etype_get and * vxge_hal_vpath_etype_get_next * */ vxge_hal_status_e vxge_hal_vpath_etype_add( vxge_hal_vpath_h vpath_handle, u64 etype); /* * vxge_hal_vpath_etype_get - Get the first ethertype entry for this vpath * from Ethertype table. * @vpath_handle: Vpath handle. * @etype: Buffer to return Ethertype * * Returns the first ethype entry in the list for this vpath. * see also: vxge_hal_vpath_etype_get_next * */ vxge_hal_status_e vxge_hal_vpath_etype_get( vxge_hal_vpath_h vpath_handle, u64 *etype); /* * vxge_hal_vpath_etype_get_next - Get the next Ethertype entry for this vpath * from Ethertype table. * @vpath_handle: Vpath handle. * @etype: Buffer to return Ethwrtype * * Returns the next Ethwrtype in the list for this vpath. * see also: vxge_hal_vpath_etype_get * */ vxge_hal_status_e vxge_hal_vpath_etype_get_next( vxge_hal_vpath_h vpath_handle, u64 *etype); /* * vxge_hal_vpath_etype_delete - Delete the Ethertype entry for this vpath * to Ethertype table. * @vpath_handle: Vpath handle. * @etype: ethertype to be added for this vpath into the list * * Adds the given Ethertype into the list for this vpath. * see also: vxge_hal_vpath_etype_add, vxge_hal_vpath_etype_get and * vxge_hal_vpath_etype_get_next * */ vxge_hal_status_e vxge_hal_vpath_etype_delete( vxge_hal_vpath_h vpath_handle, u64 etype); /* * vxge_hal_vpath_port_add - Add the port entry for this vpath * to port number table. * @vpath_handle: Vpath handle. * @port_type: if 0 - Src port or 1 - Dest port * @protocol: if 0 - TCP or 1 - UDP * @port: port to be added for this vpath into the list * * Adds the given port into the list for this vpath. * see also: vxge_hal_vpath_port_delete, vxge_hal_vpath_port_get and * vxge_hal_vpath_port_get_next * */ vxge_hal_status_e vxge_hal_vpath_port_add( vxge_hal_vpath_h vpath_handle, u32 port_type, u32 protocol, u32 port); /* * vxge_hal_vpath_port_get - Get the first port number entry for this vpath * from port number table. * @vpath_handle: Vpath handle. * @port_type: Buffer to return if 0 - Src port or 1 - Dest port * @protocol: Buffer to return if 0 - TCP or 1 - UDP * @port: Buffer to return port number * * Returns the first port number entry in the list for this vpath. * see also: vxge_hal_vpath_port_get_next * */ vxge_hal_status_e vxge_hal_vpath_port_get( vxge_hal_vpath_h vpath_handle, u32 *port_type, u32 *protocol, u32 *port); /* * vxge_hal_vpath_port_get_next - Get the next port number entry for this vpath * from port number table. * @vpath_handle: Vpath handle. * @port_type: Buffer to return if 0 - Src port or 1 - Dest port * @protocol: Buffer to return if 0 - TCP or 1 - UDP * @port: Buffer to return port number * * Returns the next port number entry in the list for this vpath. * see also: vxge_hal_vpath_port_get * */ vxge_hal_status_e vxge_hal_vpath_port_get_next( vxge_hal_vpath_h vpath_handle, u32 *port_type, u32 *protocol, u32 *port); /* * vxge_hal_vpath_port_delete - Delete the port entry for this vpath * to port number table. * @vpath_handle: Vpath handle. * @port_type: if 0 - Src port or 1 - Dest port * @protocol: if 0 - TCP or 1 - UDP * @port: port to be added for this vpath into the list * * Adds the given port into the list for this vpath. * see also: vxge_hal_vpath_port_add, vxge_hal_vpath_port_get and * vxge_hal_vpath_port_get_next * */ vxge_hal_status_e vxge_hal_vpath_port_delete( vxge_hal_vpath_h vpath_handle, u32 port_type, u32 protocol, u32 port); typedef enum vxge_hal_rth_algoritms_t { RTH_ALG_NONE = -1, RTH_ALG_JENKINS = 0, RTH_ALG_MS_RSS = 1, RTH_ALG_CRC32C = 2 } vxge_hal_rth_algoritms_t; /* * struct vxge_hal_rth_hash_types_t - RTH hash types. * @hash_type_tcpipv4_en: Enables RTH field type HashTypeTcpIPv4 * @hash_type_ipv4_en: Enables RTH field type HashTypeIPv4 * @hash_type_tcpipv6_en: Enables RTH field type HashTypeTcpIPv6 * @hash_type_ipv6_en: Enables RTH field type HashTypeIPv6 * @hash_type_tcpipv6ex_en: Enables RTH field type HashTypeTcpIPv6Ex * @hash_type_ipv6ex_en: Enables RTH field type HashTypeIPv6Ex * * Used to pass RTH hash types to rts_rts_set. * * See also: vxge_hal_vpath_rts_rth_set(), vxge_hal_vpath_rts_rth_get(). */ typedef struct vxge_hal_rth_hash_types_t { u8 hash_type_tcpipv4_en; u8 hash_type_ipv4_en; u8 hash_type_tcpipv6_en; u8 hash_type_ipv6_en; u8 hash_type_tcpipv6ex_en; u8 hash_type_ipv6ex_en; } vxge_hal_rth_hash_types_t; /* * vxge_hal_vpath_udp_rth_disable - Disable UDP/RTH. * @vpath_handle: Vpath handle. * * Disable udp rth * */ vxge_hal_status_e vxge_hal_vpath_udp_rth_disable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_rts_rth_set - Set/configure RTS hashing. * @vpath_handle: Virtual Path handle. * @algorithm: Algorithm Select * @hash_type: Hash Type * @bucket_size: no of least significant bits to be used for hashing. * @it_switch: Itable switch required * * Used to set/configure all RTS hashing related stuff. * * See also: vxge_hal_vpath_rts_rth_clr(), vxge_hal_vpath_rts_rth_itable_set(). */ vxge_hal_status_e vxge_hal_vpath_rts_rth_set( vxge_hal_vpath_h vpath_handle, vxge_hal_rth_algoritms_t algorithm, vxge_hal_rth_hash_types_t *hash_type, u16 bucket_size, u16 it_switch); /* * vxge_hal_vpath_rts_rth_get - Read RTS hashing. * @vpath_handle: Virtual Path handle. * @algorithm: Buffer to return Algorithm Select * @hash_type: Buffer to return Hash Type * @table_select: Buffer to return active Table * @bucket_size: Buffer to return no of least significant bits used for hashing. * * Used to read all RTS hashing related stuff. * * See also: vxge_hal_vpath_rts_rth_clr(), vxge_hal_vpath_rts_rth_itable_set(), * vxge_hal_vpath_rts_rth_set(). */ vxge_hal_status_e vxge_hal_vpath_rts_rth_get( vxge_hal_vpath_h vpath_handle, vxge_hal_rth_algoritms_t *algorithm, vxge_hal_rth_hash_types_t *hash_type, u8 *table_select, u16 *bucket_size); /* * vxge_hal_vpath_rts_rth_key_set - Configure 40byte secret for hash calc. * * @vpath_handle: Virtual Path ahandle. * @KeySize: Number of 64-bit words * @Key: upto 40-byte array of 64-bit values * This function configures the 40-byte secret which is used for hash * calculation. * * See also: vxge_hal_vpath_rts_rth_clr(), vxge_hal_vpath_rts_rth_set(). */ vxge_hal_status_e vxge_hal_vpath_rts_rth_key_set( vxge_hal_vpath_h vpath_handle, u8 KeySize, u64 *Key); /* * vxge_hal_vpath_rts_rth_key_get - Read 40byte secret for hash calc. * * @vpath_handle: Virtual Path ahandle. * @KeySize: Number of 64-bit words * @Key: Buffer to return the key * This function reads the 40-byte secret which is used for hash * calculation. * * See also: vxge_hal_vpath_rts_rth_clr(), vxge_hal_vpath_rts_rth_set(), * vxge_hal_vpath_rts_rth_key_set(). */ vxge_hal_status_e vxge_hal_vpath_rts_rth_key_get( vxge_hal_vpath_h vpath_handle, u8 KeySize, u64 *Key); /* * vxge_hal_vpath_rts_rth_jhash_cfg_set - Configure JHASH algorithm * * @vpath_handle: Virtual Path ahandle. * @golden_ratio: Golden ratio * @init_value: Initial value * This function configures JENKIN's HASH algorithm * * See also: vxge_hal_vpath_rts_rth_clr(), vxge_hal_vpath_rts_rth_set(). */ vxge_hal_status_e vxge_hal_vpath_rts_rth_jhash_cfg_set( vxge_hal_vpath_h vpath_handle, u32 golden_ratio, u32 init_value); /* * vxge_hal_vpath_rts_rth_jhash_cfg_get - Read JHASH algorithm * * @vpath_handle: Virtual Path ahandle. * @golden_ratio: Buffer to return Golden ratio * @init_value: Buffer to return Initial value * This function reads JENKIN's HASH algorithm * * See also: vxge_hal_vpath_rts_rth_clr(), vxge_hal_vpath_rts_rth_set(), * vxge_hal_vpath_rts_rth_jhash_cfg_set(). */ vxge_hal_status_e vxge_hal_vpath_rts_rth_jhash_cfg_get( vxge_hal_vpath_h vpath_handle, u32 *golden_ratio, u32 *init_value); /* * vxge_hal_vpath_rts_rth_mask_set - Set/configure JHASH mask. * @vpath_handle: Virtual Path ahandle. * @table_size: Size of the mask table * @hash_mask_ipv6sa: IPv6SA Hash Mask * @hash_mask_ipv6da: IPv6DA Hash Mask * @hash_mask_ipv4sa: IPv4SA Hash Mask * @hash_mask_ipv4da: IPv4DA Hash Mask * @hash_mask_l4sp: L4SP Hash Mask * @hash_mask_l4dp: L4DP Hash Mask * * Used to set/configure indirection table masks. * * See also: vxge_hal_vpath_rts_rth_clr(), vxge_hal_vpath_rts_rth_set(). */ vxge_hal_status_e vxge_hal_vpath_rts_rth_mask_set( vxge_hal_vpath_h vpath_handle, u32 table_size, u32 *hash_mask_ipv6sa, u32 *hash_mask_ipv6da, u32 *hash_mask_ipv4sa, u32 *hash_mask_ipv4da, u32 *hash_mask_l4sp, u32 *hash_mask_l4dp); /* * vxge_hal_vpath_rts_rth_mask_get - Read JHASH mask. * @vpath_handle: Virtual Path ahandle. * @table_size: Size of the mask table * @hash_mask_ipv6sa: Buffer to return IPv6SA Hash Mask * @hash_mask_ipv6da: Buffer to return IPv6DA Hash Mask * @hash_mask_ipv4sa: Buffer to return IPv4SA Hash Mask * @hash_mask_ipv4da: Buffer to return IPv4DA Hash Mask * @hash_mask_l4sp: Buffer to return L4SP Hash Mask * @hash_mask_l4dp: Buffer to return L4DP Hash Mask * * Used to read rth mask. * * See also: vxge_hal_vpath_rts_rth_clr(), vxge_hal_vpath_rts_rth_set(), * vxge_hal_vpath_rts_rth_mask_set(). */ vxge_hal_status_e vxge_hal_vpath_rts_rth_mask_get( vxge_hal_vpath_h vpath_handle, u32 table_size, u32 *hash_mask_ipv6sa, u32 *hash_mask_ipv6da, u32 *hash_mask_ipv4sa, u32 *hash_mask_ipv4da, u32 *hash_mask_l4sp, u32 *hash_mask_l4dp); /* * vxge_hal_vpath_rts_rth_itable_set - Set/configure indirection table (IT). * @vpath_handles: Virtual Path handles. * @vpath_count: Number of vpath handles passed in vpath_handles * @itable: Pointer to indirection table * @itable_size: Number of entries in itable * * Used to set/configure indirection table. * It enables the required no of entries in the IT. * It adds entries to the IT. * * See also: vxge_hal_vpath_rts_rth_clr(), vxge_hal_vpath_rts_rth_set(). */ vxge_hal_status_e vxge_hal_vpath_rts_rth_itable_set( vxge_hal_vpath_h *vpath_handles, u32 vpath_count, u8 *itable, u32 itable_size); /* * vxge_hal_vpath_rts_rth_itable_get - Read indirection table (IT). * @vpath_handles: Virtual Path handles. * @vpath_count: Number of vpath handles passed in vpath_handles * @itable: Pointer to the buffer to return indirection table * @itable_size: pointer to buffer to return Number of entries in itable * * Used to read indirection table. * * See also: vxge_hal_vpath_rts_rth_clr(), vxge_hal_vpath_rts_rth_set(), * vxge_hal_vpath_rts_rth_itable_set(). */ vxge_hal_status_e vxge_hal_vpath_rts_rth_itable_get( vxge_hal_vpath_h *vpath_handles, u32 vpath_count, u8 *itable, u32 itable_size); /* * vxge_hal_vpath_rts_rth_clr - Clear RTS hashing. * @vpath_handles: Virtual Path handles. * @vpath_count: Number of vpath handles passed in vpath_handles * * This function is used to clear all RTS hashing related stuff. * * See also: vxge_hal_vpath_rts_rth_set(), vxge_hal_vpath_rts_rth_itable_set(). */ vxge_hal_status_e vxge_hal_vpath_rts_rth_clr( vxge_hal_vpath_h *vpath_handles, u32 vpath_count); /* * vxge_hal_vpath_promisc_enable - Enable promiscuous mode. * @vpath_handle: Vpath handle. * * Enable promiscuous mode of X3100 operation. * * See also: vxge_hal_vpath_promisc_disable(). */ vxge_hal_status_e vxge_hal_vpath_promisc_enable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_promisc_disable - Disable promiscuous mode. * @vpath_handle: Vpath handle. * * Disable promiscuous mode of X3100 operation. * * See also: vxge_hal_vpath_promisc_enable(). */ vxge_hal_status_e vxge_hal_vpath_promisc_disable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_bcast_enable - Enable broadcast * @vpath_handle: Vpath handle. * * Enable receiving broadcasts. */ vxge_hal_status_e vxge_hal_vpath_bcast_enable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_bcast_disable - Disable broadcast * @vpath_handle: Vpath handle. * * Disable receiving broadcasts. */ vxge_hal_status_e vxge_hal_vpath_bcast_disable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_mcast_enable - Enable multicast addresses. * @vpath_handle: Vpath handle. * * Enable X3100 multicast addresses. * Returns: VXGE_HAL_OK on success. * */ vxge_hal_status_e vxge_hal_vpath_mcast_enable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_mcast_disable - Disable multicast addresses. * @vpath_handle: Vpath handle. * * Disable X3100 multicast addresses. * Returns: VXGE_HAL_OK - success. * */ vxge_hal_status_e vxge_hal_vpath_mcast_disable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_ucast_enable - Enable unicast addresses. * @vpath_handle: Vpath handle. * * Enable X3100 unicast addresses. * Returns: VXGE_HAL_OK on success. * */ vxge_hal_status_e vxge_hal_vpath_ucast_enable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_ucast_disable - Disable unicast addresses. * @vpath_handle: Vpath handle. * * Disable X3100 unicast addresses. * Returns: VXGE_HAL_OK - success. * */ vxge_hal_status_e vxge_hal_vpath_ucast_disable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_all_vid_enable - Enable all Vlan Ids. * @vpath_handle: Vpath handle. * * Enable X3100 vlan ids. * Returns: VXGE_HAL_OK on success. * */ vxge_hal_status_e vxge_hal_vpath_all_vid_enable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_all_vid_disable - Disable all Vlan Ids. * @vpath_handle: Vpath handle. * * Disable X3100 vlan ids. * Returns: VXGE_HAL_OK - success. * */ vxge_hal_status_e vxge_hal_vpath_all_vid_disable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_strip_vlan_tag_enable - Enable strip vlan tag. * @vpath_handle: Vpath handle. * * Enable X3100 strip vlan tag. * Returns: VXGE_HAL_OK on success. * */ vxge_hal_status_e vxge_hal_vpath_strip_vlan_tag_enable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_strip_vlan_tag_disable - Disable strip vlan tag. * @vpath_handle: Vpath handle. * * Disable X3100 strip vlan tag. * Returns: VXGE_HAL_OK on success. * */ vxge_hal_status_e vxge_hal_vpath_strip_vlan_tag_disable( vxge_hal_vpath_h vpath_handle); void vxge_hal_vpath_dynamic_tti_rtimer_set(vxge_hal_vpath_h vpath_handle, u32 timer_val); void vxge_hal_vpath_dynamic_rti_rtimer_set(vxge_hal_vpath_h vpath_handle, u32 timer_val); void vxge_hal_vpath_tti_ci_set(vxge_hal_vpath_h vpath_handle); void vxge_hal_vpath_tti_ci_reset(vxge_hal_vpath_h vpath_handle); void vxge_hal_vpath_rti_ci_set(vxge_hal_vpath_h vpath_handle); void vxge_hal_vpath_rti_ci_reset(vxge_hal_vpath_h vpath_handle); /* * struct vxge_hal_vpath_tpa_params - Vpath TPA Parameters. * @tpa_lsov2_en: LSOv2 Behaviour for IP ID roll-over * 1 - enable, 0 - disable, * VXGE_HAL_DEFAULT_32 - don't change current setting * @tpa_ignore_frame_error: Ignore Frame Error. TPA may detect frame integrity * errors as it processes each frame. If this bit is set to '0', * the TPA will tag such frames as invalid and they will be dropped * by the transmit MAC. If the bit is set to '1',the frame will not * be tagged as "errored". Detectable errors include: * 1) early end-of-frame error, which occurs when the frame ends * before the number of bytes predicted by the IP "total length" * field have been received; * 2) IP version mismatches; * 3) IPv6 packets that include routing headers that are not type 0 * 4) Frames which contain IP packets but have an illegal SNAP-OUI * or LLC-CTRL fields, unless IGNORE_SNAP_OUI or IGNORE_LLC_CTRL * are set (see below). * setting the value of this field to VXGE_HAL_DEFAULT_32 - don't * change current setting * @tpa_ipv6_keep_searching: If unknown IPv6 header is found, * 0 - stop searching for TCP * 1 - keep searching for TCP * VXGE_HAL_DEFAULT_32 - don't change current setting * @tpa_l4_pshdr_present: If asserted true, indicates the host has provided a * valid pseudo header for TCP or UDP running over IPv4 or IPv6 * if set to VXGE_HAL_DEFAULT_32 - don't change current setting * @tpa_support_mobile_ipv6_hdrs: This register is somewhat equivalent to * asserting both Hercules register fields LSO_RT2_EN and * LSO_IPV6_HAO_EN. Enable/disable support for Type 2 Routing * Headers, and for Mobile-IPv6 Home Address Option (HAO), as * defined by mobile-ipv6. if set to VXGE_HAL_DEFAULT_32 - * don't change current setting * * See also: vxge_hal_vpath_tpa_set() */ typedef struct vxge_hal_vpath_tpa_params { u32 tpa_lsov2_en; u32 tpa_ignore_frame_error; u32 tpa_ipv6_keep_searching; u32 tpa_l4_pshdr_present; u32 tpa_support_mobile_ipv6_hdrs; } vxge_hal_vpath_tpa_params; /* * vxge_hal_vpath_tpa_set - Set tpa parameters. * @vpath_handle: Virtual Path ahandle. * @params: vxge_hal_vpath_tpa_params {} structure with parameters * * The function sets the tpa parametrs for the vpath. * * See also: vxge_hal_vpath_tpa_params {} */ vxge_hal_status_e vxge_hal_vpath_tpa_set( vxge_hal_vpath_h vpath_handle, vxge_hal_vpath_tpa_params *params); /* * struct vxge_hal_vpath_rpa_params - Vpath RPA Parameters. * * @rpa_ipv4_tcp_incl_ph: Determines if the pseudo-header is included in the * calculation of the L4 checksum that is passed to the host. This * field applies to TCP/IPv4 packets only. This field affects both * non-offload and LRO traffic. Note that the RPA always includes * the pseudo-header in the "Checksum Ok" L4 checksum calculation * i.e. the checksum that decides whether a frame is a candidate to * be offloaded. * 0 - Do not include the pseudo-header in L4 checksum calculation. * This setting should be used if the adapter is incorrectly * calculating the pseudo-header. * 1 - Include the pseudo-header in L4 checksum calculation * VXGE_HAL_DEFAULT_32 - don't change current setting * @rpa_ipv6_tcp_incl_ph: Determines whether the pseudo-header is included in * the calculation of the L4 checksum that is passed to the host. * This field applies to TCP/IPv6 packets only. This field affects * both non-offload and LRO traffic. Note that the RPA always * includes the pseudo-header in the "Checksum Ok" L4 checksum * calculation. i.e. the checksum that decides whether a frame * is a candidate to be offloaded. * 0 - Do not include the pseudo-header in L4 checksum calculation. * This setting should be used if the adapter is incorrectly * calculating the pseudo-header. * 1 - Include the pseudo-header in L4 checksum calculation * VXGE_HAL_DEFAULT_32 - don't change current setting * @rpa_ipv4_udp_incl_ph: Determines whether the pseudo-header is included in * the calculation of the L4 checksum that is passed to the host. * This field applies to UDP/IPv4 packets only. It only affects * non-offload traffic(since UDP frames are not candidates for LRO) * 0 - Do not include the pseudo-header in L4 checksum calculation. * This setting should be used if the adapter is incorrectly * calculating the pseudo-header. * 1 - Include the pseudo-header in L4 checksum calculation * VXGE_HAL_DEFAULT_32 - don't change current setting * @rpa_ipv6_udp_incl_ph: Determines if the pseudo-header is included in the * calculation of the L4 checksum that is passed to the host. This * field applies to UDP/IPv6 packets only. It only affects * non-offload traffic(since UDP frames are not candidates for LRO) * 0 - Do not include the pseudo-header in L4 checksum calculation. * This setting should be used if the adapter is incorrectly * calculating the pseudo-header. * 1 - Include the pseudo-header in L4 checksum calculation * VXGE_HAL_DEFAULT_32 - don't change current setting * @rpa_l4_incl_cf: Determines whether the checksum field (CF) of the received * frame is included in the calculation of the L4 checksum that is * passed to the host. This field affects both non-offload and LRO * traffic. Note that the RPA always includes the checksum field in * the "Checksum Ok" L4 checksum calculation -- i.e. the checksum * that decides whether a frame is a candidate to be offloaded. * 0 - Do not include the checksum field in L4 checksum calculation * 1 - Include the checksum field in L4 checksum calculation * VXGE_HAL_DEFAULT_32 - don't change current setting * @rpa_strip_vlan_tag: Strip VLAN Tag enable/disable. Instructs the device to * remove the VLAN tag from all received tagged frames that are not * replicated at the internal L2 switch. * 0 - Do not strip the VLAN tag. * 1 - Strip the VLAN tag. Regardless of this setting,VLAN tags are * always placed into the RxDMA descriptor. * VXGE_HAL_DEFAULT_32 - don't change current setting * @rpa_l4_comp_csum: Determines whether the calculated L4 checksum should be * complemented before it is passed to the host This field affects * both non-offload and LRO traffic. * 0 - Do not complement the calculated L4 checksum. * 1 - Complement the calculated L4 checksum * VXGE_HAL_DEFAULT_32 - don't change current setting * @rpa_l3_incl_cf: Determines whether the checksum field (CF) of the received * frame is included in the calculation of the L3 checksum that is * passed to the host. This field affects both non-offload and LRO * traffic. Note that the RPA always includes the checksum field in * the "Checksum Ok" L3 checksum calculation -- i.e. the checksum * that decides whether a frame is a candidate to be offloaded. * 0 - Do not include the checksum field in L3 checksum calculation * 1 - Include the checksum field in L3 checksum calculation * VXGE_HAL_DEFAULT_32 - don't change current setting * @rpa_l3_comp_csum: Determines whether the calculated L3 checksum should be * complemented before it is passed to the host This field affects * both non-offload and LRO traffic. * 0 - Do not complement the calculated L3 checksum. * 1 - Complement the calculated L3 checksum * VXGE_HAL_DEFAULT_32 - don't change current setting * @rpa_ucast_all_addr_en: Enables frames with any unicast address (as its * destination address) to be passed to the host. * Setting this field to VXGE_HAL_DEFAULT_32 - don't change current * setting * @rpa_mcast_all_addr_en: Enables frames with any multicast address (as its * destination address) to be passed to the host. * Setting this field to VXGE_HAL_DEFAULT_32 - don't change current * setting * @rpa_bcast_en: Enables frames with any broadicast address (as its * destination address) to be passed to the host. * Setting this field to VXGE_HAL_DEFAULT_32 - don't change current * setting * @rpa_all_vid_en: romiscuous mode, it overrides the value held in this field. * 0 - Disable; * 1 - Enable * VXGE_HAL_DEFAULT_32 - don't change current setting * Note: RXMAC_GLOBAL_CFG.AUTHORIZE_VP_ALL_VID must be set to * allow this. * * See also: vxge_hal_vpath_rpa_set() */ typedef struct vxge_hal_vpath_rpa_params { u32 rpa_ipv4_tcp_incl_ph; u32 rpa_ipv6_tcp_incl_ph; u32 rpa_ipv4_udp_incl_ph; u32 rpa_ipv6_udp_incl_ph; u32 rpa_l4_incl_cf; u32 rpa_strip_vlan_tag; u32 rpa_l4_comp_csum; u32 rpa_l3_incl_cf; u32 rpa_l3_comp_csum; u32 rpa_ucast_all_addr_en; u32 rpa_mcast_all_addr_en; u32 rpa_bcast_en; u32 rpa_all_vid_en; } vxge_hal_vpath_rpa_params; /* * vxge_hal_vpath_rpa_set - Set rpa parameters. * @vpath_handle: Virtual Path ahandle. * @params: vxge_hal_vpath_rpa_params {} structure with parameters * * The function sets the rpa parametrs for the vpath. * * See also: vxge_hal_vpath_rpa_params {} */ vxge_hal_status_e vxge_hal_vpath_rpa_set( vxge_hal_vpath_h vpath_handle, vxge_hal_vpath_rpa_params *params); /* * vxge_hal_vpath_poll_rx - Poll Rx od Virtual Path for completed * descriptors and process the same. * @vpath_handle: Virtual Path ahandle. * @got_rx: Buffer to return the flag set if receive interrupt is occured * * The function polls the Rx for the completed descriptors and calls * the upper-layer driver (ULD) via supplied completion callback. * * Returns: VXGE_HAL_OK, if the polling is completed successful. * VXGE_HAL_COMPLETIONS_REMAIN: There are still more completed * descriptors available which are yet to be processed. * * See also: vxge_hal_vpath_poll_tx() */ vxge_hal_status_e vxge_hal_vpath_poll_rx( vxge_hal_vpath_h vpath_handle, u32 *got_rx); /* * vxge_hal_vpath_poll_tx - Poll Tx for completed descriptors and process * the same. * @vpath_handle: Virtual Path ahandle. * @got_tx: Buffer to return the flag set if transmit interrupt is occured * * The function polls the Tx for the completed descriptors and calls * the upper-layer driver (ULD) via supplied completion callback. * * Returns: VXGE_HAL_OK, if the polling is completed successful. * VXGE_HAL_COMPLETIONS_REMAIN: There are still more completed * descriptors available which are yet to be processed. * * See also: vxge_hal_vpath_poll_rx(). */ vxge_hal_status_e vxge_hal_vpath_poll_tx( vxge_hal_vpath_h vpath_handle, u32 *got_tx); /* * vxge_hal_vpath_intr_enable - Enable vpath interrupts. * @vpath_handle: Virtual Path handle. * @op: One of the vxge_hal_vpath_intr_e enumerated values specifying * the type(s) of interrupts to enable. * * Enable vpath interrupts. The function is to be executed the last in * vpath initialization sequence. * * See also: vxge_hal_vpath_intr_disable() */ vxge_hal_status_e vxge_hal_vpath_intr_enable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_intr_disable - Disable vpath interrupts. * @vpath_handle: Virtual Path handle. * @op: One of the vxge_hal_vpath_intr_e enumerated values specifying * the type(s) of interrupts to disable. * * Disable vpath interrupts. * * See also: vxge_hal_vpath_intr_enable() */ vxge_hal_status_e vxge_hal_vpath_intr_disable( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_mask_all - Mask all vpath interrupts. * @vpath_handle: Virtual Path handle. * * Mask all vpath interrupts. * * See also: vxge_hal_vpath_unmask_all() */ void vxge_hal_vpath_mask_all( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_unmask_all - Unmask all vpath interrupts. * @vpath_handle: Virtual Path handle. * * Unmask all vpath interrupts. * * See also: vxge_hal_vpath_mask_all() */ void vxge_hal_vpath_unmask_all( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_begin_irq - Begin IRQ processing. * @vpath_handle: Virtual Path handle. * @skip_alarms: Do not clear the alarms * @reason: "Reason" for the interrupt, the value of vpath's * general_int_status register. * * The function performs two actions, It first checks whether (shared IRQ) the * interrupt was raised by the device. Next, it masks the device interrupts. * * Note: * vxge_hal_vpath_begin_irq() does not flush MMIO writes through the * bridge. Therefore, two back-to-back interrupts are potentially possible. * It is the responsibility of the ULD to make sure that only one * vxge_hal_vpath_continue_irq() runs at a time. * * Returns: 0, if the interrupt is not "ours" (note that in this case the * vpath remain enabled). * Otherwise, vxge_hal_vpath_begin_irq() returns 64bit general adapter * status. * See also: vxge_hal_vpath_handle_irq() */ vxge_hal_status_e vxge_hal_vpath_begin_irq( vxge_hal_vpath_h vpath_handle, u32 skip_alarms, u64 *reason); /* * vxge_hal_vpath_continue_irq - Continue handling IRQ: process all * completed descriptors. * @vpath_handle: Virtual Path handle. * * Process completed descriptors and unmask the vpath interrupts. * * The vxge_hal_vpath_continue_irq() calls upper-layer driver (ULD) * via supplied completion callback. * * Note that the vxge_hal_vpath_continue_irq is part of the _fast_ path. * To optimize the processing, the function does _not_ check for * errors and alarms. * * Returns: VXGE_HAL_OK. * * See also: vxge_hal_vpath_handle_irq(), * vxge_hal_ring_rxd_next_completed(), * vxge_hal_fifo_txdl_next_completed(), vxge_hal_ring_callback_f {}, * vxge_hal_fifo_callback_f {}. */ vxge_hal_status_e vxge_hal_vpath_continue_irq( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpathe_handle_irq - Handle vpath IRQ. * @vpath_handle: Virtual Path handle. * @skip_alarms: Do not clear the alarms * * Perform the complete handling of the line interrupt. The function * performs two calls. * First it uses vxge_hal_vpath_begin_irq() to check the reason for * the interrupt and mask the vpath interrupts. * Second, it calls vxge_hal_vpath_continue_irq() to process all * completed descriptors and re-enable the interrupts. * * Returns: VXGE_HAL_OK - success; * VXGE_HAL_ERR_WRONG_IRQ - (shared) IRQ produced by other device. * * See also: vxge_hal_vpath_begin_irq(), vxge_hal_vpath_continue_irq(). */ vxge_hal_status_e vxge_hal_vpath_handle_irq( vxge_hal_vpath_h vpath_handle, u32 skip_alarms); /* * vxge_hal_vpath_mask_tx - Mask Tx interrupts. * @vpath_handle: Virtual Path handle. * * Mask Tx device interrupts. * * See also: vxge_hal_vpath_unmask_tx(), vxge_hal_vpath_mask_rx(), * vxge_hal_vpath_clear_tx(). */ void vxge_hal_vpath_mask_tx( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_clear_tx - Acknowledge (that is, clear) the * condition that has caused the TX interrupt. * @vpath_handle: Virtual Path handle. * * Acknowledge (that is, clear) the condition that has caused * the Tx interrupt. * See also: vxge_hal_vpath_begin_irq(), vxge_hal_vpath_continue_irq(), * vxge_hal_vpath_clear_rx(), vxge_hal_vpath_mask_tx(). */ void vxge_hal_vpath_clear_tx( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_unmask_tx - Unmask Tx interrupts. * @vpath_handle: Virtual Path handle. * * Unmask Tx vpath interrupts. * * See also: vxge_hal_vpath_mask_tx(), vxge_hal_vpath_clear_tx(). */ void vxge_hal_vpath_unmask_tx( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_mask_rx - Mask Rx interrupts. * @vpath_handle: Virtual Path handle. * * Mask Rx vpath interrupts. * * See also: vxge_hal_vpath_unmask_rx(), vxge_hal_vpath_mask_tx(), * vxge_hal_vpath_clear_rx(). */ void vxge_hal_vpath_mask_rx( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_clear_rx - Acknowledge (that is, clear) the * condition that has caused the RX interrupt. * @vpath_handle: Virtual Path handle. * * Acknowledge (that is, clear) the condition that has caused * the Rx interrupt. * See also: vxge_hal_vpath_begin_irq(), vxge_hal_vpath_continue_irq(), * vxge_hal_vpath_clear_tx(), vxge_hal_vpath_mask_rx(). */ void vxge_hal_vpath_clear_rx( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_unmask_rx - Unmask Rx interrupts. * @vpath_handle: Virtual Path handle. * * Unmask Rx vpath interrupts. * * See also: vxge_hal_vpath_mask_rx(), vxge_hal_vpath_clear_rx(). */ void vxge_hal_vpath_unmask_rx( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_mask_tx_rx - Mask Tx and Rx interrupts. * @vpath_handle: Virtual Path handle. * * Mask Tx and Rx vpath interrupts. * * See also: vxge_hal_vpath_unmask_tx_rx(), vxge_hal_vpath_clear_tx_rx(). */ void vxge_hal_vpath_mask_tx_rx( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_clear_tx_rx - Acknowledge (that is, clear) the * condition that has caused the Tx and RX interrupt. * @vpath_handle: Virtual Path handle. * * Acknowledge (that is, clear) the condition that has caused * the Tx and Rx interrupt. * See also: vxge_hal_vpath_begin_irq(), vxge_hal_vpath_continue_irq(), * vxge_hal_vpath_clear_tx_rx(), vxge_hal_vpath_mask_tx_rx(). */ void vxge_hal_vpath_clear_tx_rx( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_unmask_tx_rx - Unmask Tx and Rx interrupts. * @vpath_handle: Virtual Path handle. * * Unmask Tx and Rx vpath interrupts. * * See also: vxge_hal_vpath_mask_tx_rx(), vxge_hal_vpath_clear_tx_rx(). */ void vxge_hal_vpath_unmask_tx_rx( vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_alarm_process - Process Alarms. * @vpath: Virtual Path. * @skip_alarms: Do not clear the alarms * * Process vpath alarms. * */ vxge_hal_status_e vxge_hal_vpath_alarm_process( vxge_hal_vpath_h vpath_handle, u32 skip_alarms); /* NEWCODE */ vxge_hal_status_e vxge_hal_vpath_mf_msix_set(vxge_hal_vpath_h vpath_handle, int *tim_msix_id, int alarm_msix_id); void vxge_hal_vpath_mf_msix_clear(vxge_hal_vpath_h vpath_handle, int msix_id); void vxge_hal_vpath_mf_msix_mask(vxge_hal_vpath_h vpath_handle, int msix_id); void vxge_hal_vpath_mf_msix_unmask(vxge_hal_vpath_h vpath_handle, int msix_id); /* NEWCODE */ /* * vxge_hal_vpath_msix_mode - Is MSIX enabled? * @vpath_handle: Virtual Path handle. * * Returns 0 if MSI is enabled for the specified device, * non-zero otherwise. */ u32 vxge_hal_vpath_msix_mode(vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_msix_set - Associate MSIX vectors with TIM interrupts and * alrms * @vpath_handle: Virtual Path handle. * @tim_msix_id: MSIX vectors associated with VXGE_HAL_VPATH_MSIX_MAX number of * interrupts(Valid numbers 0 to 3). * @alarm_msix_id: MSIX vector for alarm (Valid numbers 0 to 3). * * This API will associate a given MSIX vector numbers with the four TIM * interrupts and alarm interrupt. */ vxge_hal_status_e vxge_hal_vpath_msix_set(vxge_hal_vpath_h vpath_handle, int *tim_msix_id, int alarm_msix_id); /* * vxge_hal_vpath_msix_mask - Mask MSIX Vector. * @vpath_handle: Virtual Path handle. * @msix_id: MSIX ID * * The function masks the msix interrupt for the given msix_id * * Note: * * Returns: 0, * Otherwise, VXGE_HAL_ERR_WRONG_IRQ if the msix index is out of range * status. * See also: */ void vxge_hal_vpath_msix_mask(vxge_hal_vpath_h vpath_handle, int msix_id); /* * vxge_hal_vpath_msix_clear - Clear MSIX Vector. * @vpath_handle: Virtual Path handle. * @msix_id: MSI ID * * The function clears the msix interrupt for the given msix_id * * Note: * * Returns: 0, * Otherwise, VXGE_HAL_ERR_WRONG_IRQ if the msix index is out of range * status. * See also: */ void vxge_hal_vpath_msix_clear(vxge_hal_vpath_h vpath_handle, int msix_id); /* * vxge_hal_vpath_msix_unmask - Unmask MSIX Vector. * @vpath_handle: Virtual Path handle. * @msix_id: MSI ID * * The function unmasks the msix interrupt for the given msix_id * * Note: * * Returns: 0, * Otherwise, VXGE_HAL_ERR_WRONG_IRQ if the msix index is out of range * status. * See also: */ void vxge_hal_vpath_msix_unmask(vxge_hal_vpath_h vpath_handle, int msix_id); /* * vxge_hal_vpath_msix_mask_all - Mask all MSIX vectors for the vpath. * @vpath_handle: Virtual Path handle. * * The function masks the msix interrupt for the given vpath * */ void vxge_hal_vpath_msix_mask_all(vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_msix_unmask_all - Unmask all MSIX vectors for the vpath. * @vpath_handle: Virtual Path handle. * * The function unmasks the msix interrupt for the given vpath * */ void vxge_hal_vpath_msix_unmask_all(vxge_hal_vpath_h vpath_handle); /* * vxge_hal_vpath_is_rxdmem_leak - Check for the rxd memory leak. * @vpath_handle: Virtual Path handle. * * The function checks for the rxd memory leak. * */ u32 vxge_hal_vpath_is_rxdmem_leak(vxge_hal_vpath_h vpath_handle); /* * vxge_hal_rldram_test - offline test for access to the RldRam chip on * the NIC * @devh: HAL device handle. * @data: variable that returns the result of each of the test * conducted by the driver. * * This is one of the offline test that tests the read and write * access to the RldRam chip on the NIC. * Return value: * 0 on success. */ vxge_hal_status_e vxge_hal_rldram_test( vxge_hal_device_h devh, u64 *data); /* * vxge_hal_check_alignment - Check buffer alignment and calculate the * "misaligned" portion. * @dma_pointer: DMA address of the buffer. * @size: Buffer size, in bytes. * @alignment: Alignment "granularity" (see below), in bytes. * @copy_size: Maximum number of bytes to "extract" from the buffer * (in order to spost it as a separate scatter-gather entry). See below. * * Check buffer alignment and calculate "misaligned" portion, if exists. * The buffer is considered aligned if its address is multiple of * the specified @alignment. If this is the case, * vxge_hal_check_alignment() returns zero. * Otherwise, vxge_hal_check_alignment() uses the last argument, * @copy_size, * to calculate the size to "extract" from the buffer. The @copy_size * may or may not be equal @alignment. The difference between these two * arguments is that the @alignment is used to make the decision: aligned * or not aligned. While the @copy_size is used to calculate the portion * of the buffer to "extract", i.e. to post as a separate entry in the * transmit descriptor. For example, the combination * @alignment=8 and @copy_size=64 will work okay on AMD Opteron boxes. * * Note: @copy_size should be a multiple of @alignment. In many practical * cases @copy_size and @alignment will probably be equal. * * See also: vxge_hal_fifo_txdl_buffer_set_aligned(). */ u32 vxge_hal_check_alignment( dma_addr_t dma_pointer, u32 size, u32 alignment, u32 copy_size); void vxge_hw_vpath_set_zero_rx_frm_len(vxge_hal_device_h devh, u32 vp_id); void vxge_hw_vpath_wait_receive_idle(vxge_hal_device_h devh, u32 vp_id, u32 *count, u32 *total_count); #define VXGE_HW_MIN_SUCCESSIVE_IDLE_COUNT 5 #define VXGE_HW_MAX_POLLING_COUNT 160 __EXTERN_END_DECLS #include #endif /* VXGE_HAL_LL_H */ Index: stable/9/sys/dev =================================================================== --- stable/9/sys/dev (revision 273863) +++ stable/9/sys/dev (revision 273864) Property changes on: stable/9/sys/dev ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/sys/dev:r230135,254121,255932,255972-255973 Index: stable/9/sys/modules/Makefile =================================================================== --- stable/9/sys/modules/Makefile (revision 273863) +++ stable/9/sys/modules/Makefile (revision 273864) @@ -1,846 +1,860 @@ # $FreeBSD$ .include # Modules that include binary-only blobs of microcode should be selectable by # MK_SOURCELESS_UCODE option (see below). SUBDIR= \ ${_3dfx} \ ${_3dfx_linux} \ ${_aac} \ ${_aacraid} \ accf_data \ accf_dns \ accf_http \ ${_acpi} \ ae \ ${_aesni} \ age \ ${_agp} \ aha \ ${_ahb} \ ahci \ ${_aic} \ aic7xxx \ aio \ alc \ ale \ alq \ ${_amd} \ ${_amdsbwd} \ ${_amdtemp} \ amr \ ${_an} \ ${_aout} \ ${_apm} \ ${_arcmsr} \ ${_arcnet} \ ${_asmc} \ ${_asr} \ ata \ ath \ ath_pci \ ${_auxio} \ ${_bce} \ bfe \ bge \ ${_bxe} \ ${_bios} \ ${_bktr} \ ${_bm} \ bridgestp \ bwi \ bwn \ cam \ ${_canbepm} \ ${_canbus} \ ${_cardbus} \ cas \ ${_cbb} \ cc \ cd9660 \ cd9660_iconv \ ${_ce} \ ${_ciss} \ ${_cm} \ ${_cmx} \ coda \ coda5 \ ${_coff} \ ${_coretemp} \ ${_cp} \ ${_cpuctl} \ ${_cpufreq} \ ${_crypto} \ ${_cryptodev} \ ${_cs} \ ${_ctau} \ ctl \ ${_cxgb} \ cxgbe \ ${_cyclic} \ dc \ dcons \ dcons_crom \ de \ ${_dpms} \ ${_dpt} \ ${_drm} \ ${_drm2} \ ${_dtrace} \ dummynet \ ${_ed} \ ${_elink} \ ${_em} \ en \ ${_ep} \ ${_epic} \ esp \ ${_et} \ ${_ex} \ ${_exca} \ ${_ext2fs} \ ${_fatm} \ fdc \ fdescfs \ ${_fe} \ ${_filemon} \ firewire \ firmware \ ${_fxp} \ gem \ geom \ ${_glxiic} \ ${_glxsb} \ hatm \ hifn \ hme \ ${_hpt27xx} \ ${_hptiop} \ ${_hptmv} \ ${_hptnr} \ ${_hptrr} \ hwpmc \ ${_i2c} \ + ${_ibcore} \ ${_ibcs2} \ ${_ichwd} \ ${_ida} \ ${_ie} \ if_bridge \ ${_if_carp} \ if_disc \ if_edsc \ if_ef \ if_epair \ if_faith \ if_gif \ ${_if_gre} \ if_lagg \ ${_if_ndis} \ if_stf \ if_tap \ if_tun \ if_vlan \ ${_igb} \ ${_iir} \ ${_io} \ ipdivert \ + ${_ipoib} \ ${_ipfilter} \ ${_ipfw} \ ipfw_nat \ ${_ipmi} \ ip6_mroute_mod \ ip_mroute_mod \ ${_ips} \ ${_ipw} \ ${_ipwfw} \ ${_isci} \ iscsi \ isp \ ${_ispfw} \ ${_iwi} \ ${_iwifw} \ ${_iwn} \ ${_iwnfw} \ ${_ixgb} \ ${_ixgbe} \ jme \ joy \ kbdmux \ kgssapi \ kgssapi_krb5 \ khelp \ krpc \ ksyms \ le \ lge \ libalias \ libiconv \ libmbpool \ libmchain \ ${_lindev} \ ${_linprocfs} \ ${_linsysfs} \ ${_linux} \ lmc \ lpt \ mac_biba \ mac_bsdextended \ mac_ifoff \ mac_lomac \ mac_mls \ mac_none \ mac_partition \ mac_portacl \ mac_seeotheruids \ mac_stub \ mac_test \ malo \ mcd \ md \ mem \ mfi \ mii \ mlx \ ${_mlx4} \ ${_mlx4ib} \ ${_mlxen} \ ${_mly} \ mmc \ mmcsd \ mpr \ mps \ mpt \ mqueue \ mrsas \ msdosfs \ msdosfs_iconv \ ${_mse} \ msk \ ${_mthca} \ mvs \ mwl \ ${_mwlfw} \ mxge \ my \ ${_ncp} \ ${_ncv} \ ${_ndis} \ ${_netgraph} \ ${_nfe} \ nfs_common \ nfscl \ nfsclient \ nfscommon \ nfsd \ nfslock \ nfslockd \ nfsserver \ nfssvc \ nge \ nmdm \ ${_nsp} \ ntfs \ ntfs_iconv \ nullfs \ ${_nvd} \ ${_nve} \ ${_nvme} \ ${_nvram} \ ${_nwfs} \ ${_nxge} \ ${_opensolaris} \ oce \ ${_padlock} \ patm \ ${_pccard} \ ${_pcfclock} \ pcn \ ${_pf} \ ${_pflog} \ ${_pfsync} \ plip \ ${_pmc} \ portalfs \ ppbus \ ppc \ ppi \ pps \ procfs \ pseudofs \ ${_pst} \ pty \ puc \ ${_qlxge} \ ${_qlxgb} \ ${_qlxgbe} \ ral \ ${_ralfw} \ ${_random} \ rc4 \ ${_rdma} \ re \ reiserfs \ rl \ ${_runfw} \ ${_s3} \ ${_safe} \ ${_sbni} \ scc \ scd \ ${_scsi_low} \ sdhci \ sem \ send \ ${_sf} \ ${_sfxge} \ sge \ siba_bwn \ siftr \ siis \ sis \ sk \ ${_smbfs} \ ${_sn} \ ${_snc} \ snp \ ${_sound} \ ${_speaker} \ ${_splash} \ ${_sppp} \ ste \ ${_stg} \ stge \ ${_streams} \ ${_svr4} \ ${_sym} \ ${_syscons} \ sysvipc \ ${_ti} \ tl \ tmpfs \ ${_toecore} \ ${_tpm} \ trm \ ${_twa} \ twe \ tws \ tx \ ${_txp} \ uart \ ubsec \ udf \ udf_iconv \ ufs \ unionfs \ usb \ utopia \ ${_vesa} \ ${_virtio} \ vge \ ${_viawd} \ vkbd \ ${_vpo} \ vr \ vte \ vx \ ${_vxge} \ wb \ ${_wbwd} \ ${_wi} \ wlan \ wlan_acl \ wlan_amrr \ wlan_ccmp \ wlan_rssadapt \ wlan_tkip \ wlan_wep \ wlan_xauth \ ${_wpi} \ ${_wpifw} \ ${_x86bios} \ ${_xe} \ ${_xenhvm} \ xfs \ xl \ ${_zfs} \ zlib \ .if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64" _filemon= filemon .endif .if ${MACHINE_CPUARCH} != "powerpc" && ${MACHINE_CPUARCH} != "arm" && \ ${MACHINE_CPUARCH} != "mips" _syscons= syscons _vpo= vpo .endif .if ${MACHINE_CPUARCH} != "arm" && ${MACHINE_CPUARCH} != "mips" # no BUS_SPACE_UNSPECIFIED # No barrier instruction support (specific to this driver) _sym= sym # intr_disable() is a macro, causes problems .if ${MK_SOURCELESS_UCODE} != "no" _cxgb= cxgb .endif .endif .if ${MK_CRYPT} != "no" || defined(ALL_MODULES) .if exists(${.CURDIR}/../opencrypto) _crypto= crypto _cryptodev= cryptodev .endif .if exists(${.CURDIR}/../crypto) _random= random .endif .endif .if ${MK_INET_SUPPORT} != "no" || defined(ALL_MODULES) _if_gre= if_gre .endif .if (${MK_INET_SUPPORT} != "no" || ${MK_INET6_SUPPORT} != "no") || \ defined(ALL_MODULES) _if_carp= if_carp _toecore= toecore .endif .if ${MK_IPFILTER} != "no" || defined(ALL_MODULES) _ipfilter= ipfilter .endif .if ${MK_INET_SUPPORT} != "no" || defined(ALL_MODULES) _ipfw= ipfw .endif .if ${MK_NETGRAPH} != "no" || defined(ALL_MODULES) _netgraph= netgraph .endif .if (${MK_PF} != "no" && (${MK_INET_SUPPORT} != "no" || \ ${MK_INET6_SUPPORT} != "no")) || defined(ALL_MODULES) _pf= pf _pflog= pflog .if ${MK_INET_SUPPORT} != "no" _pfsync= pfsync .endif .endif .if ${MK_SOURCELESS_UCODE} != "no" _bce= bce _fatm= fatm _fxp= fxp _ispfw= ispfw _mwlfw= mwlfw _ralfw= ralfw _runfw= runfw _sf= sf _sn= sn _ti= ti _txp= txp .endif .if ${MACHINE_CPUARCH} == "i386" # XXX some of these can move to the general case when de-i386'ed # XXX some of these can move now, but are untested on other architectures. _3dfx= 3dfx _3dfx_linux= 3dfx_linux _agp= agp _aic= aic _amd= amd _an= an _aout= aout _apm= apm _arcnet= arcnet _bktr= bktr _bxe= bxe _cardbus= cardbus _cbb= cbb .if ${MK_SOURCELESS_UCODE} != "no" _ce= ce .endif _coff= coff .if ${MK_SOURCELESS_UCODE} != "no" _cp= cp .endif _cpuctl= cpuctl _cpufreq= cpufreq _cs= cs .if ${MK_CDDL} != "no" || defined(ALL_MODULES) _cyclic= cyclic .endif _dpms= dpms _drm= drm _drm2= drm2 .if ${MK_CDDL} != "no" || defined(ALL_MODULES) _dtrace= dtrace .endif _ed= ed _elink= elink _em= em _ep= ep _et= et _exca= exca _ext2fs= ext2fs _fe= fe _glxiic= glxiic _glxsb= glxsb _i2c= i2c +.if ${MK_OFED} != "no" || defined(ALL_MODULES) +_ibcore= ibcore +.endif _ibcs2= ibcs2 _ie= ie _if_ndis= if_ndis _igb= igb _io= io +.if ${MK_OFED} != "no" || defined(ALL_MODULES) +_ipoib= ipoib +.endif _lindev= lindev _linprocfs= linprocfs _linsysfs= linsysfs _linux= linux _mse= mse .if ${MK_OFED} != "no" || defined(ALL_MODULES) _mlx4= mlx4 _mlx4ib= mlx4ib _mlxen= mlxen _mthca= mthca .endif .if ${MK_NCP} != "no" _ncp= ncp .endif _ncv= ncv _ndis= ndis _nsp= nsp .if ${MK_NCP} != "no" _nwfs= nwfs .endif .if ${MK_CDDL} != "no" || defined(ALL_MODULES) _opensolaris= opensolaris .endif _pccard= pccard _pcfclock= pcfclock _pst= pst _rdma= rdma _safe= safe _sbni= sbni _scsi_low= scsi_low _smbfs= smbfs _sound= sound _speaker= speaker _splash= splash _sppp= sppp _stg= stg _streams= streams _svr4= svr4 _vxge= vxge _wbwd= wbwd _wi= wi _xe= xe .if ${MK_ZFS} != "no" || defined(ALL_MODULES) _zfs= zfs .endif .if ${MACHINE} == "i386" _aac= aac _aacraid= aacraid _acpi= acpi .if ${MK_CRYPT} != "no" || defined(ALL_MODULES) _aesni= aesni .endif _ahb= ahb _amdsbwd= amdsbwd _amdtemp= amdtemp _arcmsr= arcmsr _asmc= asmc _asr= asr _bios= bios _ciss= ciss _cm= cm _cmx= cmx _coretemp= coretemp .if ${MK_SOURCELESS_UCODE} != "no" _ctau= ctau .endif _dpt= dpt _ex= ex .if ${MK_SOURCELESS_HOST} != "no" _hpt27xx= hpt27xx .endif _hptiop= hptiop .if ${MK_SOURCELESS_HOST} != "no" _hptmv= hptmv _hptnr= hptnr _hptrr= hptrr .endif _ichwd= ichwd _ida= ida _iir= iir _ipmi= ipmi _ips= ips _ipw= ipw .if ${MK_SOURCELESS_UCODE} != "no" _ipwfw= ipwfw .endif _isci= isci _iwi= iwi .if ${MK_SOURCELESS_UCODE} != "no" _iwifw= iwifw .endif _iwn= iwn .if ${MK_SOURCELESS_UCODE} != "no" _iwnfw= iwnfw .endif _ixgb= ixgb _ixgbe= ixgbe _mly= mly _nfe= nfe _nvd= nvd .if ${MK_SOURCELESS_HOST} != "no" _nve= nve .endif _nvme= nvme _nvram= nvram _nxge= nxge _tpm= tpm _viawd= viawd _wpi= wpi .if ${MK_SOURCELESS_UCODE} != "no" _wpifw= wpifw .endif .if ${MK_CRYPT} != "no" || defined(ALL_MODULES) _padlock= padlock .endif _s3= s3 _twa= twa _vesa= vesa _virtio= virtio _x86bios= x86bios _xenhvm= xenhvm .elif ${MACHINE} == "pc98" _canbepm= canbepm _canbus= canbus _pmc= pmc _snc= snc .endif .endif .if ${MACHINE_CPUARCH} == "amd64" _aac= aac _aacraid= aacraid _aout= aout _acpi= acpi .if ${MK_CRYPT} != "no" || defined(ALL_MODULES) _aesni= aesni .endif _agp= agp _an= an _amdsbwd= amdsbwd _amdtemp= amdtemp _arcmsr= arcmsr _asmc= asmc _bxe= bxe _cardbus= cardbus _cbb= cbb _cmx= cmx _ciss= ciss _coretemp= coretemp _cpuctl= cpuctl _cpufreq= cpufreq .if ${MK_CDDL} != "no" || defined(ALL_MODULES) _cyclic= cyclic .endif _dpms= dpms _drm= drm _drm2= drm2 .if ${MK_CDDL} != "no" || defined(ALL_MODULES) _dtrace= dtrace .endif _ed= ed _et= et _em= em _exca= exca _ext2fs= ext2fs .if ${MK_SOURCELESS_HOST} != "no" _hpt27xx= hpt27xx .endif _hptiop= hptiop .if ${MK_SOURCELESS_HOST} != "no" _hptmv= hptmv _hptnr= hptnr _hptrr= hptrr .endif _i2c= i2c +.if ${MK_OFED} != "no" || defined(ALL_MODULES) +_ibcore= ibcore +.endif _ichwd= ichwd _ida= ida _if_ndis= if_ndis _igb= igb _iir= iir _io= io _ipmi= ipmi +.if ${MK_OFED} != "no" || defined(ALL_MODULES) +_ipoib= ipoib +.endif _ips= ips _ipw= ipw .if ${MK_SOURCELESS_UCODE} != "no" _ipwfw= ipwfw .endif _isci= isci _iwi= iwi .if ${MK_SOURCELESS_UCODE} != "no" _iwifw= iwifw .endif _iwn= iwn .if ${MK_SOURCELESS_UCODE} != "no" _iwnfw= iwnfw .endif _ixgb= ixgb _ixgbe= ixgbe _lindev= lindev _linprocfs= linprocfs _linsysfs= linsysfs _linux= linux _mly= mly .if ${MK_OFED} != "no" || defined(ALL_MODULES) _mlx4= mlx4 _mlx4ib= mlx4ib _mlxen= mlxen _mthca= mthca .endif _ndis= ndis _nfe= nfe _nvd= nvd .if ${MK_SOURCELESS_HOST} != "no" _nve= nve .endif _nvme= nvme _nvram= nvram _nxge= nxge .if ${MK_CDDL} != "no" || defined(ALL_MODULES) _opensolaris= opensolaris .endif .if ${MK_CRYPT} != "no" || defined(ALL_MODULES) _padlock= padlock .endif _pccard= pccard _qlxge= qlxge _qlxgb= qlxgb _qlxgbe= qlxgbe _rdma= rdma _s3= s3 _safe= safe _scsi_low= scsi_low _sfxge= sfxge _smbfs= smbfs _sound= sound _speaker= speaker _splash= splash _sppp= sppp _tpm= tpm _twa= twa _vesa= vesa _viawd= viawd _virtio= virtio _vxge= vxge _x86bios= x86bios _xenhvm= xenhvm _wbwd= wbwd _wi= wi _wpi= wpi .if ${MK_SOURCELESS_UCODE} != "no" _wpifw= wpifw .endif .if ${MK_ZFS} != "no" || defined(ALL_MODULES) _zfs= zfs .endif .endif .if ${MACHINE_CPUARCH} == "ia64" _aac= aac _aacraid= aacraid _aic= aic _an= an _arcnet= arcnet _asr= asr _bktr= bktr _cardbus= cardbus _cbb= cbb _ciss= ciss _cm= cm _cmx= cmx _coff= coff _cpufreq= cpufreq _dpt= dpt _em= em _ep= ep _et= et _exca= exca _fe= fe _hptiop= hptiop _ida= ida _igb= igb _iir= iir _ips= ips _mly= mly _pccard= pccard _scsi_low= scsi_low _smbfs= smbfs _sound= sound _splash= splash _sppp= sppp _streams= streams _tpm= tpm _twa= twa _wi= wi _xe= xe .endif .if ${MACHINE_CPUARCH} == "powerpc" _agp= agp _an= an _bm= bm _cpufreq= cpufreq _nvram= powermac_nvram _smbfs= smbfs _sound= sound .endif .if ${MACHINE_ARCH} == "powerpc64" .if ${MK_CDDL} != "no" || defined(ALL_MODULES) _opensolaris= opensolaris .endif .if ${MK_ZFS} != "no" || defined(ALL_MODULES) _zfs= zfs .endif .endif .if ${MACHINE_CPUARCH} == "sparc64" _auxio= auxio _em= em _epic= epic _i2c= i2c _igb= igb .if ${MK_CDDL} != "no" || defined(ALL_MODULES) _opensolaris= opensolaris .endif _smbfs= smbfs _sound= sound .if ${MK_ZFS} != "no" || defined(ALL_MODULES) _zfs= zfs .endif .endif .if defined(MODULES_OVERRIDE) && !defined(ALL_MODULES) SUBDIR=${MODULES_OVERRIDE} .endif .for reject in ${WITHOUT_MODULES} SUBDIR:= ${SUBDIR:N${reject}} .endfor # Calling kldxref(8) for each module is expensive. .if !defined(NO_XREF) .MAKEFLAGS+= -DNO_XREF afterinstall: @if type kldxref >/dev/null 2>&1; then \ ${ECHO} kldxref ${DESTDIR}${KMODDIR}; \ kldxref ${DESTDIR}${KMODDIR}; \ fi .endif .include Index: stable/9/sys/modules/ibcore/Makefile =================================================================== --- stable/9/sys/modules/ibcore/Makefile (nonexistent) +++ stable/9/sys/modules/ibcore/Makefile (revision 273864) @@ -0,0 +1,23 @@ +# $FreeBSD$ +.PATH: ${.CURDIR}/../../ofed/drivers/infiniband/core +.PATH: ${.CURDIR}/../../ofed/include/linux + +.include + +KMOD = ibcore +SRCS = addr.c cm_msgs.h iwcm.c mad_rmpp.h sa_query.c ucma.c uverbs_cmd.c +SRCS+= agent.c local_sa.c iwcm.h multicast.c smi.c ud_header.c uverbs_main.c +SRCS+= agent.h core_priv.h mad.c notice.c smi.h umem.c uverbs_marshall.c +SRCS+= cache.c device.c mad_priv.h packer.c sysfs.c user_mad.c verbs.c +SRCS+= cm.c fmr_pool.c mad_rmpp.c sa.h ucm.c uverbs.h cma.c +SRCS+= linux_compat.c linux_radix.c linux_idr.c +SRCS+= vnode_if.h device_if.h bus_if.h pci_if.h opt_inet.h + +CFLAGS+= -I${.CURDIR}/../../ofed/drivers/infiniband/core +CFLAGS+= -I${.CURDIR}/../mlx4ib +CFLAGS+= -I${.CURDIR}/../../ofed/include/ +CFLAGS+= -DINET6 -DINET -DOFED + +.include + +CFLAGS+= -Wno-cast-qual -Wno-pointer-arith -fms-extensions Property changes on: stable/9/sys/modules/ibcore/Makefile ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: stable/9/sys/modules/ipoib/Makefile =================================================================== --- stable/9/sys/modules/ipoib/Makefile (nonexistent) +++ stable/9/sys/modules/ipoib/Makefile (revision 273864) @@ -0,0 +1,31 @@ +# $FreeBSD$ +.PATH: ${.CURDIR}/../../ofed/drivers/infiniband/ulp/ipoib +.PATH: ${.CURDIR}/../../ofed/include/linux + +.include + +KMOD = ipoib +SRCS = device_if.h bus_if.h opt_ofed.h vnode_if.h opt_inet.h opt_inet6.h +SRCS += ipoib_cm.c ipoib_ib.c ipoib_main.c ipoib_multicast.c ipoib_verbs.c ipoib.h +SRCS+= linux_compat.c linux_radix.c linux_idr.c + +CFLAGS+= -I${.CURDIR}/../../ofed/drivers/infiniband/ulp/ipoib +CFLAGS+= -I${.CURDIR}/../ibcore +CFLAGS+= -I${.CURDIR}/../../ofed/include/ +CFLAGS+= -DINET6 -DINET -DOFED + +.if !defined(KERNBUILDDIR) +.if ${MK_INET_SUPPORT} != "no" +opt_inet.h: + @echo "#define INET 1" > ${.TARGET} +.endif + +.if ${MK_INET6_SUPPORT} != "no" +opt_inet6.h: + @echo "#define INET6 1" > ${.TARGET} +.endif +.endif + +.include + +CFLAGS+= -Wno-cast-qual -Wno-pointer-arith -fms-extensions Property changes on: stable/9/sys/modules/ipoib/Makefile ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: stable/9/sys/modules/mlx4/Makefile =================================================================== --- stable/9/sys/modules/mlx4/Makefile (revision 273863) +++ stable/9/sys/modules/mlx4/Makefile (revision 273864) @@ -1,29 +1,30 @@ # $FreeBSD$ .PATH: ${.CURDIR}/../../ofed/drivers/net/mlx4 .include +.PATH: ${.CURDIR}/../../ofed/include/linux KMOD = mlx4 -SRCS = device_if.h bus_if.h pci_if.h vnode_if.h -SRCS+= alloc.c catas.c cmd.c cq.c eq.c fw.c icm.c intf.c main.c mcg.c mr.c -SRCS+= pd.c port.c profile.c qp.c reset.c sense.c srq.c xrcd.c -SRCS+= opt_inet.h opt_inet6.h +SRCS = device_if.h bus_if.h pci_if.h vnode_if.h opt_inet.h opt_inet6.h +SRCS+= alloc.c catas.c cmd.c cq.c eq.c fw.c icm.c intf.c main.c mcg.c mr.c linux_compat.c linux_radix.c linux_idr.c +SRCS+= pd.c port.c profile.c qp.c reset.c sense.c srq.c resource_tracker.c sys_tune.c CFLAGS+= -I${.CURDIR}/../../ofed/drivers/net/mlx4 CFLAGS+= -I${.CURDIR}/../../ofed/include/ .if !defined(KERNBUILDDIR) .if ${MK_INET_SUPPORT} != "no" opt_inet.h: @echo "#define INET 1" > ${.TARGET} .endif .if ${MK_INET6_SUPPORT} != "no" opt_inet6.h: @echo "#define INET6 1" > ${.TARGET} .endif .endif .include CFLAGS+= -Wno-cast-qual -Wno-pointer-arith ${GCC_MS_EXTENSIONS} + Index: stable/9/sys/modules/mlx4ib/Makefile =================================================================== --- stable/9/sys/modules/mlx4ib/Makefile (revision 273863) +++ stable/9/sys/modules/mlx4ib/Makefile (revision 273864) @@ -1,27 +1,35 @@ # $FreeBSD$ .PATH: ${.CURDIR}/../../ofed/drivers/infiniband/hw/mlx4 +.PATH: ${.CURDIR}/../../ofed/include/linux .include KMOD = mlx4ib SRCS = device_if.h bus_if.h pci_if.h vnode_if.h -SRCS+= ah.c cq.c doorbell.c mad.c main.c mr.c qp.c srq.c wc.c +SRCS+= linux_compat.c linux_radix.c linux_idr.c +SRCS+= alias_GUID.c mcg.c sysfs.c ah.c cq.c doorbell.c mad.c main.c mr.c qp.c srq.c wc.c cm.c SRCS+= opt_inet.h opt_inet6.h +#CFLAGS+= -I${.CURDIR}/../../ofed/include/ +#CFLAGS+= -I${.CURDIR}/../../../../include +CFLAGS+= -I${.CURDIR}/../../ofed/drivers/infiniband/hw/mlx4 CFLAGS+= -I${.CURDIR}/../../ofed/include/ +CFLAGS+= -DCONFIG_INFINIBAND_USER_MEM +CFLAGS+= -DINET6 -DINET -DOFED +CFLAGS+= -fms-extensions .if !defined(KERNBUILDDIR) .if ${MK_INET_SUPPORT} != "no" opt_inet.h: @echo "#define INET 1" > ${.TARGET} .endif .if ${MK_INET6_SUPPORT} != "no" opt_inet6.h: @echo "#define INET6 1" > ${.TARGET} .endif .endif .include CFLAGS+= -Wno-cast-qual -Wno-pointer-arith ${GCC_MS_EXTENSIONS} Index: stable/9/sys/modules/mlxen/Makefile =================================================================== --- stable/9/sys/modules/mlxen/Makefile (revision 273863) +++ stable/9/sys/modules/mlxen/Makefile (revision 273864) @@ -1,28 +1,29 @@ # $FreeBSD$ .PATH: ${.CURDIR}/../../ofed/drivers/net/mlx4 .include KMOD = mlxen SRCS = device_if.h bus_if.h pci_if.h vnode_if.h -SRCS += en_cq.c en_frag.c en_main.c en_netdev.c en_port.c en_resources.c -SRCS += en_rx.c en_tx.c +SRCS += en_cq.c en_main.c en_netdev.c en_port.c en_resources.c +SRCS += en_rx.c en_tx.c utils.c SRCS += opt_inet.h opt_inet6.h CFLAGS+= -I${.CURDIR}/../../ofed/drivers/net/mlx4 CFLAGS+= -I${.CURDIR}/../../ofed/include/ +CFLAGS+= -fms-extensions .if !defined(KERNBUILDDIR) .if ${MK_INET_SUPPORT} != "no" opt_inet.h: @echo "#define INET 1" > ${.TARGET} .endif .if ${MK_INET6_SUPPORT} != "no" opt_inet6.h: @echo "#define INET6 1" > ${.TARGET} .endif .endif .include CFLAGS+= -Wno-cast-qual -Wno-pointer-arith ${GCC_MS_EXTENSIONS} Index: stable/9/sys/modules/rdma/Makefile =================================================================== --- stable/9/sys/modules/rdma/Makefile (revision 273863) +++ stable/9/sys/modules/rdma/Makefile (revision 273864) @@ -1,9 +1,9 @@ # $FreeBSD$ -SUBDIR= addr -SUBDIR+= cma -SUBDIR+= iwcm -SUBDIR+= core +#SUBDIR= addr +#SUBDIR+= cma +#SUBDIR+= iwcm +#SUBDIR+= core SUBDIR+= krping .include Index: stable/9/sys/modules =================================================================== --- stable/9/sys/modules (revision 273863) +++ stable/9/sys/modules (revision 273864) Property changes on: stable/9/sys/modules ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/sys/modules:r254735,255932 Index: stable/9/sys/ofed/drivers/infiniband/core/addr.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/addr.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/addr.c (revision 273864) @@ -1,645 +1,641 @@ /* * Copyright (c) 2005 Voltaire Inc. All rights reserved. * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved. * Copyright (c) 2005 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include -#include -#include #include #include -#include -#include #include MODULE_AUTHOR("Sean Hefty"); MODULE_DESCRIPTION("IB Address Translation"); MODULE_LICENSE("Dual BSD/GPL"); struct addr_req { struct list_head list; struct sockaddr_storage src_addr; struct sockaddr_storage dst_addr; struct rdma_dev_addr *addr; struct rdma_addr_client *client; void *context; void (*callback)(int status, struct sockaddr *src_addr, struct rdma_dev_addr *addr, void *context); unsigned long timeout; int status; }; static void process_req(struct work_struct *work); static DEFINE_MUTEX(lock); static LIST_HEAD(req_list); static struct delayed_work work; static struct workqueue_struct *addr_wq; void rdma_addr_register_client(struct rdma_addr_client *client) { atomic_set(&client->refcount, 1); init_completion(&client->comp); } EXPORT_SYMBOL(rdma_addr_register_client); static inline void put_client(struct rdma_addr_client *client) { if (atomic_dec_and_test(&client->refcount)) complete(&client->comp); } void rdma_addr_unregister_client(struct rdma_addr_client *client) { put_client(client); wait_for_completion(&client->comp); } EXPORT_SYMBOL(rdma_addr_unregister_client); #ifdef __linux__ int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct net_device *dev, const unsigned char *dst_dev_addr) { dev_addr->dev_type = dev->type; memcpy(dev_addr->src_dev_addr, dev->dev_addr, MAX_ADDR_LEN); memcpy(dev_addr->broadcast, dev->broadcast, MAX_ADDR_LEN); if (dst_dev_addr) memcpy(dev_addr->dst_dev_addr, dst_dev_addr, MAX_ADDR_LEN); dev_addr->bound_dev_if = dev->ifindex; return 0; } #else int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct ifnet *dev, const unsigned char *dst_dev_addr) { if (dev->if_type == IFT_INFINIBAND) dev_addr->dev_type = ARPHRD_INFINIBAND; else if (dev->if_type == IFT_ETHER) dev_addr->dev_type = ARPHRD_ETHER; else dev_addr->dev_type = 0; memcpy(dev_addr->src_dev_addr, IF_LLADDR(dev), dev->if_addrlen); memcpy(dev_addr->broadcast, __DECONST(char *, dev->if_broadcastaddr), dev->if_addrlen); if (dst_dev_addr) memcpy(dev_addr->dst_dev_addr, dst_dev_addr, dev->if_addrlen); dev_addr->bound_dev_if = dev->if_index; return 0; } #endif EXPORT_SYMBOL(rdma_copy_addr); int rdma_translate_ip(struct sockaddr *addr, struct rdma_dev_addr *dev_addr) { struct net_device *dev; int ret = -EADDRNOTAVAIL; if (dev_addr->bound_dev_if) { dev = dev_get_by_index(&init_net, dev_addr->bound_dev_if); if (!dev) return -ENODEV; ret = rdma_copy_addr(dev_addr, dev, NULL); dev_put(dev); return ret; } switch (addr->sa_family) { #ifdef INET case AF_INET: dev = ip_dev_find(NULL, ((struct sockaddr_in *) addr)->sin_addr.s_addr); if (!dev) return ret; ret = rdma_copy_addr(dev_addr, dev, NULL); dev_put(dev); break; #endif #if defined(INET6) case AF_INET6: #ifdef __linux__ read_lock(&dev_base_lock); for_each_netdev(&init_net, dev) { if (ipv6_chk_addr(&init_net, &((struct sockaddr_in6 *) addr)->sin6_addr, dev, 1)) { ret = rdma_copy_addr(dev_addr, dev, NULL); break; } } read_unlock(&dev_base_lock); #else { struct sockaddr_in6 *sin6; struct ifaddr *ifa; in_port_t port; sin6 = (struct sockaddr_in6 *)addr; port = sin6->sin6_port; sin6->sin6_port = 0; ifa = ifa_ifwithaddr(addr); sin6->sin6_port = port; if (ifa == NULL) { ret = -ENODEV; break; } ret = rdma_copy_addr(dev_addr, ifa->ifa_ifp, NULL); ifa_free(ifa); break; } #endif break; #endif } return ret; } EXPORT_SYMBOL(rdma_translate_ip); static void set_timeout(unsigned long time) { unsigned long delay; cancel_delayed_work(&work); delay = time - jiffies; if ((long)delay <= 0) delay = 1; queue_delayed_work(addr_wq, &work, delay); } static void queue_req(struct addr_req *req) { struct addr_req *temp_req; mutex_lock(&lock); list_for_each_entry_reverse(temp_req, &req_list, list) { if (time_after_eq(req->timeout, temp_req->timeout)) break; } list_add(&req->list, &temp_req->list); if (req_list.next == &req->list) set_timeout(req->timeout); mutex_unlock(&lock); } #ifdef __linux__ static int addr4_resolve(struct sockaddr_in *src_in, struct sockaddr_in *dst_in, struct rdma_dev_addr *addr) { __be32 src_ip = src_in->sin_addr.s_addr; __be32 dst_ip = dst_in->sin_addr.s_addr; struct flowi fl; struct rtable *rt; struct neighbour *neigh; int ret; memset(&fl, 0, sizeof fl); fl.nl_u.ip4_u.daddr = dst_ip; fl.nl_u.ip4_u.saddr = src_ip; fl.oif = addr->bound_dev_if; ret = ip_route_output_key(&init_net, &rt, &fl); if (ret) goto out; src_in->sin_family = AF_INET; src_in->sin_addr.s_addr = rt->rt_src; if (rt->idev->dev->flags & IFF_LOOPBACK) { ret = rdma_translate_ip((struct sockaddr *) dst_in, addr); if (!ret) memcpy(addr->dst_dev_addr, addr->src_dev_addr, MAX_ADDR_LEN); goto put; } /* If the device does ARP internally, return 'done' */ if (rt->idev->dev->flags & IFF_NOARP) { rdma_copy_addr(addr, rt->idev->dev, NULL); goto put; } neigh = neigh_lookup(&arp_tbl, &rt->rt_gateway, rt->idev->dev); if (!neigh || !(neigh->nud_state & NUD_VALID)) { neigh_event_send(rt->u.dst.neighbour, NULL); ret = -ENODATA; if (neigh) goto release; goto put; } ret = rdma_copy_addr(addr, neigh->dev, neigh->ha); release: neigh_release(neigh); put: ip_rt_put(rt); out: return ret; } #if defined(INET6) static int addr6_resolve(struct sockaddr_in6 *src_in, struct sockaddr_in6 *dst_in, struct rdma_dev_addr *addr) { struct flowi fl; struct neighbour *neigh; struct dst_entry *dst; int ret; memset(&fl, 0, sizeof fl); ipv6_addr_copy(&fl.fl6_dst, &dst_in->sin6_addr); ipv6_addr_copy(&fl.fl6_src, &src_in->sin6_addr); fl.oif = addr->bound_dev_if; dst = ip6_route_output(&init_net, NULL, &fl); if ((ret = dst->error)) goto put; if (ipv6_addr_any(&fl.fl6_src)) { ret = ipv6_dev_get_saddr(&init_net, ip6_dst_idev(dst)->dev, &fl.fl6_dst, 0, &fl.fl6_src); if (ret) goto put; src_in->sin6_family = AF_INET6; ipv6_addr_copy(&src_in->sin6_addr, &fl.fl6_src); } if (dst->dev->flags & IFF_LOOPBACK) { ret = rdma_translate_ip((struct sockaddr *) dst_in, addr); if (!ret) memcpy(addr->dst_dev_addr, addr->src_dev_addr, MAX_ADDR_LEN); goto put; } /* If the device does ARP internally, return 'done' */ if (dst->dev->flags & IFF_NOARP) { ret = rdma_copy_addr(addr, dst->dev, NULL); goto put; } neigh = dst->neighbour; if (!neigh || !(neigh->nud_state & NUD_VALID)) { neigh_event_send(dst->neighbour, NULL); ret = -ENODATA; goto put; } ret = rdma_copy_addr(addr, dst->dev, neigh->ha); put: dst_release(dst); return ret; } #else static int addr6_resolve(struct sockaddr_in6 *src_in, struct sockaddr_in6 *dst_in, struct rdma_dev_addr *addr) { return -EADDRNOTAVAIL; } #endif #else #include static int addr_resolve(struct sockaddr *src_in, struct sockaddr *dst_in, struct rdma_dev_addr *addr) { struct sockaddr_in *sin; struct sockaddr_in6 *sin6; struct ifaddr *ifa; struct ifnet *ifp; #if defined(INET) || defined(INET6) struct llentry *lle; #endif struct rtentry *rte; in_port_t port; u_char edst[MAX_ADDR_LEN]; int multi; int bcast; - int error; + int error = 0; /* * Determine whether the address is unicast, multicast, or broadcast * and whether the source interface is valid. */ multi = 0; bcast = 0; sin = NULL; sin6 = NULL; ifp = NULL; rte = NULL; switch (dst_in->sa_family) { #ifdef INET case AF_INET: sin = (struct sockaddr_in *)dst_in; if (sin->sin_addr.s_addr == INADDR_BROADCAST) bcast = 1; if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) multi = 1; sin = (struct sockaddr_in *)src_in; if (sin->sin_addr.s_addr != INADDR_ANY) { /* * Address comparison fails if the port is set * cache it here to be restored later. */ port = sin->sin_port; sin->sin_port = 0; memset(&sin->sin_zero, 0, sizeof(sin->sin_zero)); } else src_in = NULL; break; #endif #ifdef INET6 case AF_INET6: sin6 = (struct sockaddr_in6 *)dst_in; if (IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) multi = 1; sin6 = (struct sockaddr_in6 *)src_in; if (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) { port = sin6->sin6_port; sin6->sin6_port = 0; } else src_in = NULL; break; #endif default: return -EINVAL; } /* * If we have a source address to use look it up first and verify * that it is a local interface. */ if (src_in) { ifa = ifa_ifwithaddr(src_in); if (sin) sin->sin_port = port; if (sin6) sin6->sin6_port = port; if (ifa == NULL) return -ENETUNREACH; ifp = ifa->ifa_ifp; ifa_free(ifa); if (bcast || multi) goto mcast; } /* * Make sure the route exists and has a valid link. */ rte = rtalloc1(dst_in, 1, 0); if (rte == NULL || rte->rt_ifp == NULL || !RT_LINK_IS_UP(rte->rt_ifp)) { if (rte) RTFREE_LOCKED(rte); return -EHOSTUNREACH; } /* * If it's not multicast or broadcast and the route doesn't match the * requested interface return unreachable. Otherwise fetch the * correct interface pointer and unlock the route. */ if (multi || bcast) { if (ifp == NULL) ifp = rte->rt_ifp; RTFREE_LOCKED(rte); } else if (ifp && ifp != rte->rt_ifp) { RTFREE_LOCKED(rte); return -ENETUNREACH; } else { if (ifp == NULL) ifp = rte->rt_ifp; RT_UNLOCK(rte); } mcast: if (bcast) return rdma_copy_addr(addr, ifp, ifp->if_broadcastaddr); if (multi) { struct sockaddr *llsa; error = ifp->if_resolvemulti(ifp, &llsa, dst_in); if (error) return -error; error = rdma_copy_addr(addr, ifp, LLADDR((struct sockaddr_dl *)llsa)); free(llsa, M_IFMADDR); return error; } /* * Resolve the link local address. */ switch (dst_in->sa_family) { #ifdef INET case AF_INET: error = arpresolve(ifp, rte, NULL, dst_in, edst, &lle); break; #endif #ifdef INET6 case AF_INET6: error = nd6_storelladdr(ifp, NULL, dst_in, (u_char *)edst, &lle); break; #endif default: /* XXX: Shouldn't happen. */ error = -EINVAL; } RTFREE(rte); if (error == 0) return rdma_copy_addr(addr, ifp, edst); if (error == EWOULDBLOCK) return -ENODATA; return -error; } #endif static void process_req(struct work_struct *work) { struct addr_req *req, *temp_req; struct sockaddr *src_in, *dst_in; struct list_head done_list; INIT_LIST_HEAD(&done_list); mutex_lock(&lock); list_for_each_entry_safe(req, temp_req, &req_list, list) { if (req->status == -ENODATA) { src_in = (struct sockaddr *) &req->src_addr; dst_in = (struct sockaddr *) &req->dst_addr; req->status = addr_resolve(src_in, dst_in, req->addr); if (req->status && time_after_eq(jiffies, req->timeout)) req->status = -ETIMEDOUT; else if (req->status == -ENODATA) continue; } list_move_tail(&req->list, &done_list); } if (!list_empty(&req_list)) { req = list_entry(req_list.next, struct addr_req, list); set_timeout(req->timeout); } mutex_unlock(&lock); list_for_each_entry_safe(req, temp_req, &done_list, list) { list_del(&req->list); req->callback(req->status, (struct sockaddr *) &req->src_addr, req->addr, req->context); put_client(req->client); kfree(req); } } int rdma_resolve_ip(struct rdma_addr_client *client, struct sockaddr *src_addr, struct sockaddr *dst_addr, struct rdma_dev_addr *addr, int timeout_ms, void (*callback)(int status, struct sockaddr *src_addr, struct rdma_dev_addr *addr, void *context), void *context) { struct sockaddr *src_in, *dst_in; struct addr_req *req; int ret = 0; req = kzalloc(sizeof *req, GFP_KERNEL); if (!req) return -ENOMEM; src_in = (struct sockaddr *) &req->src_addr; dst_in = (struct sockaddr *) &req->dst_addr; if (src_addr) { if (src_addr->sa_family != dst_addr->sa_family) { ret = -EINVAL; goto err; } memcpy(src_in, src_addr, ip_addr_size(src_addr)); } else { src_in->sa_family = dst_addr->sa_family; } memcpy(dst_in, dst_addr, ip_addr_size(dst_addr)); req->addr = addr; req->callback = callback; req->context = context; req->client = client; atomic_inc(&client->refcount); req->status = addr_resolve(src_in, dst_in, addr); switch (req->status) { case 0: req->timeout = jiffies; queue_req(req); break; case -ENODATA: req->timeout = msecs_to_jiffies(timeout_ms) + jiffies; queue_req(req); break; default: ret = req->status; atomic_dec(&client->refcount); goto err; } return ret; err: kfree(req); return ret; } EXPORT_SYMBOL(rdma_resolve_ip); void rdma_addr_cancel(struct rdma_dev_addr *addr) { struct addr_req *req, *temp_req; mutex_lock(&lock); list_for_each_entry_safe(req, temp_req, &req_list, list) { if (req->addr == addr) { req->status = -ECANCELED; req->timeout = jiffies; list_move(&req->list, &req_list); set_timeout(req->timeout); break; } } mutex_unlock(&lock); } EXPORT_SYMBOL(rdma_addr_cancel); static int netevent_callback(struct notifier_block *self, unsigned long event, void *ctx) { if (event == NETEVENT_NEIGH_UPDATE) { #ifdef __linux__ struct neighbour *neigh = ctx; if (neigh->nud_state & NUD_VALID) { set_timeout(jiffies); } #else set_timeout(jiffies); #endif } return 0; } static struct notifier_block nb = { .notifier_call = netevent_callback }; static int addr_init(void) { INIT_DELAYED_WORK(&work, process_req); addr_wq = create_singlethread_workqueue("ib_addr"); if (!addr_wq) return -ENOMEM; register_netevent_notifier(&nb); return 0; } static void addr_cleanup(void) { unregister_netevent_notifier(&nb); destroy_workqueue(addr_wq); } module_init(addr_init); module_exit(addr_cleanup); Index: stable/9/sys/ofed/drivers/infiniband/core/cm.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/cm.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/cm.c (revision 273864) @@ -1,3894 +1,3897 @@ /* * Copyright (c) 2004-2007 Intel Corporation. All rights reserved. * Copyright (c) 2004 Topspin Corporation. All rights reserved. * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include +#include +#include + #include #include #include "cm_msgs.h" MODULE_AUTHOR("Sean Hefty"); MODULE_DESCRIPTION("InfiniBand CM"); MODULE_LICENSE("Dual BSD/GPL"); #define PFX "ib_cm: " /* * Limit CM message timeouts to something reasonable: * 8 seconds per message, with up to 15 retries */ static int max_timeout = 21; module_param(max_timeout, int, 0644); MODULE_PARM_DESC(max_timeout, "Maximum IB CM per message timeout " "(default=21, or ~8 seconds)"); static void cm_add_one(struct ib_device *device); static void cm_remove_one(struct ib_device *device); static struct ib_client cm_client = { .name = "cm", .add = cm_add_one, .remove = cm_remove_one }; static struct ib_cm { spinlock_t lock; struct list_head device_list; rwlock_t device_lock; struct rb_root listen_service_table; u64 listen_service_id; /* struct rb_root peer_service_table; todo: fix peer to peer */ struct rb_root remote_qp_table; struct rb_root remote_id_table; struct rb_root remote_sidr_table; struct idr local_id_table; __be32 random_id_operand; struct list_head timewait_list; struct workqueue_struct *wq; } cm; /* Counter indexes ordered by attribute ID */ enum { CM_REQ_COUNTER, CM_MRA_COUNTER, CM_REJ_COUNTER, CM_REP_COUNTER, CM_RTU_COUNTER, CM_DREQ_COUNTER, CM_DREP_COUNTER, CM_SIDR_REQ_COUNTER, CM_SIDR_REP_COUNTER, CM_LAP_COUNTER, CM_APR_COUNTER, CM_ATTR_COUNT, CM_ATTR_ID_OFFSET = 0x0010, }; enum { CM_XMIT, CM_XMIT_RETRIES, CM_RECV, CM_RECV_DUPLICATES, CM_COUNTER_GROUPS }; static char const counter_group_names[CM_COUNTER_GROUPS] [sizeof("cm_rx_duplicates")] = { "cm_tx_msgs", "cm_tx_retries", "cm_rx_msgs", "cm_rx_duplicates" }; struct cm_counter_group { struct kobject obj; atomic_long_t counter[CM_ATTR_COUNT]; }; struct cm_counter_attribute { struct attribute attr; int index; }; #define CM_COUNTER_ATTR(_name, _index) \ struct cm_counter_attribute cm_##_name##_counter_attr = { \ .attr = { .name = __stringify(_name), .mode = 0444 }, \ .index = _index \ } static CM_COUNTER_ATTR(req, CM_REQ_COUNTER); static CM_COUNTER_ATTR(mra, CM_MRA_COUNTER); static CM_COUNTER_ATTR(rej, CM_REJ_COUNTER); static CM_COUNTER_ATTR(rep, CM_REP_COUNTER); static CM_COUNTER_ATTR(rtu, CM_RTU_COUNTER); static CM_COUNTER_ATTR(dreq, CM_DREQ_COUNTER); static CM_COUNTER_ATTR(drep, CM_DREP_COUNTER); static CM_COUNTER_ATTR(sidr_req, CM_SIDR_REQ_COUNTER); static CM_COUNTER_ATTR(sidr_rep, CM_SIDR_REP_COUNTER); static CM_COUNTER_ATTR(lap, CM_LAP_COUNTER); static CM_COUNTER_ATTR(apr, CM_APR_COUNTER); static struct attribute *cm_counter_default_attrs[] = { &cm_req_counter_attr.attr, &cm_mra_counter_attr.attr, &cm_rej_counter_attr.attr, &cm_rep_counter_attr.attr, &cm_rtu_counter_attr.attr, &cm_dreq_counter_attr.attr, &cm_drep_counter_attr.attr, &cm_sidr_req_counter_attr.attr, &cm_sidr_rep_counter_attr.attr, &cm_lap_counter_attr.attr, &cm_apr_counter_attr.attr, NULL }; struct cm_port { struct cm_device *cm_dev; struct ib_mad_agent *mad_agent; struct kobject port_obj; u8 port_num; struct cm_counter_group counter_group[CM_COUNTER_GROUPS]; }; struct cm_device { struct list_head list; struct ib_device *ib_device; struct device *device; u8 ack_delay; struct cm_port *port[0]; }; struct cm_av { struct cm_port *port; union ib_gid dgid; struct ib_ah_attr ah_attr; u16 pkey_index; u8 timeout; }; struct cm_work { struct delayed_work work; struct list_head list; struct cm_port *port; struct ib_mad_recv_wc *mad_recv_wc; /* Received MADs */ __be32 local_id; /* Established / timewait */ __be32 remote_id; struct ib_cm_event cm_event; struct ib_sa_path_rec path[0]; }; struct cm_timewait_info { struct cm_work work; /* Must be first. */ struct list_head list; struct rb_node remote_qp_node; struct rb_node remote_id_node; __be64 remote_ca_guid; __be32 remote_qpn; u8 inserted_remote_qp; u8 inserted_remote_id; }; struct cm_id_private { struct ib_cm_id id; struct rb_node service_node; struct rb_node sidr_id_node; spinlock_t lock; /* Do not acquire inside cm.lock */ struct completion comp; atomic_t refcount; struct ib_mad_send_buf *msg; struct cm_timewait_info *timewait_info; /* todo: use alternate port on send failure */ struct cm_av av; struct cm_av alt_av; struct ib_cm_compare_data *compare_data; void *private_data; __be64 tid; __be32 local_qpn; __be32 remote_qpn; enum ib_qp_type qp_type; __be32 sq_psn; __be32 rq_psn; int timeout_ms; enum ib_mtu path_mtu; __be16 pkey; u8 private_data_len; u8 max_cm_retries; u8 peer_to_peer; u8 responder_resources; u8 initiator_depth; u8 retry_count; u8 rnr_retry_count; u8 service_timeout; u8 target_ack_delay; struct list_head work_list; atomic_t work_count; }; static void cm_work_handler(struct work_struct *work); static inline void cm_deref_id(struct cm_id_private *cm_id_priv) { if (atomic_dec_and_test(&cm_id_priv->refcount)) complete(&cm_id_priv->comp); } static int cm_alloc_msg(struct cm_id_private *cm_id_priv, struct ib_mad_send_buf **msg) { struct ib_mad_agent *mad_agent; struct ib_mad_send_buf *m; struct ib_ah *ah; mad_agent = cm_id_priv->av.port->mad_agent; ah = ib_create_ah(mad_agent->qp->pd, &cm_id_priv->av.ah_attr); if (IS_ERR(ah)) return PTR_ERR(ah); m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn, cm_id_priv->av.pkey_index, 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA, GFP_ATOMIC); if (IS_ERR(m)) { ib_destroy_ah(ah); return PTR_ERR(m); } /* Timeout set by caller if response is expected. */ m->ah = ah; m->retries = cm_id_priv->max_cm_retries; atomic_inc(&cm_id_priv->refcount); m->context[0] = cm_id_priv; *msg = m; return 0; } static int cm_alloc_response_msg(struct cm_port *port, struct ib_mad_recv_wc *mad_recv_wc, struct ib_mad_send_buf **msg) { struct ib_mad_send_buf *m; struct ib_ah *ah; ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc, mad_recv_wc->recv_buf.grh, port->port_num); if (IS_ERR(ah)) return PTR_ERR(ah); m = ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index, 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA, GFP_ATOMIC); if (IS_ERR(m)) { ib_destroy_ah(ah); return PTR_ERR(m); } m->ah = ah; *msg = m; return 0; } static void cm_free_msg(struct ib_mad_send_buf *msg) { ib_destroy_ah(msg->ah); if (msg->context[0]) cm_deref_id(msg->context[0]); ib_free_send_mad(msg); } static void * cm_copy_private_data(const void *private_data, u8 private_data_len) { void *data; if (!private_data || !private_data_len) return NULL; data = kmemdup(private_data, private_data_len, GFP_KERNEL); if (!data) return ERR_PTR(-ENOMEM); return data; } static void cm_set_private_data(struct cm_id_private *cm_id_priv, void *private_data, u8 private_data_len) { if (cm_id_priv->private_data && cm_id_priv->private_data_len) kfree(cm_id_priv->private_data); cm_id_priv->private_data = private_data; cm_id_priv->private_data_len = private_data_len; } static void cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc, struct ib_grh *grh, struct cm_av *av) { av->port = port; av->pkey_index = wc->pkey_index; ib_init_ah_from_wc(port->cm_dev->ib_device, port->port_num, wc, grh, &av->ah_attr); } static int cm_init_av_by_path(struct ib_sa_path_rec *path, struct cm_av *av) { struct cm_device *cm_dev; struct cm_port *port = NULL; unsigned long flags; int ret; u8 p; read_lock_irqsave(&cm.device_lock, flags); list_for_each_entry(cm_dev, &cm.device_list, list) { if (!ib_find_cached_gid(cm_dev->ib_device, &path->sgid, &p, NULL)) { port = cm_dev->port[p-1]; break; } } read_unlock_irqrestore(&cm.device_lock, flags); if (!port) return -EINVAL; ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num, be16_to_cpu(path->pkey), &av->pkey_index); if (ret) return ret; av->port = port; ib_init_ah_from_path(cm_dev->ib_device, port->port_num, path, &av->ah_attr); av->timeout = path->packet_life_time + 1; return 0; } static int cm_alloc_id(struct cm_id_private *cm_id_priv) { unsigned long flags; int ret, id; static int next_id; do { spin_lock_irqsave(&cm.lock, flags); ret = idr_get_new_above(&cm.local_id_table, cm_id_priv, next_id, &id); if (!ret) next_id = ((unsigned) id + 1) & MAX_ID_MASK; spin_unlock_irqrestore(&cm.lock, flags); } while( (ret == -EAGAIN) && idr_pre_get(&cm.local_id_table, GFP_KERNEL) ); cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand; return ret; } static void cm_free_id(__be32 local_id) { spin_lock_irq(&cm.lock); idr_remove(&cm.local_id_table, (__force int) (local_id ^ cm.random_id_operand)); spin_unlock_irq(&cm.lock); } static struct cm_id_private * cm_get_id(__be32 local_id, __be32 remote_id) { struct cm_id_private *cm_id_priv; cm_id_priv = idr_find(&cm.local_id_table, (__force int) (local_id ^ cm.random_id_operand)); if (cm_id_priv) { if (cm_id_priv->id.remote_id == remote_id) atomic_inc(&cm_id_priv->refcount); else cm_id_priv = NULL; } return cm_id_priv; } static struct cm_id_private * cm_acquire_id(__be32 local_id, __be32 remote_id) { struct cm_id_private *cm_id_priv; spin_lock_irq(&cm.lock); cm_id_priv = cm_get_id(local_id, remote_id); spin_unlock_irq(&cm.lock); return cm_id_priv; } static void cm_mask_copy(u8 *dst, u8 *src, u8 *mask) { int i; for (i = 0; i < IB_CM_COMPARE_SIZE / sizeof(unsigned long); i++) ((unsigned long *) dst)[i] = ((unsigned long *) src)[i] & ((unsigned long *) mask)[i]; } static int cm_compare_data(struct ib_cm_compare_data *src_data, struct ib_cm_compare_data *dst_data) { u8 src[IB_CM_COMPARE_SIZE]; u8 dst[IB_CM_COMPARE_SIZE]; if (!src_data || !dst_data) return 0; cm_mask_copy(src, src_data->data, dst_data->mask); cm_mask_copy(dst, dst_data->data, src_data->mask); return memcmp(src, dst, IB_CM_COMPARE_SIZE); } static int cm_compare_private_data(u8 *private_data, struct ib_cm_compare_data *dst_data) { u8 src[IB_CM_COMPARE_SIZE]; if (!dst_data) return 0; cm_mask_copy(src, private_data, dst_data->mask); return memcmp(src, dst_data->data, IB_CM_COMPARE_SIZE); } /* * Trivial helpers to strip endian annotation and compare; the * endianness doesn't actually matter since we just need a stable * order for the RB tree. */ static int be32_lt(__be32 a, __be32 b) { return (__force u32) a < (__force u32) b; } static int be32_gt(__be32 a, __be32 b) { return (__force u32) a > (__force u32) b; } static int be64_lt(__be64 a, __be64 b) { return (__force u64) a < (__force u64) b; } static int be64_gt(__be64 a, __be64 b) { return (__force u64) a > (__force u64) b; } static struct cm_id_private * cm_insert_listen(struct cm_id_private *cm_id_priv) { struct rb_node **link = &cm.listen_service_table.rb_node; struct rb_node *parent = NULL; struct cm_id_private *cur_cm_id_priv; __be64 service_id = cm_id_priv->id.service_id; __be64 service_mask = cm_id_priv->id.service_mask; int data_cmp; while (*link) { parent = *link; cur_cm_id_priv = rb_entry(parent, struct cm_id_private, service_node); data_cmp = cm_compare_data(cm_id_priv->compare_data, cur_cm_id_priv->compare_data); if ((cur_cm_id_priv->id.service_mask & service_id) == (service_mask & cur_cm_id_priv->id.service_id) && (cm_id_priv->id.device == cur_cm_id_priv->id.device) && !data_cmp) return cur_cm_id_priv; if (cm_id_priv->id.device < cur_cm_id_priv->id.device) link = &(*link)->rb_left; else if (cm_id_priv->id.device > cur_cm_id_priv->id.device) link = &(*link)->rb_right; else if (be64_lt(service_id, cur_cm_id_priv->id.service_id)) link = &(*link)->rb_left; else if (be64_gt(service_id, cur_cm_id_priv->id.service_id)) link = &(*link)->rb_right; else if (data_cmp < 0) link = &(*link)->rb_left; else link = &(*link)->rb_right; } rb_link_node(&cm_id_priv->service_node, parent, link); rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table); return NULL; } static struct cm_id_private * cm_find_listen(struct ib_device *device, __be64 service_id, u8 *private_data) { struct rb_node *node = cm.listen_service_table.rb_node; struct cm_id_private *cm_id_priv; int data_cmp; while (node) { cm_id_priv = rb_entry(node, struct cm_id_private, service_node); data_cmp = cm_compare_private_data(private_data, cm_id_priv->compare_data); if ((cm_id_priv->id.service_mask & service_id) == cm_id_priv->id.service_id && (cm_id_priv->id.device == device) && !data_cmp) return cm_id_priv; if (device < cm_id_priv->id.device) node = node->rb_left; else if (device > cm_id_priv->id.device) node = node->rb_right; else if (be64_lt(service_id, cm_id_priv->id.service_id)) node = node->rb_left; else if (be64_gt(service_id, cm_id_priv->id.service_id)) node = node->rb_right; else if (data_cmp < 0) node = node->rb_left; else node = node->rb_right; } return NULL; } static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info *timewait_info) { struct rb_node **link = &cm.remote_id_table.rb_node; struct rb_node *parent = NULL; struct cm_timewait_info *cur_timewait_info; __be64 remote_ca_guid = timewait_info->remote_ca_guid; __be32 remote_id = timewait_info->work.remote_id; while (*link) { parent = *link; cur_timewait_info = rb_entry(parent, struct cm_timewait_info, remote_id_node); if (be32_lt(remote_id, cur_timewait_info->work.remote_id)) link = &(*link)->rb_left; else if (be32_gt(remote_id, cur_timewait_info->work.remote_id)) link = &(*link)->rb_right; else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid)) link = &(*link)->rb_left; else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid)) link = &(*link)->rb_right; else return cur_timewait_info; } timewait_info->inserted_remote_id = 1; rb_link_node(&timewait_info->remote_id_node, parent, link); rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table); return NULL; } static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid, __be32 remote_id) { struct rb_node *node = cm.remote_id_table.rb_node; struct cm_timewait_info *timewait_info; while (node) { timewait_info = rb_entry(node, struct cm_timewait_info, remote_id_node); if (be32_lt(remote_id, timewait_info->work.remote_id)) node = node->rb_left; else if (be32_gt(remote_id, timewait_info->work.remote_id)) node = node->rb_right; else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid)) node = node->rb_left; else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid)) node = node->rb_right; else return timewait_info; } return NULL; } static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info *timewait_info) { struct rb_node **link = &cm.remote_qp_table.rb_node; struct rb_node *parent = NULL; struct cm_timewait_info *cur_timewait_info; __be64 remote_ca_guid = timewait_info->remote_ca_guid; __be32 remote_qpn = timewait_info->remote_qpn; while (*link) { parent = *link; cur_timewait_info = rb_entry(parent, struct cm_timewait_info, remote_qp_node); if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn)) link = &(*link)->rb_left; else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn)) link = &(*link)->rb_right; else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid)) link = &(*link)->rb_left; else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid)) link = &(*link)->rb_right; else return cur_timewait_info; } timewait_info->inserted_remote_qp = 1; rb_link_node(&timewait_info->remote_qp_node, parent, link); rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table); return NULL; } static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private *cm_id_priv) { struct rb_node **link = &cm.remote_sidr_table.rb_node; struct rb_node *parent = NULL; struct cm_id_private *cur_cm_id_priv; union ib_gid *port_gid = &cm_id_priv->av.dgid; __be32 remote_id = cm_id_priv->id.remote_id; while (*link) { parent = *link; cur_cm_id_priv = rb_entry(parent, struct cm_id_private, sidr_id_node); if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id)) link = &(*link)->rb_left; else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id)) link = &(*link)->rb_right; else { int cmp; cmp = memcmp(port_gid, &cur_cm_id_priv->av.dgid, sizeof *port_gid); if (cmp < 0) link = &(*link)->rb_left; else if (cmp > 0) link = &(*link)->rb_right; else return cur_cm_id_priv; } } rb_link_node(&cm_id_priv->sidr_id_node, parent, link); rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table); return NULL; } static void cm_reject_sidr_req(struct cm_id_private *cm_id_priv, enum ib_cm_sidr_status status) { struct ib_cm_sidr_rep_param param; memset(¶m, 0, sizeof param); param.status = status; ib_send_cm_sidr_rep(&cm_id_priv->id, ¶m); } struct ib_cm_id *ib_create_cm_id(struct ib_device *device, ib_cm_handler cm_handler, void *context) { struct cm_id_private *cm_id_priv; int ret; cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL); if (!cm_id_priv) return ERR_PTR(-ENOMEM); cm_id_priv->id.state = IB_CM_IDLE; cm_id_priv->id.device = device; cm_id_priv->id.cm_handler = cm_handler; cm_id_priv->id.context = context; cm_id_priv->id.remote_cm_qpn = 1; ret = cm_alloc_id(cm_id_priv); if (ret) goto error; spin_lock_init(&cm_id_priv->lock); init_completion(&cm_id_priv->comp); INIT_LIST_HEAD(&cm_id_priv->work_list); atomic_set(&cm_id_priv->work_count, -1); atomic_set(&cm_id_priv->refcount, 1); return &cm_id_priv->id; error: kfree(cm_id_priv); return ERR_PTR(-ENOMEM); } EXPORT_SYMBOL(ib_create_cm_id); static struct cm_work * cm_dequeue_work(struct cm_id_private *cm_id_priv) { struct cm_work *work; if (list_empty(&cm_id_priv->work_list)) return NULL; work = list_entry(cm_id_priv->work_list.next, struct cm_work, list); list_del(&work->list); return work; } static void cm_free_work(struct cm_work *work) { if (work->mad_recv_wc) ib_free_recv_mad(work->mad_recv_wc); kfree(work); } static inline int cm_convert_to_ms(int iba_time) { /* approximate conversion to ms from 4.096us x 2^iba_time */ return 1 << max(iba_time - 8, 0); } /* * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time * Because of how ack_timeout is stored, adding one doubles the timeout. * To avoid large timeouts, select the max(ack_delay, life_time + 1), and * increment it (round up) only if the other is within 50%. */ static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time) { int ack_timeout = packet_life_time + 1; if (ack_timeout >= ca_ack_delay) ack_timeout += (ca_ack_delay >= (ack_timeout - 1)); else ack_timeout = ca_ack_delay + (ack_timeout >= (ca_ack_delay - 1)); return min(31, ack_timeout); } static void cm_cleanup_timewait(struct cm_timewait_info *timewait_info) { if (timewait_info->inserted_remote_id) { rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table); timewait_info->inserted_remote_id = 0; } if (timewait_info->inserted_remote_qp) { rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table); timewait_info->inserted_remote_qp = 0; } } static struct cm_timewait_info * cm_create_timewait_info(__be32 local_id) { struct cm_timewait_info *timewait_info; timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL); if (!timewait_info) return ERR_PTR(-ENOMEM); timewait_info->work.local_id = local_id; INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler); timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT; return timewait_info; } static void cm_enter_timewait(struct cm_id_private *cm_id_priv) { int wait_time; unsigned long flags; spin_lock_irqsave(&cm.lock, flags); cm_cleanup_timewait(cm_id_priv->timewait_info); list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list); spin_unlock_irqrestore(&cm.lock, flags); /* * The cm_id could be destroyed by the user before we exit timewait. * To protect against this, we search for the cm_id after exiting * timewait before notifying the user that we've exited timewait. */ cm_id_priv->id.state = IB_CM_TIMEWAIT; wait_time = cm_convert_to_ms(cm_id_priv->av.timeout); queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work, msecs_to_jiffies(wait_time)); cm_id_priv->timewait_info = NULL; } static void cm_reset_to_idle(struct cm_id_private *cm_id_priv) { unsigned long flags; cm_id_priv->id.state = IB_CM_IDLE; if (cm_id_priv->timewait_info) { spin_lock_irqsave(&cm.lock, flags); cm_cleanup_timewait(cm_id_priv->timewait_info); spin_unlock_irqrestore(&cm.lock, flags); kfree(cm_id_priv->timewait_info); cm_id_priv->timewait_info = NULL; } } static void cm_destroy_id(struct ib_cm_id *cm_id, int err) { struct cm_id_private *cm_id_priv; struct cm_work *work; cm_id_priv = container_of(cm_id, struct cm_id_private, id); retest: spin_lock_irq(&cm_id_priv->lock); switch (cm_id->state) { case IB_CM_LISTEN: cm_id->state = IB_CM_IDLE; spin_unlock_irq(&cm_id_priv->lock); spin_lock_irq(&cm.lock); rb_erase(&cm_id_priv->service_node, &cm.listen_service_table); spin_unlock_irq(&cm.lock); break; case IB_CM_SIDR_REQ_SENT: cm_id->state = IB_CM_IDLE; ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); spin_unlock_irq(&cm_id_priv->lock); break; case IB_CM_SIDR_REQ_RCVD: spin_unlock_irq(&cm_id_priv->lock); cm_reject_sidr_req(cm_id_priv, IB_SIDR_REJECT); break; case IB_CM_REQ_SENT: ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); spin_unlock_irq(&cm_id_priv->lock); ib_send_cm_rej(cm_id, IB_CM_REJ_TIMEOUT, &cm_id_priv->id.device->node_guid, sizeof cm_id_priv->id.device->node_guid, NULL, 0); break; case IB_CM_REQ_RCVD: if (err == -ENOMEM) { /* Do not reject to allow future retries. */ cm_reset_to_idle(cm_id_priv); spin_unlock_irq(&cm_id_priv->lock); } else { spin_unlock_irq(&cm_id_priv->lock); ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0, NULL, 0); } break; case IB_CM_MRA_REQ_RCVD: case IB_CM_REP_SENT: case IB_CM_MRA_REP_RCVD: ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); /* Fall through */ case IB_CM_MRA_REQ_SENT: case IB_CM_REP_RCVD: case IB_CM_MRA_REP_SENT: spin_unlock_irq(&cm_id_priv->lock); ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0, NULL, 0); break; case IB_CM_ESTABLISHED: spin_unlock_irq(&cm_id_priv->lock); ib_send_cm_dreq(cm_id, NULL, 0); goto retest; case IB_CM_DREQ_SENT: ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); cm_enter_timewait(cm_id_priv); spin_unlock_irq(&cm_id_priv->lock); break; case IB_CM_DREQ_RCVD: spin_unlock_irq(&cm_id_priv->lock); ib_send_cm_drep(cm_id, NULL, 0); break; default: spin_unlock_irq(&cm_id_priv->lock); break; } cm_free_id(cm_id->local_id); cm_deref_id(cm_id_priv); wait_for_completion(&cm_id_priv->comp); while ((work = cm_dequeue_work(cm_id_priv)) != NULL) cm_free_work(work); kfree(cm_id_priv->compare_data); kfree(cm_id_priv->private_data); kfree(cm_id_priv); } void ib_destroy_cm_id(struct ib_cm_id *cm_id) { cm_destroy_id(cm_id, 0); } EXPORT_SYMBOL(ib_destroy_cm_id); int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask, struct ib_cm_compare_data *compare_data) { struct cm_id_private *cm_id_priv, *cur_cm_id_priv; unsigned long flags; int ret = 0; service_mask = service_mask ? service_mask : ~cpu_to_be64(0); service_id &= service_mask; if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID && (service_id != IB_CM_ASSIGN_SERVICE_ID)) return -EINVAL; cm_id_priv = container_of(cm_id, struct cm_id_private, id); if (cm_id->state != IB_CM_IDLE) return -EINVAL; if (compare_data) { cm_id_priv->compare_data = kzalloc(sizeof *compare_data, GFP_KERNEL); if (!cm_id_priv->compare_data) return -ENOMEM; cm_mask_copy(cm_id_priv->compare_data->data, compare_data->data, compare_data->mask); memcpy(cm_id_priv->compare_data->mask, compare_data->mask, IB_CM_COMPARE_SIZE); } cm_id->state = IB_CM_LISTEN; spin_lock_irqsave(&cm.lock, flags); if (service_id == IB_CM_ASSIGN_SERVICE_ID) { cm_id->service_id = cpu_to_be64(cm.listen_service_id++); cm_id->service_mask = ~cpu_to_be64(0); } else { cm_id->service_id = service_id; cm_id->service_mask = service_mask; } cur_cm_id_priv = cm_insert_listen(cm_id_priv); spin_unlock_irqrestore(&cm.lock, flags); if (cur_cm_id_priv) { cm_id->state = IB_CM_IDLE; kfree(cm_id_priv->compare_data); cm_id_priv->compare_data = NULL; ret = -EBUSY; } return ret; } EXPORT_SYMBOL(ib_cm_listen); static __be64 cm_form_tid(struct cm_id_private *cm_id_priv, enum cm_msg_sequence msg_seq) { u64 hi_tid, low_tid; hi_tid = ((u64) cm_id_priv->av.port->mad_agent->hi_tid) << 32; low_tid = (u64) ((__force u32)cm_id_priv->id.local_id | (msg_seq << 30)); return cpu_to_be64(hi_tid | low_tid); } static void cm_format_mad_hdr(struct ib_mad_hdr *hdr, __be16 attr_id, __be64 tid) { hdr->base_version = IB_MGMT_BASE_VERSION; hdr->mgmt_class = IB_MGMT_CLASS_CM; hdr->class_version = IB_CM_CLASS_VERSION; hdr->method = IB_MGMT_METHOD_SEND; hdr->attr_id = attr_id; hdr->tid = tid; } static void cm_format_req(struct cm_req_msg *req_msg, struct cm_id_private *cm_id_priv, struct ib_cm_req_param *param) { struct ib_sa_path_rec *pri_path = param->primary_path; struct ib_sa_path_rec *alt_path = param->alternate_path; cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID, cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_REQ)); req_msg->local_comm_id = cm_id_priv->id.local_id; req_msg->service_id = param->service_id; req_msg->local_ca_guid = cm_id_priv->id.device->node_guid; cm_req_set_local_qpn(req_msg, cpu_to_be32(param->qp_num)); cm_req_set_resp_res(req_msg, param->responder_resources); cm_req_set_init_depth(req_msg, param->initiator_depth); cm_req_set_remote_resp_timeout(req_msg, param->remote_cm_response_timeout); if (param->remote_cm_response_timeout > (u8) max_timeout) { printk(KERN_WARNING PFX "req remote_cm_response_timeout %d > " "%d, decreasing\n", param->remote_cm_response_timeout, max_timeout); cm_req_set_remote_resp_timeout(req_msg, (u8) max_timeout); } cm_req_set_qp_type(req_msg, param->qp_type); cm_req_set_flow_ctrl(req_msg, param->flow_control); cm_req_set_starting_psn(req_msg, cpu_to_be32(param->starting_psn)); cm_req_set_local_resp_timeout(req_msg, param->local_cm_response_timeout); if (param->local_cm_response_timeout > (u8) max_timeout) { printk(KERN_WARNING PFX "req local_cm_response_timeout %d > " "%d, decreasing\n", param->local_cm_response_timeout, max_timeout); cm_req_set_local_resp_timeout(req_msg, (u8) max_timeout); } cm_req_set_retry_count(req_msg, param->retry_count); req_msg->pkey = param->primary_path->pkey; cm_req_set_path_mtu(req_msg, param->primary_path->mtu); cm_req_set_rnr_retry_count(req_msg, param->rnr_retry_count); cm_req_set_max_cm_retries(req_msg, param->max_cm_retries); cm_req_set_srq(req_msg, param->srq); if (pri_path->hop_limit <= 1) { req_msg->primary_local_lid = pri_path->slid; req_msg->primary_remote_lid = pri_path->dlid; } else { /* Work-around until there's a way to obtain remote LID info */ req_msg->primary_local_lid = IB_LID_PERMISSIVE; req_msg->primary_remote_lid = IB_LID_PERMISSIVE; } req_msg->primary_local_gid = pri_path->sgid; req_msg->primary_remote_gid = pri_path->dgid; cm_req_set_primary_flow_label(req_msg, pri_path->flow_label); cm_req_set_primary_packet_rate(req_msg, pri_path->rate); req_msg->primary_traffic_class = pri_path->traffic_class; req_msg->primary_hop_limit = pri_path->hop_limit; cm_req_set_primary_sl(req_msg, pri_path->sl); cm_req_set_primary_subnet_local(req_msg, (pri_path->hop_limit <= 1)); cm_req_set_primary_local_ack_timeout(req_msg, cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay, pri_path->packet_life_time)); if (alt_path) { if (alt_path->hop_limit <= 1) { req_msg->alt_local_lid = alt_path->slid; req_msg->alt_remote_lid = alt_path->dlid; } else { req_msg->alt_local_lid = IB_LID_PERMISSIVE; req_msg->alt_remote_lid = IB_LID_PERMISSIVE; } req_msg->alt_local_gid = alt_path->sgid; req_msg->alt_remote_gid = alt_path->dgid; cm_req_set_alt_flow_label(req_msg, alt_path->flow_label); cm_req_set_alt_packet_rate(req_msg, alt_path->rate); req_msg->alt_traffic_class = alt_path->traffic_class; req_msg->alt_hop_limit = alt_path->hop_limit; cm_req_set_alt_sl(req_msg, alt_path->sl); cm_req_set_alt_subnet_local(req_msg, (alt_path->hop_limit <= 1)); cm_req_set_alt_local_ack_timeout(req_msg, cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay, alt_path->packet_life_time)); } if (param->private_data && param->private_data_len) memcpy(req_msg->private_data, param->private_data, param->private_data_len); } static int cm_validate_req_param(struct ib_cm_req_param *param) { /* peer-to-peer not supported */ if (param->peer_to_peer) return -EINVAL; if (!param->primary_path) return -EINVAL; if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC) return -EINVAL; if (param->private_data && param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE) return -EINVAL; if (param->alternate_path && (param->alternate_path->pkey != param->primary_path->pkey || param->alternate_path->mtu != param->primary_path->mtu)) return -EINVAL; return 0; } int ib_send_cm_req(struct ib_cm_id *cm_id, struct ib_cm_req_param *param) { struct cm_id_private *cm_id_priv; struct cm_req_msg *req_msg; unsigned long flags; int ret; ret = cm_validate_req_param(param); if (ret) return ret; /* Verify that we're not in timewait. */ cm_id_priv = container_of(cm_id, struct cm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state != IB_CM_IDLE) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); ret = -EINVAL; goto out; } spin_unlock_irqrestore(&cm_id_priv->lock, flags); cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv-> id.local_id); if (IS_ERR(cm_id_priv->timewait_info)) { ret = PTR_ERR(cm_id_priv->timewait_info); goto out; } ret = cm_init_av_by_path(param->primary_path, &cm_id_priv->av); if (ret) goto error1; if (param->alternate_path) { ret = cm_init_av_by_path(param->alternate_path, &cm_id_priv->alt_av); if (ret) goto error1; } cm_id->service_id = param->service_id; cm_id->service_mask = ~cpu_to_be64(0); cm_id_priv->timeout_ms = cm_convert_to_ms( param->primary_path->packet_life_time) * 2 + cm_convert_to_ms( param->remote_cm_response_timeout); if (cm_id_priv->timeout_ms > cm_convert_to_ms(max_timeout)) { printk(KERN_WARNING PFX "req timeout_ms %d > %d, decreasing\n", cm_id_priv->timeout_ms, cm_convert_to_ms(max_timeout)); cm_id_priv->timeout_ms = cm_convert_to_ms(max_timeout); } cm_id_priv->max_cm_retries = param->max_cm_retries; cm_id_priv->initiator_depth = param->initiator_depth; cm_id_priv->responder_resources = param->responder_resources; cm_id_priv->retry_count = param->retry_count; cm_id_priv->path_mtu = param->primary_path->mtu; cm_id_priv->pkey = param->primary_path->pkey; cm_id_priv->qp_type = param->qp_type; ret = cm_alloc_msg(cm_id_priv, &cm_id_priv->msg); if (ret) goto error1; req_msg = (struct cm_req_msg *) cm_id_priv->msg->mad; cm_format_req(req_msg, cm_id_priv, param); cm_id_priv->tid = req_msg->hdr.tid; cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms; cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT; cm_id_priv->local_qpn = cm_req_get_local_qpn(req_msg); cm_id_priv->rq_psn = cm_req_get_starting_psn(req_msg); spin_lock_irqsave(&cm_id_priv->lock, flags); ret = ib_post_send_mad(cm_id_priv->msg, NULL); if (ret) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); goto error2; } BUG_ON(cm_id->state != IB_CM_IDLE); cm_id->state = IB_CM_REQ_SENT; spin_unlock_irqrestore(&cm_id_priv->lock, flags); return 0; error2: cm_free_msg(cm_id_priv->msg); error1: kfree(cm_id_priv->timewait_info); out: return ret; } EXPORT_SYMBOL(ib_send_cm_req); static int cm_issue_rej(struct cm_port *port, struct ib_mad_recv_wc *mad_recv_wc, enum ib_cm_rej_reason reason, enum cm_msg_response msg_rejected, void *ari, u8 ari_length) { struct ib_mad_send_buf *msg = NULL; struct cm_rej_msg *rej_msg, *rcv_msg; int ret; ret = cm_alloc_response_msg(port, mad_recv_wc, &msg); if (ret) return ret; /* We just need common CM header information. Cast to any message. */ rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad; rej_msg = (struct cm_rej_msg *) msg->mad; cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid); rej_msg->remote_comm_id = rcv_msg->local_comm_id; rej_msg->local_comm_id = rcv_msg->remote_comm_id; cm_rej_set_msg_rejected(rej_msg, msg_rejected); rej_msg->reason = cpu_to_be16(reason); if (ari && ari_length) { cm_rej_set_reject_info_len(rej_msg, ari_length); memcpy(rej_msg->ari, ari, ari_length); } ret = ib_post_send_mad(msg, NULL); if (ret) cm_free_msg(msg); return ret; } static inline int cm_is_active_peer(__be64 local_ca_guid, __be64 remote_ca_guid, __be32 local_qpn, __be32 remote_qpn) { return (be64_to_cpu(local_ca_guid) > be64_to_cpu(remote_ca_guid) || ((local_ca_guid == remote_ca_guid) && (be32_to_cpu(local_qpn) > be32_to_cpu(remote_qpn)))); } static void cm_format_paths_from_req(struct cm_req_msg *req_msg, struct ib_sa_path_rec *primary_path, struct ib_sa_path_rec *alt_path) { memset(primary_path, 0, sizeof *primary_path); primary_path->dgid = req_msg->primary_local_gid; primary_path->sgid = req_msg->primary_remote_gid; primary_path->dlid = req_msg->primary_local_lid; primary_path->slid = req_msg->primary_remote_lid; primary_path->flow_label = cm_req_get_primary_flow_label(req_msg); primary_path->hop_limit = req_msg->primary_hop_limit; primary_path->traffic_class = req_msg->primary_traffic_class; primary_path->reversible = 1; primary_path->pkey = req_msg->pkey; primary_path->sl = cm_req_get_primary_sl(req_msg); primary_path->mtu_selector = IB_SA_EQ; primary_path->mtu = cm_req_get_path_mtu(req_msg); primary_path->rate_selector = IB_SA_EQ; primary_path->rate = cm_req_get_primary_packet_rate(req_msg); primary_path->packet_life_time_selector = IB_SA_EQ; primary_path->packet_life_time = cm_req_get_primary_local_ack_timeout(req_msg); primary_path->packet_life_time -= (primary_path->packet_life_time > 0); if (req_msg->alt_local_lid) { memset(alt_path, 0, sizeof *alt_path); alt_path->dgid = req_msg->alt_local_gid; alt_path->sgid = req_msg->alt_remote_gid; alt_path->dlid = req_msg->alt_local_lid; alt_path->slid = req_msg->alt_remote_lid; alt_path->flow_label = cm_req_get_alt_flow_label(req_msg); alt_path->hop_limit = req_msg->alt_hop_limit; alt_path->traffic_class = req_msg->alt_traffic_class; alt_path->reversible = 1; alt_path->pkey = req_msg->pkey; alt_path->sl = cm_req_get_alt_sl(req_msg); alt_path->mtu_selector = IB_SA_EQ; alt_path->mtu = cm_req_get_path_mtu(req_msg); alt_path->rate_selector = IB_SA_EQ; alt_path->rate = cm_req_get_alt_packet_rate(req_msg); alt_path->packet_life_time_selector = IB_SA_EQ; alt_path->packet_life_time = cm_req_get_alt_local_ack_timeout(req_msg); alt_path->packet_life_time -= (alt_path->packet_life_time > 0); } } static void cm_format_req_event(struct cm_work *work, struct cm_id_private *cm_id_priv, struct ib_cm_id *listen_id) { struct cm_req_msg *req_msg; struct ib_cm_req_event_param *param; req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.req_rcvd; param->listen_id = listen_id; param->port = cm_id_priv->av.port->port_num; param->primary_path = &work->path[0]; if (req_msg->alt_local_lid) param->alternate_path = &work->path[1]; else param->alternate_path = NULL; param->remote_ca_guid = req_msg->local_ca_guid; param->remote_qkey = be32_to_cpu(req_msg->local_qkey); param->remote_qpn = be32_to_cpu(cm_req_get_local_qpn(req_msg)); param->qp_type = cm_req_get_qp_type(req_msg); param->starting_psn = be32_to_cpu(cm_req_get_starting_psn(req_msg)); param->responder_resources = cm_req_get_init_depth(req_msg); param->initiator_depth = cm_req_get_resp_res(req_msg); param->local_cm_response_timeout = cm_req_get_remote_resp_timeout(req_msg); param->flow_control = cm_req_get_flow_ctrl(req_msg); param->remote_cm_response_timeout = cm_req_get_local_resp_timeout(req_msg); param->retry_count = cm_req_get_retry_count(req_msg); param->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg); param->srq = cm_req_get_srq(req_msg); work->cm_event.private_data = &req_msg->private_data; } static void cm_process_work(struct cm_id_private *cm_id_priv, struct cm_work *work) { int ret; /* We will typically only have the current event to report. */ ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event); cm_free_work(work); while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) { spin_lock_irq(&cm_id_priv->lock); work = cm_dequeue_work(cm_id_priv); spin_unlock_irq(&cm_id_priv->lock); BUG_ON(!work); ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event); cm_free_work(work); } cm_deref_id(cm_id_priv); if (ret) cm_destroy_id(&cm_id_priv->id, ret); } static void cm_format_mra(struct cm_mra_msg *mra_msg, struct cm_id_private *cm_id_priv, enum cm_msg_response msg_mraed, u8 service_timeout, const void *private_data, u8 private_data_len) { cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid); cm_mra_set_msg_mraed(mra_msg, msg_mraed); mra_msg->local_comm_id = cm_id_priv->id.local_id; mra_msg->remote_comm_id = cm_id_priv->id.remote_id; cm_mra_set_service_timeout(mra_msg, service_timeout); if (private_data && private_data_len) memcpy(mra_msg->private_data, private_data, private_data_len); } static void cm_format_rej(struct cm_rej_msg *rej_msg, struct cm_id_private *cm_id_priv, enum ib_cm_rej_reason reason, void *ari, u8 ari_length, const void *private_data, u8 private_data_len) { cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid); rej_msg->remote_comm_id = cm_id_priv->id.remote_id; switch(cm_id_priv->id.state) { case IB_CM_REQ_RCVD: rej_msg->local_comm_id = 0; cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ); break; case IB_CM_MRA_REQ_SENT: rej_msg->local_comm_id = cm_id_priv->id.local_id; cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ); break; case IB_CM_REP_RCVD: case IB_CM_MRA_REP_SENT: rej_msg->local_comm_id = cm_id_priv->id.local_id; cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REP); break; default: rej_msg->local_comm_id = cm_id_priv->id.local_id; cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_OTHER); break; } rej_msg->reason = cpu_to_be16(reason); if (ari && ari_length) { cm_rej_set_reject_info_len(rej_msg, ari_length); memcpy(rej_msg->ari, ari, ari_length); } if (private_data && private_data_len) memcpy(rej_msg->private_data, private_data, private_data_len); } static void cm_dup_req_handler(struct cm_work *work, struct cm_id_private *cm_id_priv) { struct ib_mad_send_buf *msg = NULL; int ret; atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_REQ_COUNTER]); /* Quick state check to discard duplicate REQs. */ if (cm_id_priv->id.state == IB_CM_REQ_RCVD) return; ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg); if (ret) return; spin_lock_irq(&cm_id_priv->lock); switch (cm_id_priv->id.state) { case IB_CM_MRA_REQ_SENT: cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout, cm_id_priv->private_data, cm_id_priv->private_data_len); break; case IB_CM_TIMEWAIT: cm_format_rej((struct cm_rej_msg *) msg->mad, cm_id_priv, IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0); break; default: goto unlock; } spin_unlock_irq(&cm_id_priv->lock); ret = ib_post_send_mad(msg, NULL); if (ret) goto free; return; unlock: spin_unlock_irq(&cm_id_priv->lock); free: cm_free_msg(msg); } static struct cm_id_private * cm_match_req(struct cm_work *work, struct cm_id_private *cm_id_priv) { struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv; struct cm_timewait_info *timewait_info; struct cm_req_msg *req_msg; req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad; /* Check for possible duplicate REQ. */ spin_lock_irq(&cm.lock); timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info); if (timewait_info) { cur_cm_id_priv = cm_get_id(timewait_info->work.local_id, timewait_info->work.remote_id); spin_unlock_irq(&cm.lock); if (cur_cm_id_priv) { cm_dup_req_handler(work, cur_cm_id_priv); cm_deref_id(cur_cm_id_priv); } return NULL; } /* Check for stale connections. */ timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info); if (timewait_info) { cm_cleanup_timewait(cm_id_priv->timewait_info); spin_unlock_irq(&cm.lock); cm_issue_rej(work->port, work->mad_recv_wc, IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ, NULL, 0); return NULL; } /* Find matching listen request. */ listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device, req_msg->service_id, req_msg->private_data); if (!listen_cm_id_priv) { cm_cleanup_timewait(cm_id_priv->timewait_info); spin_unlock_irq(&cm.lock); cm_issue_rej(work->port, work->mad_recv_wc, IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ, NULL, 0); goto out; } atomic_inc(&listen_cm_id_priv->refcount); atomic_inc(&cm_id_priv->refcount); cm_id_priv->id.state = IB_CM_REQ_RCVD; atomic_inc(&cm_id_priv->work_count); spin_unlock_irq(&cm.lock); out: return listen_cm_id_priv; } /* * Work-around for inter-subnet connections. If the LIDs are permissive, * we need to override the LID/SL data in the REQ with the LID information * in the work completion. */ static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc) { if (!cm_req_get_primary_subnet_local(req_msg)) { if (req_msg->primary_local_lid == IB_LID_PERMISSIVE) { req_msg->primary_local_lid = cpu_to_be16(wc->slid); cm_req_set_primary_sl(req_msg, wc->sl); } if (req_msg->primary_remote_lid == IB_LID_PERMISSIVE) req_msg->primary_remote_lid = cpu_to_be16(wc->dlid_path_bits); } if (!cm_req_get_alt_subnet_local(req_msg)) { if (req_msg->alt_local_lid == IB_LID_PERMISSIVE) { req_msg->alt_local_lid = cpu_to_be16(wc->slid); cm_req_set_alt_sl(req_msg, wc->sl); } if (req_msg->alt_remote_lid == IB_LID_PERMISSIVE) req_msg->alt_remote_lid = cpu_to_be16(wc->dlid_path_bits); } } static int cm_req_handler(struct cm_work *work) { struct ib_cm_id *cm_id; struct cm_id_private *cm_id_priv, *listen_cm_id_priv; struct cm_req_msg *req_msg; int ret; req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad; cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL); if (IS_ERR(cm_id)) return PTR_ERR(cm_id); cm_id_priv = container_of(cm_id, struct cm_id_private, id); cm_id_priv->id.remote_id = req_msg->local_comm_id; cm_init_av_for_response(work->port, work->mad_recv_wc->wc, work->mad_recv_wc->recv_buf.grh, &cm_id_priv->av); cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv-> id.local_id); if (IS_ERR(cm_id_priv->timewait_info)) { ret = PTR_ERR(cm_id_priv->timewait_info); goto destroy; } cm_id_priv->timewait_info->work.remote_id = req_msg->local_comm_id; cm_id_priv->timewait_info->remote_ca_guid = req_msg->local_ca_guid; cm_id_priv->timewait_info->remote_qpn = cm_req_get_local_qpn(req_msg); listen_cm_id_priv = cm_match_req(work, cm_id_priv); if (!listen_cm_id_priv) { ret = -EINVAL; kfree(cm_id_priv->timewait_info); goto destroy; } cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler; cm_id_priv->id.context = listen_cm_id_priv->id.context; cm_id_priv->id.service_id = req_msg->service_id; cm_id_priv->id.service_mask = ~cpu_to_be64(0); cm_process_routed_req(req_msg, work->mad_recv_wc->wc); cm_format_paths_from_req(req_msg, &work->path[0], &work->path[1]); ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av); if (ret) { ib_get_cached_gid(work->port->cm_dev->ib_device, work->port->port_num, 0, &work->path[0].sgid); ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID, &work->path[0].sgid, sizeof work->path[0].sgid, NULL, 0); goto rejected; } if (req_msg->alt_local_lid) { ret = cm_init_av_by_path(&work->path[1], &cm_id_priv->alt_av); if (ret) { ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_ALT_GID, &work->path[0].sgid, sizeof work->path[0].sgid, NULL, 0); goto rejected; } } cm_id_priv->tid = req_msg->hdr.tid; cm_id_priv->timeout_ms = cm_convert_to_ms( cm_req_get_local_resp_timeout(req_msg)); if (cm_req_get_local_resp_timeout(req_msg) > (u8) max_timeout) { printk(KERN_WARNING PFX "rcvd cm_local_resp_timeout %d > %d, " "decreasing used timeout_ms\n", cm_req_get_local_resp_timeout(req_msg), max_timeout); cm_id_priv->timeout_ms = cm_convert_to_ms(max_timeout); } cm_id_priv->max_cm_retries = cm_req_get_max_cm_retries(req_msg); cm_id_priv->remote_qpn = cm_req_get_local_qpn(req_msg); cm_id_priv->initiator_depth = cm_req_get_resp_res(req_msg); cm_id_priv->responder_resources = cm_req_get_init_depth(req_msg); cm_id_priv->path_mtu = cm_req_get_path_mtu(req_msg); cm_id_priv->pkey = req_msg->pkey; cm_id_priv->sq_psn = cm_req_get_starting_psn(req_msg); cm_id_priv->retry_count = cm_req_get_retry_count(req_msg); cm_id_priv->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg); cm_id_priv->qp_type = cm_req_get_qp_type(req_msg); cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id); cm_process_work(cm_id_priv, work); cm_deref_id(listen_cm_id_priv); return 0; rejected: atomic_dec(&cm_id_priv->refcount); cm_deref_id(listen_cm_id_priv); destroy: ib_destroy_cm_id(cm_id); return ret; } static void cm_format_rep(struct cm_rep_msg *rep_msg, struct cm_id_private *cm_id_priv, struct ib_cm_rep_param *param) { cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid); rep_msg->local_comm_id = cm_id_priv->id.local_id; rep_msg->remote_comm_id = cm_id_priv->id.remote_id; cm_rep_set_local_qpn(rep_msg, cpu_to_be32(param->qp_num)); cm_rep_set_starting_psn(rep_msg, cpu_to_be32(param->starting_psn)); rep_msg->resp_resources = param->responder_resources; rep_msg->initiator_depth = param->initiator_depth; cm_rep_set_target_ack_delay(rep_msg, cm_id_priv->av.port->cm_dev->ack_delay); cm_rep_set_failover(rep_msg, param->failover_accepted); cm_rep_set_flow_ctrl(rep_msg, param->flow_control); cm_rep_set_rnr_retry_count(rep_msg, param->rnr_retry_count); cm_rep_set_srq(rep_msg, param->srq); rep_msg->local_ca_guid = cm_id_priv->id.device->node_guid; if (param->private_data && param->private_data_len) memcpy(rep_msg->private_data, param->private_data, param->private_data_len); } int ib_send_cm_rep(struct ib_cm_id *cm_id, struct ib_cm_rep_param *param) { struct cm_id_private *cm_id_priv; struct ib_mad_send_buf *msg; struct cm_rep_msg *rep_msg; unsigned long flags; int ret; if (param->private_data && param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE) return -EINVAL; cm_id_priv = container_of(cm_id, struct cm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state != IB_CM_REQ_RCVD && cm_id->state != IB_CM_MRA_REQ_SENT) { ret = -EINVAL; goto out; } ret = cm_alloc_msg(cm_id_priv, &msg); if (ret) goto out; rep_msg = (struct cm_rep_msg *) msg->mad; cm_format_rep(rep_msg, cm_id_priv, param); msg->timeout_ms = cm_id_priv->timeout_ms; msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT; ret = ib_post_send_mad(msg, NULL); if (ret) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); cm_free_msg(msg); return ret; } cm_id->state = IB_CM_REP_SENT; cm_id_priv->msg = msg; cm_id_priv->initiator_depth = param->initiator_depth; cm_id_priv->responder_resources = param->responder_resources; cm_id_priv->rq_psn = cm_rep_get_starting_psn(rep_msg); cm_id_priv->local_qpn = cm_rep_get_local_qpn(rep_msg); out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } EXPORT_SYMBOL(ib_send_cm_rep); static void cm_format_rtu(struct cm_rtu_msg *rtu_msg, struct cm_id_private *cm_id_priv, const void *private_data, u8 private_data_len) { cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid); rtu_msg->local_comm_id = cm_id_priv->id.local_id; rtu_msg->remote_comm_id = cm_id_priv->id.remote_id; if (private_data && private_data_len) memcpy(rtu_msg->private_data, private_data, private_data_len); } int ib_send_cm_rtu(struct ib_cm_id *cm_id, const void *private_data, u8 private_data_len) { struct cm_id_private *cm_id_priv; struct ib_mad_send_buf *msg; unsigned long flags; void *data; int ret; if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE) return -EINVAL; data = cm_copy_private_data(private_data, private_data_len); if (IS_ERR(data)) return PTR_ERR(data); cm_id_priv = container_of(cm_id, struct cm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state != IB_CM_REP_RCVD && cm_id->state != IB_CM_MRA_REP_SENT) { ret = -EINVAL; goto error; } ret = cm_alloc_msg(cm_id_priv, &msg); if (ret) goto error; cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv, private_data, private_data_len); ret = ib_post_send_mad(msg, NULL); if (ret) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); cm_free_msg(msg); kfree(data); return ret; } cm_id->state = IB_CM_ESTABLISHED; cm_set_private_data(cm_id_priv, data, private_data_len); spin_unlock_irqrestore(&cm_id_priv->lock, flags); return 0; error: spin_unlock_irqrestore(&cm_id_priv->lock, flags); kfree(data); return ret; } EXPORT_SYMBOL(ib_send_cm_rtu); static void cm_format_rep_event(struct cm_work *work) { struct cm_rep_msg *rep_msg; struct ib_cm_rep_event_param *param; rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.rep_rcvd; param->remote_ca_guid = rep_msg->local_ca_guid; param->remote_qkey = be32_to_cpu(rep_msg->local_qkey); param->remote_qpn = be32_to_cpu(cm_rep_get_local_qpn(rep_msg)); param->starting_psn = be32_to_cpu(cm_rep_get_starting_psn(rep_msg)); param->responder_resources = rep_msg->initiator_depth; param->initiator_depth = rep_msg->resp_resources; param->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg); param->failover_accepted = cm_rep_get_failover(rep_msg); param->flow_control = cm_rep_get_flow_ctrl(rep_msg); param->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg); param->srq = cm_rep_get_srq(rep_msg); work->cm_event.private_data = &rep_msg->private_data; } static void cm_dup_rep_handler(struct cm_work *work) { struct cm_id_private *cm_id_priv; struct cm_rep_msg *rep_msg; struct ib_mad_send_buf *msg = NULL; int ret; rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad; cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id, rep_msg->local_comm_id); if (!cm_id_priv) return; atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_REP_COUNTER]); ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg); if (ret) goto deref; spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state == IB_CM_ESTABLISHED) cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv, cm_id_priv->private_data, cm_id_priv->private_data_len); else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT) cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout, cm_id_priv->private_data, cm_id_priv->private_data_len); else goto unlock; spin_unlock_irq(&cm_id_priv->lock); ret = ib_post_send_mad(msg, NULL); if (ret) goto free; goto deref; unlock: spin_unlock_irq(&cm_id_priv->lock); free: cm_free_msg(msg); deref: cm_deref_id(cm_id_priv); } static int cm_rep_handler(struct cm_work *work) { struct cm_id_private *cm_id_priv; struct cm_rep_msg *rep_msg; int ret; rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad; cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id, 0); if (!cm_id_priv) { cm_dup_rep_handler(work); return -EINVAL; } cm_format_rep_event(work); spin_lock_irq(&cm_id_priv->lock); switch (cm_id_priv->id.state) { case IB_CM_REQ_SENT: case IB_CM_MRA_REQ_RCVD: break; default: spin_unlock_irq(&cm_id_priv->lock); ret = -EINVAL; goto error; } cm_id_priv->timewait_info->work.remote_id = rep_msg->local_comm_id; cm_id_priv->timewait_info->remote_ca_guid = rep_msg->local_ca_guid; cm_id_priv->timewait_info->remote_qpn = cm_rep_get_local_qpn(rep_msg); spin_lock(&cm.lock); /* Check for duplicate REP. */ if (cm_insert_remote_id(cm_id_priv->timewait_info)) { spin_unlock(&cm.lock); spin_unlock_irq(&cm_id_priv->lock); ret = -EINVAL; goto error; } /* Check for a stale connection. */ if (cm_insert_remote_qpn(cm_id_priv->timewait_info)) { rb_erase(&cm_id_priv->timewait_info->remote_id_node, &cm.remote_id_table); cm_id_priv->timewait_info->inserted_remote_id = 0; spin_unlock(&cm.lock); spin_unlock_irq(&cm_id_priv->lock); cm_issue_rej(work->port, work->mad_recv_wc, IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP, NULL, 0); ret = -EINVAL; goto error; } spin_unlock(&cm.lock); cm_id_priv->id.state = IB_CM_REP_RCVD; cm_id_priv->id.remote_id = rep_msg->local_comm_id; cm_id_priv->remote_qpn = cm_rep_get_local_qpn(rep_msg); cm_id_priv->initiator_depth = rep_msg->resp_resources; cm_id_priv->responder_resources = rep_msg->initiator_depth; cm_id_priv->sq_psn = cm_rep_get_starting_psn(rep_msg); cm_id_priv->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg); cm_id_priv->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg); cm_id_priv->av.timeout = cm_ack_timeout(cm_id_priv->target_ack_delay, cm_id_priv->av.timeout - 1); cm_id_priv->alt_av.timeout = cm_ack_timeout(cm_id_priv->target_ack_delay, cm_id_priv->alt_av.timeout - 1); /* todo: handle peer_to_peer */ ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); ret = atomic_inc_and_test(&cm_id_priv->work_count); if (!ret) list_add_tail(&work->list, &cm_id_priv->work_list); spin_unlock_irq(&cm_id_priv->lock); if (ret) cm_process_work(cm_id_priv, work); else cm_deref_id(cm_id_priv); return 0; error: cm_deref_id(cm_id_priv); return ret; } static int cm_establish_handler(struct cm_work *work) { struct cm_id_private *cm_id_priv; int ret; /* See comment in cm_establish about lookup. */ cm_id_priv = cm_acquire_id(work->local_id, work->remote_id); if (!cm_id_priv) return -EINVAL; spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_ESTABLISHED) { spin_unlock_irq(&cm_id_priv->lock); goto out; } ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); ret = atomic_inc_and_test(&cm_id_priv->work_count); if (!ret) list_add_tail(&work->list, &cm_id_priv->work_list); spin_unlock_irq(&cm_id_priv->lock); if (ret) cm_process_work(cm_id_priv, work); else cm_deref_id(cm_id_priv); return 0; out: cm_deref_id(cm_id_priv); return -EINVAL; } static int cm_rtu_handler(struct cm_work *work) { struct cm_id_private *cm_id_priv; struct cm_rtu_msg *rtu_msg; int ret; rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad; cm_id_priv = cm_acquire_id(rtu_msg->remote_comm_id, rtu_msg->local_comm_id); if (!cm_id_priv) return -EINVAL; work->cm_event.private_data = &rtu_msg->private_data; spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_REP_SENT && cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) { spin_unlock_irq(&cm_id_priv->lock); atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_RTU_COUNTER]); goto out; } cm_id_priv->id.state = IB_CM_ESTABLISHED; ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); ret = atomic_inc_and_test(&cm_id_priv->work_count); if (!ret) list_add_tail(&work->list, &cm_id_priv->work_list); spin_unlock_irq(&cm_id_priv->lock); if (ret) cm_process_work(cm_id_priv, work); else cm_deref_id(cm_id_priv); return 0; out: cm_deref_id(cm_id_priv); return -EINVAL; } static void cm_format_dreq(struct cm_dreq_msg *dreq_msg, struct cm_id_private *cm_id_priv, const void *private_data, u8 private_data_len) { cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID, cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_DREQ)); dreq_msg->local_comm_id = cm_id_priv->id.local_id; dreq_msg->remote_comm_id = cm_id_priv->id.remote_id; cm_dreq_set_remote_qpn(dreq_msg, cm_id_priv->remote_qpn); if (private_data && private_data_len) memcpy(dreq_msg->private_data, private_data, private_data_len); } int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data, u8 private_data_len) { struct cm_id_private *cm_id_priv; struct ib_mad_send_buf *msg; unsigned long flags; int ret; if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE) return -EINVAL; cm_id_priv = container_of(cm_id, struct cm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state != IB_CM_ESTABLISHED) { ret = -EINVAL; goto out; } ret = cm_alloc_msg(cm_id_priv, &msg); if (ret) { cm_enter_timewait(cm_id_priv); goto out; } cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv, private_data, private_data_len); msg->timeout_ms = cm_id_priv->timeout_ms; msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT; ret = ib_post_send_mad(msg, NULL); if (ret) { cm_enter_timewait(cm_id_priv); spin_unlock_irqrestore(&cm_id_priv->lock, flags); cm_free_msg(msg); return ret; } cm_id->state = IB_CM_DREQ_SENT; cm_id_priv->msg = msg; out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } EXPORT_SYMBOL(ib_send_cm_dreq); static void cm_format_drep(struct cm_drep_msg *drep_msg, struct cm_id_private *cm_id_priv, const void *private_data, u8 private_data_len) { cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid); drep_msg->local_comm_id = cm_id_priv->id.local_id; drep_msg->remote_comm_id = cm_id_priv->id.remote_id; if (private_data && private_data_len) memcpy(drep_msg->private_data, private_data, private_data_len); } int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data, u8 private_data_len) { struct cm_id_private *cm_id_priv; struct ib_mad_send_buf *msg; unsigned long flags; void *data; int ret; if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE) return -EINVAL; data = cm_copy_private_data(private_data, private_data_len); if (IS_ERR(data)) return PTR_ERR(data); cm_id_priv = container_of(cm_id, struct cm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state != IB_CM_DREQ_RCVD) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); kfree(data); return -EINVAL; } cm_set_private_data(cm_id_priv, data, private_data_len); cm_enter_timewait(cm_id_priv); ret = cm_alloc_msg(cm_id_priv, &msg); if (ret) goto out; cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv, private_data, private_data_len); ret = ib_post_send_mad(msg, NULL); if (ret) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); cm_free_msg(msg); return ret; } out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } EXPORT_SYMBOL(ib_send_cm_drep); static int cm_issue_drep(struct cm_port *port, struct ib_mad_recv_wc *mad_recv_wc) { struct ib_mad_send_buf *msg = NULL; struct cm_dreq_msg *dreq_msg; struct cm_drep_msg *drep_msg; int ret; ret = cm_alloc_response_msg(port, mad_recv_wc, &msg); if (ret) return ret; dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad; drep_msg = (struct cm_drep_msg *) msg->mad; cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid); drep_msg->remote_comm_id = dreq_msg->local_comm_id; drep_msg->local_comm_id = dreq_msg->remote_comm_id; ret = ib_post_send_mad(msg, NULL); if (ret) cm_free_msg(msg); return ret; } static int cm_dreq_handler(struct cm_work *work) { struct cm_id_private *cm_id_priv; struct cm_dreq_msg *dreq_msg; struct ib_mad_send_buf *msg = NULL; int ret; dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad; cm_id_priv = cm_acquire_id(dreq_msg->remote_comm_id, dreq_msg->local_comm_id); if (!cm_id_priv) { atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_DREQ_COUNTER]); cm_issue_drep(work->port, work->mad_recv_wc); return -EINVAL; } work->cm_event.private_data = &dreq_msg->private_data; spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->local_qpn != cm_dreq_get_remote_qpn(dreq_msg)) goto unlock; switch (cm_id_priv->id.state) { case IB_CM_REP_SENT: case IB_CM_DREQ_SENT: ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); break; case IB_CM_ESTABLISHED: case IB_CM_MRA_REP_RCVD: break; case IB_CM_TIMEWAIT: atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_DREQ_COUNTER]); if (cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg)) goto unlock; cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv, cm_id_priv->private_data, cm_id_priv->private_data_len); spin_unlock_irq(&cm_id_priv->lock); if (ib_post_send_mad(msg, NULL)) cm_free_msg(msg); goto deref; case IB_CM_DREQ_RCVD: atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_DREQ_COUNTER]); goto unlock; default: goto unlock; } cm_id_priv->id.state = IB_CM_DREQ_RCVD; cm_id_priv->tid = dreq_msg->hdr.tid; ret = atomic_inc_and_test(&cm_id_priv->work_count); if (!ret) list_add_tail(&work->list, &cm_id_priv->work_list); spin_unlock_irq(&cm_id_priv->lock); if (ret) cm_process_work(cm_id_priv, work); else cm_deref_id(cm_id_priv); return 0; unlock: spin_unlock_irq(&cm_id_priv->lock); deref: cm_deref_id(cm_id_priv); return -EINVAL; } static int cm_drep_handler(struct cm_work *work) { struct cm_id_private *cm_id_priv; struct cm_drep_msg *drep_msg; int ret; drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad; cm_id_priv = cm_acquire_id(drep_msg->remote_comm_id, drep_msg->local_comm_id); if (!cm_id_priv) return -EINVAL; work->cm_event.private_data = &drep_msg->private_data; spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_DREQ_SENT && cm_id_priv->id.state != IB_CM_DREQ_RCVD) { spin_unlock_irq(&cm_id_priv->lock); goto out; } cm_enter_timewait(cm_id_priv); ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); ret = atomic_inc_and_test(&cm_id_priv->work_count); if (!ret) list_add_tail(&work->list, &cm_id_priv->work_list); spin_unlock_irq(&cm_id_priv->lock); if (ret) cm_process_work(cm_id_priv, work); else cm_deref_id(cm_id_priv); return 0; out: cm_deref_id(cm_id_priv); return -EINVAL; } int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason, void *ari, u8 ari_length, const void *private_data, u8 private_data_len) { struct cm_id_private *cm_id_priv; struct ib_mad_send_buf *msg; unsigned long flags; int ret; if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) || (ari && ari_length > IB_CM_REJ_ARI_LENGTH)) return -EINVAL; cm_id_priv = container_of(cm_id, struct cm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); switch (cm_id->state) { case IB_CM_REQ_SENT: case IB_CM_MRA_REQ_RCVD: case IB_CM_REQ_RCVD: case IB_CM_MRA_REQ_SENT: case IB_CM_REP_RCVD: case IB_CM_MRA_REP_SENT: ret = cm_alloc_msg(cm_id_priv, &msg); if (!ret) cm_format_rej((struct cm_rej_msg *) msg->mad, cm_id_priv, reason, ari, ari_length, private_data, private_data_len); cm_reset_to_idle(cm_id_priv); break; case IB_CM_REP_SENT: case IB_CM_MRA_REP_RCVD: ret = cm_alloc_msg(cm_id_priv, &msg); if (!ret) cm_format_rej((struct cm_rej_msg *) msg->mad, cm_id_priv, reason, ari, ari_length, private_data, private_data_len); cm_enter_timewait(cm_id_priv); break; default: ret = -EINVAL; goto out; } if (ret) goto out; ret = ib_post_send_mad(msg, NULL); if (ret) cm_free_msg(msg); out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } EXPORT_SYMBOL(ib_send_cm_rej); static void cm_format_rej_event(struct cm_work *work) { struct cm_rej_msg *rej_msg; struct ib_cm_rej_event_param *param; rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.rej_rcvd; param->ari = rej_msg->ari; param->ari_length = cm_rej_get_reject_info_len(rej_msg); param->reason = __be16_to_cpu(rej_msg->reason); work->cm_event.private_data = &rej_msg->private_data; } static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg) { struct cm_timewait_info *timewait_info; struct cm_id_private *cm_id_priv; __be32 remote_id; remote_id = rej_msg->local_comm_id; if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_TIMEOUT) { spin_lock_irq(&cm.lock); timewait_info = cm_find_remote_id( *((__be64 *) rej_msg->ari), remote_id); if (!timewait_info) { spin_unlock_irq(&cm.lock); return NULL; } cm_id_priv = idr_find(&cm.local_id_table, (__force int) (timewait_info->work.local_id ^ cm.random_id_operand)); if (cm_id_priv) { if (cm_id_priv->id.remote_id == remote_id) atomic_inc(&cm_id_priv->refcount); else cm_id_priv = NULL; } spin_unlock_irq(&cm.lock); } else if (cm_rej_get_msg_rejected(rej_msg) == CM_MSG_RESPONSE_REQ) cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, 0); else cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, remote_id); return cm_id_priv; } static int cm_rej_handler(struct cm_work *work) { struct cm_id_private *cm_id_priv; struct cm_rej_msg *rej_msg; int ret; rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad; cm_id_priv = cm_acquire_rejected_id(rej_msg); if (!cm_id_priv) return -EINVAL; cm_format_rej_event(work); spin_lock_irq(&cm_id_priv->lock); switch (cm_id_priv->id.state) { case IB_CM_REQ_SENT: case IB_CM_MRA_REQ_RCVD: case IB_CM_REP_SENT: case IB_CM_MRA_REP_RCVD: ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); /* fall through */ case IB_CM_REQ_RCVD: case IB_CM_MRA_REQ_SENT: if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_STALE_CONN) cm_enter_timewait(cm_id_priv); else cm_reset_to_idle(cm_id_priv); break; case IB_CM_DREQ_SENT: ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); /* fall through */ case IB_CM_REP_RCVD: case IB_CM_MRA_REP_SENT: case IB_CM_ESTABLISHED: cm_enter_timewait(cm_id_priv); break; default: spin_unlock_irq(&cm_id_priv->lock); ret = -EINVAL; goto out; } ret = atomic_inc_and_test(&cm_id_priv->work_count); if (!ret) list_add_tail(&work->list, &cm_id_priv->work_list); spin_unlock_irq(&cm_id_priv->lock); if (ret) cm_process_work(cm_id_priv, work); else cm_deref_id(cm_id_priv); return 0; out: cm_deref_id(cm_id_priv); return -EINVAL; } int ib_send_cm_mra(struct ib_cm_id *cm_id, u8 service_timeout, const void *private_data, u8 private_data_len) { struct cm_id_private *cm_id_priv; struct ib_mad_send_buf *msg; enum ib_cm_state cm_state; enum ib_cm_lap_state lap_state; enum cm_msg_response msg_response; void *data; unsigned long flags; int ret; if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE) return -EINVAL; data = cm_copy_private_data(private_data, private_data_len); if (IS_ERR(data)) return PTR_ERR(data); cm_id_priv = container_of(cm_id, struct cm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); switch(cm_id_priv->id.state) { case IB_CM_REQ_RCVD: cm_state = IB_CM_MRA_REQ_SENT; lap_state = cm_id->lap_state; msg_response = CM_MSG_RESPONSE_REQ; break; case IB_CM_REP_RCVD: cm_state = IB_CM_MRA_REP_SENT; lap_state = cm_id->lap_state; msg_response = CM_MSG_RESPONSE_REP; break; case IB_CM_ESTABLISHED: if (cm_id->lap_state == IB_CM_LAP_RCVD) { cm_state = cm_id->state; lap_state = IB_CM_MRA_LAP_SENT; msg_response = CM_MSG_RESPONSE_OTHER; break; } default: ret = -EINVAL; goto error1; } if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) { ret = cm_alloc_msg(cm_id_priv, &msg); if (ret) goto error1; cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, msg_response, service_timeout, private_data, private_data_len); ret = ib_post_send_mad(msg, NULL); if (ret) goto error2; } cm_id->state = cm_state; cm_id->lap_state = lap_state; cm_id_priv->service_timeout = service_timeout; cm_set_private_data(cm_id_priv, data, private_data_len); spin_unlock_irqrestore(&cm_id_priv->lock, flags); return 0; error1: spin_unlock_irqrestore(&cm_id_priv->lock, flags); kfree(data); return ret; error2: spin_unlock_irqrestore(&cm_id_priv->lock, flags); kfree(data); cm_free_msg(msg); return ret; } EXPORT_SYMBOL(ib_send_cm_mra); static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg) { switch (cm_mra_get_msg_mraed(mra_msg)) { case CM_MSG_RESPONSE_REQ: return cm_acquire_id(mra_msg->remote_comm_id, 0); case CM_MSG_RESPONSE_REP: case CM_MSG_RESPONSE_OTHER: return cm_acquire_id(mra_msg->remote_comm_id, mra_msg->local_comm_id); default: return NULL; } } static int cm_mra_handler(struct cm_work *work) { struct cm_id_private *cm_id_priv; struct cm_mra_msg *mra_msg; int timeout, ret; mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad; cm_id_priv = cm_acquire_mraed_id(mra_msg); if (!cm_id_priv) return -EINVAL; work->cm_event.private_data = &mra_msg->private_data; work->cm_event.param.mra_rcvd.service_timeout = cm_mra_get_service_timeout(mra_msg); timeout = cm_convert_to_ms(cm_mra_get_service_timeout(mra_msg)) + cm_convert_to_ms(cm_id_priv->av.timeout); if (timeout > cm_convert_to_ms(max_timeout)) { printk(KERN_WARNING PFX "calculated mra timeout %d > %d, " "decreasing used timeout_ms\n", timeout, cm_convert_to_ms(max_timeout)); timeout = cm_convert_to_ms(max_timeout); } spin_lock_irq(&cm_id_priv->lock); switch (cm_id_priv->id.state) { case IB_CM_REQ_SENT: if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REQ || ib_modify_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg, timeout)) goto out; cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD; break; case IB_CM_REP_SENT: if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REP || ib_modify_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg, timeout)) goto out; cm_id_priv->id.state = IB_CM_MRA_REP_RCVD; break; case IB_CM_ESTABLISHED: if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_OTHER || cm_id_priv->id.lap_state != IB_CM_LAP_SENT || ib_modify_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg, timeout)) { if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD) atomic_long_inc(&work->port-> counter_group[CM_RECV_DUPLICATES]. counter[CM_MRA_COUNTER]); goto out; } cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD; break; case IB_CM_MRA_REQ_RCVD: case IB_CM_MRA_REP_RCVD: atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_MRA_COUNTER]); /* fall through */ default: goto out; } cm_id_priv->msg->context[1] = (void *) (unsigned long) cm_id_priv->id.state; ret = atomic_inc_and_test(&cm_id_priv->work_count); if (!ret) list_add_tail(&work->list, &cm_id_priv->work_list); spin_unlock_irq(&cm_id_priv->lock); if (ret) cm_process_work(cm_id_priv, work); else cm_deref_id(cm_id_priv); return 0; out: spin_unlock_irq(&cm_id_priv->lock); cm_deref_id(cm_id_priv); return -EINVAL; } static void cm_format_lap(struct cm_lap_msg *lap_msg, struct cm_id_private *cm_id_priv, struct ib_sa_path_rec *alternate_path, const void *private_data, u8 private_data_len) { cm_format_mad_hdr(&lap_msg->hdr, CM_LAP_ATTR_ID, cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_LAP)); lap_msg->local_comm_id = cm_id_priv->id.local_id; lap_msg->remote_comm_id = cm_id_priv->id.remote_id; cm_lap_set_remote_qpn(lap_msg, cm_id_priv->remote_qpn); /* todo: need remote CM response timeout */ cm_lap_set_remote_resp_timeout(lap_msg, 0x1F); lap_msg->alt_local_lid = alternate_path->slid; lap_msg->alt_remote_lid = alternate_path->dlid; lap_msg->alt_local_gid = alternate_path->sgid; lap_msg->alt_remote_gid = alternate_path->dgid; cm_lap_set_flow_label(lap_msg, alternate_path->flow_label); cm_lap_set_traffic_class(lap_msg, alternate_path->traffic_class); lap_msg->alt_hop_limit = alternate_path->hop_limit; cm_lap_set_packet_rate(lap_msg, alternate_path->rate); cm_lap_set_sl(lap_msg, alternate_path->sl); cm_lap_set_subnet_local(lap_msg, 1); /* local only... */ cm_lap_set_local_ack_timeout(lap_msg, cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay, alternate_path->packet_life_time)); if (private_data && private_data_len) memcpy(lap_msg->private_data, private_data, private_data_len); } int ib_send_cm_lap(struct ib_cm_id *cm_id, struct ib_sa_path_rec *alternate_path, const void *private_data, u8 private_data_len) { struct cm_id_private *cm_id_priv; struct ib_mad_send_buf *msg; unsigned long flags; int ret; if (private_data && private_data_len > IB_CM_LAP_PRIVATE_DATA_SIZE) return -EINVAL; cm_id_priv = container_of(cm_id, struct cm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state != IB_CM_ESTABLISHED || (cm_id->lap_state != IB_CM_LAP_UNINIT && cm_id->lap_state != IB_CM_LAP_IDLE)) { ret = -EINVAL; goto out; } ret = cm_init_av_by_path(alternate_path, &cm_id_priv->alt_av); if (ret) goto out; cm_id_priv->alt_av.timeout = cm_ack_timeout(cm_id_priv->target_ack_delay, cm_id_priv->alt_av.timeout - 1); ret = cm_alloc_msg(cm_id_priv, &msg); if (ret) goto out; cm_format_lap((struct cm_lap_msg *) msg->mad, cm_id_priv, alternate_path, private_data, private_data_len); msg->timeout_ms = cm_id_priv->timeout_ms; msg->context[1] = (void *) (unsigned long) IB_CM_ESTABLISHED; ret = ib_post_send_mad(msg, NULL); if (ret) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); cm_free_msg(msg); return ret; } cm_id->lap_state = IB_CM_LAP_SENT; cm_id_priv->msg = msg; out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } EXPORT_SYMBOL(ib_send_cm_lap); static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv, struct ib_sa_path_rec *path, struct cm_lap_msg *lap_msg) { memset(path, 0, sizeof *path); path->dgid = lap_msg->alt_local_gid; path->sgid = lap_msg->alt_remote_gid; path->dlid = lap_msg->alt_local_lid; path->slid = lap_msg->alt_remote_lid; path->flow_label = cm_lap_get_flow_label(lap_msg); path->hop_limit = lap_msg->alt_hop_limit; path->traffic_class = cm_lap_get_traffic_class(lap_msg); path->reversible = 1; path->pkey = cm_id_priv->pkey; path->sl = cm_lap_get_sl(lap_msg); path->mtu_selector = IB_SA_EQ; path->mtu = cm_id_priv->path_mtu; path->rate_selector = IB_SA_EQ; path->rate = cm_lap_get_packet_rate(lap_msg); path->packet_life_time_selector = IB_SA_EQ; path->packet_life_time = cm_lap_get_local_ack_timeout(lap_msg); path->packet_life_time -= (path->packet_life_time > 0); } static int cm_lap_handler(struct cm_work *work) { struct cm_id_private *cm_id_priv; struct cm_lap_msg *lap_msg; struct ib_cm_lap_event_param *param; struct ib_mad_send_buf *msg = NULL; int ret; /* todo: verify LAP request and send reject APR if invalid. */ lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad; cm_id_priv = cm_acquire_id(lap_msg->remote_comm_id, lap_msg->local_comm_id); if (!cm_id_priv) return -EINVAL; param = &work->cm_event.param.lap_rcvd; param->alternate_path = &work->path[0]; cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg); work->cm_event.private_data = &lap_msg->private_data; spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_ESTABLISHED) goto unlock; switch (cm_id_priv->id.lap_state) { case IB_CM_LAP_UNINIT: case IB_CM_LAP_IDLE: break; case IB_CM_MRA_LAP_SENT: atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_LAP_COUNTER]); if (cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg)) goto unlock; cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, CM_MSG_RESPONSE_OTHER, cm_id_priv->service_timeout, cm_id_priv->private_data, cm_id_priv->private_data_len); spin_unlock_irq(&cm_id_priv->lock); if (ib_post_send_mad(msg, NULL)) cm_free_msg(msg); goto deref; case IB_CM_LAP_RCVD: atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_LAP_COUNTER]); goto unlock; default: goto unlock; } cm_id_priv->id.lap_state = IB_CM_LAP_RCVD; cm_id_priv->tid = lap_msg->hdr.tid; cm_init_av_for_response(work->port, work->mad_recv_wc->wc, work->mad_recv_wc->recv_buf.grh, &cm_id_priv->av); cm_init_av_by_path(param->alternate_path, &cm_id_priv->alt_av); ret = atomic_inc_and_test(&cm_id_priv->work_count); if (!ret) list_add_tail(&work->list, &cm_id_priv->work_list); spin_unlock_irq(&cm_id_priv->lock); if (ret) cm_process_work(cm_id_priv, work); else cm_deref_id(cm_id_priv); return 0; unlock: spin_unlock_irq(&cm_id_priv->lock); deref: cm_deref_id(cm_id_priv); return -EINVAL; } static void cm_format_apr(struct cm_apr_msg *apr_msg, struct cm_id_private *cm_id_priv, enum ib_cm_apr_status status, void *info, u8 info_length, const void *private_data, u8 private_data_len) { cm_format_mad_hdr(&apr_msg->hdr, CM_APR_ATTR_ID, cm_id_priv->tid); apr_msg->local_comm_id = cm_id_priv->id.local_id; apr_msg->remote_comm_id = cm_id_priv->id.remote_id; apr_msg->ap_status = (u8) status; if (info && info_length) { apr_msg->info_length = info_length; memcpy(apr_msg->info, info, info_length); } if (private_data && private_data_len) memcpy(apr_msg->private_data, private_data, private_data_len); } int ib_send_cm_apr(struct ib_cm_id *cm_id, enum ib_cm_apr_status status, void *info, u8 info_length, const void *private_data, u8 private_data_len) { struct cm_id_private *cm_id_priv; struct ib_mad_send_buf *msg; unsigned long flags; int ret; if ((private_data && private_data_len > IB_CM_APR_PRIVATE_DATA_SIZE) || (info && info_length > IB_CM_APR_INFO_LENGTH)) return -EINVAL; cm_id_priv = container_of(cm_id, struct cm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state != IB_CM_ESTABLISHED || (cm_id->lap_state != IB_CM_LAP_RCVD && cm_id->lap_state != IB_CM_MRA_LAP_SENT)) { ret = -EINVAL; goto out; } ret = cm_alloc_msg(cm_id_priv, &msg); if (ret) goto out; cm_format_apr((struct cm_apr_msg *) msg->mad, cm_id_priv, status, info, info_length, private_data, private_data_len); ret = ib_post_send_mad(msg, NULL); if (ret) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); cm_free_msg(msg); return ret; } cm_id->lap_state = IB_CM_LAP_IDLE; out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } EXPORT_SYMBOL(ib_send_cm_apr); static int cm_apr_handler(struct cm_work *work) { struct cm_id_private *cm_id_priv; struct cm_apr_msg *apr_msg; int ret; apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad; cm_id_priv = cm_acquire_id(apr_msg->remote_comm_id, apr_msg->local_comm_id); if (!cm_id_priv) return -EINVAL; /* Unmatched reply. */ work->cm_event.param.apr_rcvd.ap_status = apr_msg->ap_status; work->cm_event.param.apr_rcvd.apr_info = &apr_msg->info; work->cm_event.param.apr_rcvd.info_len = apr_msg->info_length; work->cm_event.private_data = &apr_msg->private_data; spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_ESTABLISHED || (cm_id_priv->id.lap_state != IB_CM_LAP_SENT && cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) { spin_unlock_irq(&cm_id_priv->lock); goto out; } cm_id_priv->id.lap_state = IB_CM_LAP_IDLE; ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); cm_id_priv->msg = NULL; ret = atomic_inc_and_test(&cm_id_priv->work_count); if (!ret) list_add_tail(&work->list, &cm_id_priv->work_list); spin_unlock_irq(&cm_id_priv->lock); if (ret) cm_process_work(cm_id_priv, work); else cm_deref_id(cm_id_priv); return 0; out: cm_deref_id(cm_id_priv); return -EINVAL; } static int cm_timewait_handler(struct cm_work *work) { struct cm_timewait_info *timewait_info; struct cm_id_private *cm_id_priv; int ret; timewait_info = (struct cm_timewait_info *)work; spin_lock_irq(&cm.lock); list_del(&timewait_info->list); spin_unlock_irq(&cm.lock); cm_id_priv = cm_acquire_id(timewait_info->work.local_id, timewait_info->work.remote_id); if (!cm_id_priv) return -EINVAL; spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_TIMEWAIT || cm_id_priv->remote_qpn != timewait_info->remote_qpn) { spin_unlock_irq(&cm_id_priv->lock); goto out; } cm_id_priv->id.state = IB_CM_IDLE; ret = atomic_inc_and_test(&cm_id_priv->work_count); if (!ret) list_add_tail(&work->list, &cm_id_priv->work_list); spin_unlock_irq(&cm_id_priv->lock); if (ret) cm_process_work(cm_id_priv, work); else cm_deref_id(cm_id_priv); return 0; out: cm_deref_id(cm_id_priv); return -EINVAL; } static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg, struct cm_id_private *cm_id_priv, struct ib_cm_sidr_req_param *param) { cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID, cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_SIDR)); sidr_req_msg->request_id = cm_id_priv->id.local_id; sidr_req_msg->pkey = param->path->pkey; sidr_req_msg->service_id = param->service_id; if (param->private_data && param->private_data_len) memcpy(sidr_req_msg->private_data, param->private_data, param->private_data_len); } int ib_send_cm_sidr_req(struct ib_cm_id *cm_id, struct ib_cm_sidr_req_param *param) { struct cm_id_private *cm_id_priv; struct ib_mad_send_buf *msg; unsigned long flags; int ret; if (!param->path || (param->private_data && param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE)) return -EINVAL; cm_id_priv = container_of(cm_id, struct cm_id_private, id); ret = cm_init_av_by_path(param->path, &cm_id_priv->av); if (ret) goto out; cm_id->service_id = param->service_id; cm_id->service_mask = ~cpu_to_be64(0); cm_id_priv->timeout_ms = param->timeout_ms; if (cm_id_priv->timeout_ms > cm_convert_to_ms(max_timeout)) { printk(KERN_WARNING PFX "sidr req timeout_ms %d > %d, " "decreasing used timeout_ms\n", param->timeout_ms, cm_convert_to_ms(max_timeout)); cm_id_priv->timeout_ms = cm_convert_to_ms(max_timeout); } cm_id_priv->max_cm_retries = param->max_cm_retries; ret = cm_alloc_msg(cm_id_priv, &msg); if (ret) goto out; cm_format_sidr_req((struct cm_sidr_req_msg *) msg->mad, cm_id_priv, param); msg->timeout_ms = cm_id_priv->timeout_ms; msg->context[1] = (void *) (unsigned long) IB_CM_SIDR_REQ_SENT; spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state == IB_CM_IDLE) ret = ib_post_send_mad(msg, NULL); else ret = -EINVAL; if (ret) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); cm_free_msg(msg); goto out; } cm_id->state = IB_CM_SIDR_REQ_SENT; cm_id_priv->msg = msg; spin_unlock_irqrestore(&cm_id_priv->lock, flags); out: return ret; } EXPORT_SYMBOL(ib_send_cm_sidr_req); static void cm_format_sidr_req_event(struct cm_work *work, struct ib_cm_id *listen_id) { struct cm_sidr_req_msg *sidr_req_msg; struct ib_cm_sidr_req_event_param *param; sidr_req_msg = (struct cm_sidr_req_msg *) work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.sidr_req_rcvd; param->pkey = __be16_to_cpu(sidr_req_msg->pkey); param->listen_id = listen_id; param->port = work->port->port_num; work->cm_event.private_data = &sidr_req_msg->private_data; } static int cm_sidr_req_handler(struct cm_work *work) { struct ib_cm_id *cm_id; struct cm_id_private *cm_id_priv, *cur_cm_id_priv; struct cm_sidr_req_msg *sidr_req_msg; struct ib_wc *wc; cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL); if (IS_ERR(cm_id)) return PTR_ERR(cm_id); cm_id_priv = container_of(cm_id, struct cm_id_private, id); /* Record SGID/SLID and request ID for lookup. */ sidr_req_msg = (struct cm_sidr_req_msg *) work->mad_recv_wc->recv_buf.mad; wc = work->mad_recv_wc->wc; cm_id_priv->av.dgid.global.subnet_prefix = cpu_to_be64(wc->slid); cm_id_priv->av.dgid.global.interface_id = 0; cm_init_av_for_response(work->port, work->mad_recv_wc->wc, work->mad_recv_wc->recv_buf.grh, &cm_id_priv->av); cm_id_priv->id.remote_id = sidr_req_msg->request_id; cm_id_priv->tid = sidr_req_msg->hdr.tid; atomic_inc(&cm_id_priv->work_count); spin_lock_irq(&cm.lock); cur_cm_id_priv = cm_insert_remote_sidr(cm_id_priv); if (cur_cm_id_priv) { spin_unlock_irq(&cm.lock); atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_SIDR_REQ_COUNTER]); goto out; /* Duplicate message. */ } cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD; cur_cm_id_priv = cm_find_listen(cm_id->device, sidr_req_msg->service_id, sidr_req_msg->private_data); if (!cur_cm_id_priv) { spin_unlock_irq(&cm.lock); cm_reject_sidr_req(cm_id_priv, IB_SIDR_UNSUPPORTED); goto out; /* No match. */ } atomic_inc(&cur_cm_id_priv->refcount); spin_unlock_irq(&cm.lock); cm_id_priv->id.cm_handler = cur_cm_id_priv->id.cm_handler; cm_id_priv->id.context = cur_cm_id_priv->id.context; cm_id_priv->id.service_id = sidr_req_msg->service_id; cm_id_priv->id.service_mask = ~cpu_to_be64(0); cm_format_sidr_req_event(work, &cur_cm_id_priv->id); cm_process_work(cm_id_priv, work); cm_deref_id(cur_cm_id_priv); return 0; out: ib_destroy_cm_id(&cm_id_priv->id); return -EINVAL; } static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg, struct cm_id_private *cm_id_priv, struct ib_cm_sidr_rep_param *param) { cm_format_mad_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID, cm_id_priv->tid); sidr_rep_msg->request_id = cm_id_priv->id.remote_id; sidr_rep_msg->status = param->status; cm_sidr_rep_set_qpn(sidr_rep_msg, cpu_to_be32(param->qp_num)); sidr_rep_msg->service_id = cm_id_priv->id.service_id; sidr_rep_msg->qkey = cpu_to_be32(param->qkey); if (param->info && param->info_length) memcpy(sidr_rep_msg->info, param->info, param->info_length); if (param->private_data && param->private_data_len) memcpy(sidr_rep_msg->private_data, param->private_data, param->private_data_len); } int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id, struct ib_cm_sidr_rep_param *param) { struct cm_id_private *cm_id_priv; struct ib_mad_send_buf *msg; unsigned long flags; int ret; if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) || (param->private_data && param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE)) return -EINVAL; cm_id_priv = container_of(cm_id, struct cm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state != IB_CM_SIDR_REQ_RCVD) { ret = -EINVAL; goto error; } ret = cm_alloc_msg(cm_id_priv, &msg); if (ret) goto error; cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv, param); ret = ib_post_send_mad(msg, NULL); if (ret) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); cm_free_msg(msg); return ret; } cm_id->state = IB_CM_IDLE; spin_unlock_irqrestore(&cm_id_priv->lock, flags); spin_lock_irqsave(&cm.lock, flags); rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table); spin_unlock_irqrestore(&cm.lock, flags); return 0; error: spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } EXPORT_SYMBOL(ib_send_cm_sidr_rep); static void cm_format_sidr_rep_event(struct cm_work *work) { struct cm_sidr_rep_msg *sidr_rep_msg; struct ib_cm_sidr_rep_event_param *param; sidr_rep_msg = (struct cm_sidr_rep_msg *) work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.sidr_rep_rcvd; param->status = sidr_rep_msg->status; param->qkey = be32_to_cpu(sidr_rep_msg->qkey); param->qpn = be32_to_cpu(cm_sidr_rep_get_qpn(sidr_rep_msg)); param->info = &sidr_rep_msg->info; param->info_len = sidr_rep_msg->info_length; work->cm_event.private_data = &sidr_rep_msg->private_data; } static int cm_sidr_rep_handler(struct cm_work *work) { struct cm_sidr_rep_msg *sidr_rep_msg; struct cm_id_private *cm_id_priv; sidr_rep_msg = (struct cm_sidr_rep_msg *) work->mad_recv_wc->recv_buf.mad; cm_id_priv = cm_acquire_id(sidr_rep_msg->request_id, 0); if (!cm_id_priv) return -EINVAL; /* Unmatched reply. */ spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) { spin_unlock_irq(&cm_id_priv->lock); goto out; } cm_id_priv->id.state = IB_CM_IDLE; ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); spin_unlock_irq(&cm_id_priv->lock); cm_format_sidr_rep_event(work); cm_process_work(cm_id_priv, work); return 0; out: cm_deref_id(cm_id_priv); return -EINVAL; } static void cm_process_send_error(struct ib_mad_send_buf *msg, enum ib_wc_status wc_status) { struct cm_id_private *cm_id_priv; struct ib_cm_event cm_event; enum ib_cm_state state; int ret; memset(&cm_event, 0, sizeof cm_event); cm_id_priv = msg->context[0]; /* Discard old sends or ones without a response. */ spin_lock_irq(&cm_id_priv->lock); state = (enum ib_cm_state) (unsigned long) msg->context[1]; if (msg != cm_id_priv->msg || state != cm_id_priv->id.state) goto discard; switch (state) { case IB_CM_REQ_SENT: case IB_CM_MRA_REQ_RCVD: cm_reset_to_idle(cm_id_priv); cm_event.event = IB_CM_REQ_ERROR; break; case IB_CM_REP_SENT: case IB_CM_MRA_REP_RCVD: cm_reset_to_idle(cm_id_priv); cm_event.event = IB_CM_REP_ERROR; break; case IB_CM_DREQ_SENT: cm_enter_timewait(cm_id_priv); cm_event.event = IB_CM_DREQ_ERROR; break; case IB_CM_SIDR_REQ_SENT: cm_id_priv->id.state = IB_CM_IDLE; cm_event.event = IB_CM_SIDR_REQ_ERROR; break; default: goto discard; } spin_unlock_irq(&cm_id_priv->lock); cm_event.param.send_status = wc_status; /* No other events can occur on the cm_id at this point. */ ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event); cm_free_msg(msg); if (ret) ib_destroy_cm_id(&cm_id_priv->id); return; discard: spin_unlock_irq(&cm_id_priv->lock); cm_free_msg(msg); } static void cm_send_handler(struct ib_mad_agent *mad_agent, struct ib_mad_send_wc *mad_send_wc) { struct ib_mad_send_buf *msg = mad_send_wc->send_buf; struct cm_port *port; u16 attr_index; port = mad_agent->context; attr_index = be16_to_cpu(((struct ib_mad_hdr *) msg->mad)->attr_id) - CM_ATTR_ID_OFFSET; /* * If the send was in response to a received message (context[0] is not * set to a cm_id), and is not a REJ, then it is a send that was * manually retried. */ if (!msg->context[0] && (attr_index != CM_REJ_COUNTER)) msg->retries = 1; atomic_long_add(1 + msg->retries, &port->counter_group[CM_XMIT].counter[attr_index]); if (msg->retries) atomic_long_add(msg->retries, &port->counter_group[CM_XMIT_RETRIES]. counter[attr_index]); switch (mad_send_wc->status) { case IB_WC_SUCCESS: case IB_WC_WR_FLUSH_ERR: cm_free_msg(msg); break; default: if (msg->context[0] && msg->context[1]) cm_process_send_error(msg, mad_send_wc->status); else cm_free_msg(msg); break; } } static void cm_work_handler(struct work_struct *_work) { struct cm_work *work = container_of(_work, struct cm_work, work.work); int ret; switch (work->cm_event.event) { case IB_CM_REQ_RECEIVED: ret = cm_req_handler(work); break; case IB_CM_MRA_RECEIVED: ret = cm_mra_handler(work); break; case IB_CM_REJ_RECEIVED: ret = cm_rej_handler(work); break; case IB_CM_REP_RECEIVED: ret = cm_rep_handler(work); break; case IB_CM_RTU_RECEIVED: ret = cm_rtu_handler(work); break; case IB_CM_USER_ESTABLISHED: ret = cm_establish_handler(work); break; case IB_CM_DREQ_RECEIVED: ret = cm_dreq_handler(work); break; case IB_CM_DREP_RECEIVED: ret = cm_drep_handler(work); break; case IB_CM_SIDR_REQ_RECEIVED: ret = cm_sidr_req_handler(work); break; case IB_CM_SIDR_REP_RECEIVED: ret = cm_sidr_rep_handler(work); break; case IB_CM_LAP_RECEIVED: ret = cm_lap_handler(work); break; case IB_CM_APR_RECEIVED: ret = cm_apr_handler(work); break; case IB_CM_TIMEWAIT_EXIT: ret = cm_timewait_handler(work); break; default: ret = -EINVAL; break; } if (ret) cm_free_work(work); } static int cm_establish(struct ib_cm_id *cm_id) { struct cm_id_private *cm_id_priv; struct cm_work *work; unsigned long flags; int ret = 0; work = kmalloc(sizeof *work, GFP_ATOMIC); if (!work) return -ENOMEM; cm_id_priv = container_of(cm_id, struct cm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); switch (cm_id->state) { case IB_CM_REP_SENT: case IB_CM_MRA_REP_RCVD: cm_id->state = IB_CM_ESTABLISHED; break; case IB_CM_ESTABLISHED: ret = -EISCONN; break; default: ret = -EINVAL; break; } spin_unlock_irqrestore(&cm_id_priv->lock, flags); if (ret) { kfree(work); goto out; } /* * The CM worker thread may try to destroy the cm_id before it * can execute this work item. To prevent potential deadlock, * we need to find the cm_id once we're in the context of the * worker thread, rather than holding a reference on it. */ INIT_DELAYED_WORK(&work->work, cm_work_handler); work->local_id = cm_id->local_id; work->remote_id = cm_id->remote_id; work->mad_recv_wc = NULL; work->cm_event.event = IB_CM_USER_ESTABLISHED; queue_delayed_work(cm.wq, &work->work, 0); out: return ret; } static int cm_migrate(struct ib_cm_id *cm_id) { struct cm_id_private *cm_id_priv; unsigned long flags; int ret = 0; cm_id_priv = container_of(cm_id, struct cm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id->state == IB_CM_ESTABLISHED && (cm_id->lap_state == IB_CM_LAP_UNINIT || cm_id->lap_state == IB_CM_LAP_IDLE)) { cm_id->lap_state = IB_CM_LAP_IDLE; cm_id_priv->av = cm_id_priv->alt_av; } else ret = -EINVAL; spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event) { int ret; switch (event) { case IB_EVENT_COMM_EST: ret = cm_establish(cm_id); break; case IB_EVENT_PATH_MIG: ret = cm_migrate(cm_id); break; default: ret = -EINVAL; } return ret; } EXPORT_SYMBOL(ib_cm_notify); static void cm_recv_handler(struct ib_mad_agent *mad_agent, struct ib_mad_recv_wc *mad_recv_wc) { struct cm_port *port = mad_agent->context; struct cm_work *work; enum ib_cm_event_type event; u16 attr_id; int paths = 0; switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) { case CM_REQ_ATTR_ID: paths = 1 + (((struct cm_req_msg *) mad_recv_wc->recv_buf.mad)-> alt_local_lid != 0); event = IB_CM_REQ_RECEIVED; break; case CM_MRA_ATTR_ID: event = IB_CM_MRA_RECEIVED; break; case CM_REJ_ATTR_ID: event = IB_CM_REJ_RECEIVED; break; case CM_REP_ATTR_ID: event = IB_CM_REP_RECEIVED; break; case CM_RTU_ATTR_ID: event = IB_CM_RTU_RECEIVED; break; case CM_DREQ_ATTR_ID: event = IB_CM_DREQ_RECEIVED; break; case CM_DREP_ATTR_ID: event = IB_CM_DREP_RECEIVED; break; case CM_SIDR_REQ_ATTR_ID: event = IB_CM_SIDR_REQ_RECEIVED; break; case CM_SIDR_REP_ATTR_ID: event = IB_CM_SIDR_REP_RECEIVED; break; case CM_LAP_ATTR_ID: paths = 1; event = IB_CM_LAP_RECEIVED; break; case CM_APR_ATTR_ID: event = IB_CM_APR_RECEIVED; break; default: ib_free_recv_mad(mad_recv_wc); return; } attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id); atomic_long_inc(&port->counter_group[CM_RECV]. counter[attr_id - CM_ATTR_ID_OFFSET]); work = kmalloc(sizeof *work + sizeof(struct ib_sa_path_rec) * paths, GFP_KERNEL); if (!work) { ib_free_recv_mad(mad_recv_wc); return; } INIT_DELAYED_WORK(&work->work, cm_work_handler); work->cm_event.event = event; work->mad_recv_wc = mad_recv_wc; work->port = port; queue_delayed_work(cm.wq, &work->work, 0); } static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { unsigned long flags; int ret; spin_lock_irqsave(&cm_id_priv->lock, flags); switch (cm_id_priv->id.state) { case IB_CM_REQ_SENT: case IB_CM_MRA_REQ_RCVD: case IB_CM_REQ_RCVD: case IB_CM_MRA_REQ_SENT: case IB_CM_REP_RCVD: case IB_CM_MRA_REP_SENT: case IB_CM_REP_SENT: case IB_CM_MRA_REP_RCVD: case IB_CM_ESTABLISHED: *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX | IB_QP_PORT; qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE; if (cm_id_priv->responder_resources) qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_ATOMIC; qp_attr->pkey_index = cm_id_priv->av.pkey_index; qp_attr->port_num = cm_id_priv->av.port->port_num; ret = 0; break; default: ret = -EINVAL; break; } spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { unsigned long flags; int ret; spin_lock_irqsave(&cm_id_priv->lock, flags); switch (cm_id_priv->id.state) { case IB_CM_REQ_RCVD: case IB_CM_MRA_REQ_SENT: case IB_CM_REP_RCVD: case IB_CM_MRA_REP_SENT: case IB_CM_REP_SENT: case IB_CM_MRA_REP_RCVD: case IB_CM_ESTABLISHED: *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN | IB_QP_RQ_PSN; qp_attr->ah_attr = cm_id_priv->av.ah_attr; qp_attr->path_mtu = cm_id_priv->path_mtu; qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn); qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn); if (cm_id_priv->qp_type == IB_QPT_RC) { *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER; qp_attr->max_dest_rd_atomic = cm_id_priv->responder_resources; qp_attr->min_rnr_timer = 0; } if (cm_id_priv->alt_av.ah_attr.dlid) { *qp_attr_mask |= IB_QP_ALT_PATH; qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num; qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index; qp_attr->alt_timeout = cm_id_priv->alt_av.timeout; qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr; } ret = 0; break; default: ret = -EINVAL; break; } spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { unsigned long flags; int ret; spin_lock_irqsave(&cm_id_priv->lock, flags); switch (cm_id_priv->id.state) { /* Allow transition to RTS before sending REP */ case IB_CM_REQ_RCVD: case IB_CM_MRA_REQ_SENT: case IB_CM_REP_RCVD: case IB_CM_MRA_REP_SENT: case IB_CM_REP_SENT: case IB_CM_MRA_REP_RCVD: case IB_CM_ESTABLISHED: if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) { *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN; qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn); if (cm_id_priv->qp_type == IB_QPT_RC) { *qp_attr_mask |= IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY | IB_QP_MAX_QP_RD_ATOMIC; qp_attr->timeout = cm_id_priv->av.timeout; qp_attr->retry_cnt = cm_id_priv->retry_count; qp_attr->rnr_retry = cm_id_priv->rnr_retry_count; qp_attr->max_rd_atomic = cm_id_priv->initiator_depth; } if (cm_id_priv->alt_av.ah_attr.dlid) { *qp_attr_mask |= IB_QP_PATH_MIG_STATE; qp_attr->path_mig_state = IB_MIG_REARM; } } else { *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE; qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num; qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index; qp_attr->alt_timeout = cm_id_priv->alt_av.timeout; qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr; qp_attr->path_mig_state = IB_MIG_REARM; } ret = 0; break; default: ret = -EINVAL; break; } spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } int ib_cm_init_qp_attr(struct ib_cm_id *cm_id, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { struct cm_id_private *cm_id_priv; int ret; cm_id_priv = container_of(cm_id, struct cm_id_private, id); switch (qp_attr->qp_state) { case IB_QPS_INIT: ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask); break; case IB_QPS_RTR: ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask); break; case IB_QPS_RTS: ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask); break; default: ret = -EINVAL; break; } return ret; } EXPORT_SYMBOL(ib_cm_init_qp_attr); static void cm_get_ack_delay(struct cm_device *cm_dev) { struct ib_device_attr attr; if (ib_query_device(cm_dev->ib_device, &attr)) cm_dev->ack_delay = 0; /* acks will rely on packet life time */ else cm_dev->ack_delay = attr.local_ca_ack_delay; } static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr, char *buf) { struct cm_counter_group *group; struct cm_counter_attribute *cm_attr; group = container_of(obj, struct cm_counter_group, obj); cm_attr = container_of(attr, struct cm_counter_attribute, attr); return sprintf(buf, "%ld\n", atomic_long_read(&group->counter[cm_attr->index])); } static struct sysfs_ops cm_counter_ops = { .show = cm_show_counter }; static struct kobj_type cm_counter_obj_type = { .sysfs_ops = &cm_counter_ops, .default_attrs = cm_counter_default_attrs }; static void cm_release_port_obj(struct kobject *obj) { struct cm_port *cm_port; cm_port = container_of(obj, struct cm_port, port_obj); kfree(cm_port); } static struct kobj_type cm_port_obj_type = { .release = cm_release_port_obj }; struct class cm_class = { .name = "infiniband_cm", }; EXPORT_SYMBOL(cm_class); static int cm_create_port_fs(struct cm_port *port) { int i, ret; ret = kobject_init_and_add(&port->port_obj, &cm_port_obj_type, &port->cm_dev->device->kobj, "%d", port->port_num); if (ret) { kfree(port); return ret; } for (i = 0; i < CM_COUNTER_GROUPS; i++) { ret = kobject_init_and_add(&port->counter_group[i].obj, &cm_counter_obj_type, &port->port_obj, "%s", counter_group_names[i]); if (ret) goto error; } return 0; error: while (i--) kobject_put(&port->counter_group[i].obj); kobject_put(&port->port_obj); return ret; } static void cm_remove_port_fs(struct cm_port *port) { int i; for (i = 0; i < CM_COUNTER_GROUPS; i++) kobject_put(&port->counter_group[i].obj); kobject_put(&port->port_obj); } static void cm_add_one(struct ib_device *ib_device) { struct cm_device *cm_dev; struct cm_port *port; struct ib_mad_reg_req reg_req = { .mgmt_class = IB_MGMT_CLASS_CM, .mgmt_class_version = IB_CM_CLASS_VERSION }; struct ib_port_modify port_modify = { .set_port_cap_mask = IB_PORT_CM_SUP }; unsigned long flags; int ret; u8 i; if (rdma_node_get_transport(ib_device->node_type) != RDMA_TRANSPORT_IB) return; cm_dev = kzalloc(sizeof(*cm_dev) + sizeof(*port) * ib_device->phys_port_cnt, GFP_KERNEL); if (!cm_dev) return; cm_dev->ib_device = ib_device; cm_get_ack_delay(cm_dev); cm_dev->device = device_create(&cm_class, &ib_device->dev, MKDEV(0, 0), NULL, "%s", ib_device->name); if (!cm_dev->device) { kfree(cm_dev); return; } set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask); for (i = 1; i <= ib_device->phys_port_cnt; i++) { port = kzalloc(sizeof *port, GFP_KERNEL); if (!port) goto error1; cm_dev->port[i-1] = port; port->cm_dev = cm_dev; port->port_num = i; ret = cm_create_port_fs(port); if (ret) goto error1; port->mad_agent = ib_register_mad_agent(ib_device, i, IB_QPT_GSI, ®_req, 0, cm_send_handler, cm_recv_handler, port); if (IS_ERR(port->mad_agent)) goto error2; ret = ib_modify_port(ib_device, i, 0, &port_modify); if (ret) goto error3; } ib_set_client_data(ib_device, &cm_client, cm_dev); write_lock_irqsave(&cm.device_lock, flags); list_add_tail(&cm_dev->list, &cm.device_list); write_unlock_irqrestore(&cm.device_lock, flags); return; error3: ib_unregister_mad_agent(port->mad_agent); error2: cm_remove_port_fs(port); error1: port_modify.set_port_cap_mask = 0; port_modify.clr_port_cap_mask = IB_PORT_CM_SUP; while (--i) { port = cm_dev->port[i-1]; ib_modify_port(ib_device, port->port_num, 0, &port_modify); ib_unregister_mad_agent(port->mad_agent); cm_remove_port_fs(port); } device_unregister(cm_dev->device); kfree(cm_dev); } static void cm_remove_one(struct ib_device *ib_device) { struct cm_device *cm_dev; struct cm_port *port; struct ib_port_modify port_modify = { .clr_port_cap_mask = IB_PORT_CM_SUP }; unsigned long flags; int i; cm_dev = ib_get_client_data(ib_device, &cm_client); if (!cm_dev) return; write_lock_irqsave(&cm.device_lock, flags); list_del(&cm_dev->list); write_unlock_irqrestore(&cm.device_lock, flags); for (i = 1; i <= ib_device->phys_port_cnt; i++) { port = cm_dev->port[i-1]; ib_modify_port(ib_device, port->port_num, 0, &port_modify); ib_unregister_mad_agent(port->mad_agent); flush_workqueue(cm.wq); cm_remove_port_fs(port); } device_unregister(cm_dev->device); kfree(cm_dev); } static int __init ib_cm_init(void) { int ret; memset(&cm, 0, sizeof cm); INIT_LIST_HEAD(&cm.device_list); rwlock_init(&cm.device_lock); spin_lock_init(&cm.lock); cm.listen_service_table = RB_ROOT; cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID); cm.remote_id_table = RB_ROOT; cm.remote_qp_table = RB_ROOT; cm.remote_sidr_table = RB_ROOT; idr_init(&cm.local_id_table); get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand); idr_pre_get(&cm.local_id_table, GFP_KERNEL); INIT_LIST_HEAD(&cm.timewait_list); ret = class_register(&cm_class); if (ret) return -ENOMEM; cm.wq = create_workqueue("ib_cm"); if (!cm.wq) { ret = -ENOMEM; goto error1; } ret = ib_register_client(&cm_client); if (ret) goto error2; return 0; error2: destroy_workqueue(cm.wq); error1: class_unregister(&cm_class); return ret; } static void __exit ib_cm_cleanup(void) { struct cm_timewait_info *timewait_info, *tmp; spin_lock_irq(&cm.lock); list_for_each_entry(timewait_info, &cm.timewait_list, list) cancel_delayed_work(&timewait_info->work.work); spin_unlock_irq(&cm.lock); ib_unregister_client(&cm_client); destroy_workqueue(cm.wq); list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) { list_del(&timewait_info->list); kfree(timewait_info); } class_unregister(&cm_class); idr_destroy(&cm.local_id_table); } module_init_order(ib_cm_init, SI_ORDER_SECOND); -module_exit(ib_cm_cleanup); +module_exit_order(ib_cm_cleanup, SI_ORDER_FIRST); Index: stable/9/sys/ofed/drivers/infiniband/core/cma.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/cma.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/cma.c (revision 273864) @@ -1,3420 +1,3420 @@ /* * Copyright (c) 2005 Voltaire Inc. All rights reserved. * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved. * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include MODULE_AUTHOR("Sean Hefty"); MODULE_DESCRIPTION("Generic RDMA CM Agent"); MODULE_LICENSE("Dual BSD/GPL"); static int tavor_quirk = 0; module_param_named(tavor_quirk, tavor_quirk, int, 0644); MODULE_PARM_DESC(tavor_quirk, "Tavor performance quirk: limit MTU to 1K if > 0"); int unify_tcp_port_space = 1; module_param(unify_tcp_port_space, int, 0644); MODULE_PARM_DESC(unify_tcp_port_space, "Unify the host TCP and RDMA port " "space allocation (default=1)"); #define CMA_CM_RESPONSE_TIMEOUT 20 #define CMA_MAX_CM_RETRIES 15 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24) #define IBOE_PACKET_LIFETIME 18 static int cma_response_timeout = CMA_CM_RESPONSE_TIMEOUT; module_param_named(cma_response_timeout, cma_response_timeout, int, 0644); MODULE_PARM_DESC(cma_response_timeout, "CMA_CM_RESPONSE_TIMEOUT default=20"); static int def_prec2sl = 3; module_param_named(def_prec2sl, def_prec2sl, int, 0644); MODULE_PARM_DESC(def_prec2sl, "Default value for SL priority with RoCE. Valid values 0 - 7"); static void cma_add_one(struct ib_device *device); static void cma_remove_one(struct ib_device *device); static struct ib_client cma_client = { .name = "cma", .add = cma_add_one, .remove = cma_remove_one }; static struct ib_sa_client sa_client; static struct rdma_addr_client addr_client; static LIST_HEAD(dev_list); static LIST_HEAD(listen_any_list); static DEFINE_MUTEX(lock); static struct workqueue_struct *cma_wq; static DEFINE_IDR(sdp_ps); static DEFINE_IDR(tcp_ps); static DEFINE_IDR(udp_ps); static DEFINE_IDR(ipoib_ps); #if defined(INET) static int next_port; #endif struct cma_device { struct list_head list; struct ib_device *device; struct completion comp; atomic_t refcount; struct list_head id_list; }; enum cma_state { CMA_IDLE, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY, CMA_ROUTE_RESOLVED, CMA_CONNECT, CMA_DISCONNECT, CMA_ADDR_BOUND, CMA_LISTEN, CMA_DEVICE_REMOVAL, CMA_DESTROYING }; struct rdma_bind_list { struct idr *ps; struct hlist_head owners; unsigned short port; }; /* * Device removal can occur at anytime, so we need extra handling to * serialize notifying the user of device removal with other callbacks. * We do this by disabling removal notification while a callback is in process, * and reporting it after the callback completes. */ struct rdma_id_private { struct rdma_cm_id id; struct rdma_bind_list *bind_list; struct socket *sock; struct hlist_node node; struct list_head list; /* listen_any_list or cma_device.list */ struct list_head listen_list; /* per device listens */ struct cma_device *cma_dev; struct list_head mc_list; int internal_id; enum cma_state state; spinlock_t lock; struct mutex qp_mutex; struct completion comp; atomic_t refcount; struct mutex handler_mutex; int backlog; int timeout_ms; struct ib_sa_query *query; int query_id; union { struct ib_cm_id *ib; struct iw_cm_id *iw; } cm_id; u32 seq_num; u32 qkey; u32 qp_num; u8 srq; u8 tos; }; struct cma_multicast { struct rdma_id_private *id_priv; union { struct ib_sa_multicast *ib; } multicast; struct list_head list; void *context; struct sockaddr_storage addr; struct kref mcref; }; struct cma_work { struct work_struct work; struct rdma_id_private *id; enum cma_state old_state; enum cma_state new_state; struct rdma_cm_event event; }; struct cma_ndev_work { struct work_struct work; struct rdma_id_private *id; struct rdma_cm_event event; }; struct iboe_mcast_work { struct work_struct work; struct rdma_id_private *id; struct cma_multicast *mc; }; union cma_ip_addr { struct in6_addr ip6; struct { __be32 pad[3]; __be32 addr; } ip4; }; struct cma_hdr { u8 cma_version; u8 ip_version; /* IP version: 7:4 */ __be16 port; union cma_ip_addr src_addr; union cma_ip_addr dst_addr; }; struct sdp_hh { u8 bsdh[16]; u8 sdp_version; /* Major version: 7:4 */ u8 ip_version; /* IP version: 7:4 */ u8 sdp_specific1[10]; __be16 port; __be16 sdp_specific2; union cma_ip_addr src_addr; union cma_ip_addr dst_addr; }; struct sdp_hah { u8 bsdh[16]; u8 sdp_version; }; #define CMA_VERSION 0x00 #define SDP_MAJ_VERSION 0x2 static int cma_comp(struct rdma_id_private *id_priv, enum cma_state comp) { unsigned long flags; int ret; spin_lock_irqsave(&id_priv->lock, flags); ret = (id_priv->state == comp); spin_unlock_irqrestore(&id_priv->lock, flags); return ret; } static int cma_comp_exch(struct rdma_id_private *id_priv, enum cma_state comp, enum cma_state exch) { unsigned long flags; int ret; spin_lock_irqsave(&id_priv->lock, flags); if ((ret = (id_priv->state == comp))) id_priv->state = exch; spin_unlock_irqrestore(&id_priv->lock, flags); return ret; } static enum cma_state cma_exch(struct rdma_id_private *id_priv, enum cma_state exch) { unsigned long flags; enum cma_state old; spin_lock_irqsave(&id_priv->lock, flags); old = id_priv->state; id_priv->state = exch; spin_unlock_irqrestore(&id_priv->lock, flags); return old; } static inline u8 cma_get_ip_ver(struct cma_hdr *hdr) { return hdr->ip_version >> 4; } static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver) { hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF); } static inline u8 sdp_get_majv(u8 sdp_version) { return sdp_version >> 4; } static inline u8 sdp_get_ip_ver(struct sdp_hh *hh) { return hh->ip_version >> 4; } static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver) { hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF); } static inline int cma_is_ud_ps(enum rdma_port_space ps) { return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB); } static void cma_attach_to_dev(struct rdma_id_private *id_priv, struct cma_device *cma_dev) { atomic_inc(&cma_dev->refcount); id_priv->cma_dev = cma_dev; id_priv->id.device = cma_dev->device; id_priv->id.route.addr.dev_addr.transport = rdma_node_get_transport(cma_dev->device->node_type); list_add_tail(&id_priv->list, &cma_dev->id_list); } static inline void cma_deref_dev(struct cma_device *cma_dev) { if (atomic_dec_and_test(&cma_dev->refcount)) complete(&cma_dev->comp); } static inline void release_mc(struct kref *kref) { struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref); kfree(mc->multicast.ib); kfree(mc); } static void cma_detach_from_dev(struct rdma_id_private *id_priv) { list_del(&id_priv->list); cma_deref_dev(id_priv->cma_dev); id_priv->cma_dev = NULL; } static int cma_set_qkey(struct rdma_id_private *id_priv) { struct ib_sa_mcmember_rec rec; int ret = 0; if (id_priv->qkey) return 0; switch (id_priv->id.ps) { case RDMA_PS_UDP: id_priv->qkey = RDMA_UDP_QKEY; break; case RDMA_PS_IPOIB: ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid); ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, &rec.mgid, &rec); if (!ret) id_priv->qkey = be32_to_cpu(rec.qkey); break; default: break; } return ret; } static int cma_acquire_dev(struct rdma_id_private *id_priv) { struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; struct cma_device *cma_dev; union ib_gid gid; int ret = -ENODEV; if (dev_addr->dev_type != ARPHRD_INFINIBAND) { iboe_addr_get_sgid(dev_addr, &gid); list_for_each_entry(cma_dev, &dev_list, list) { ret = ib_find_cached_gid(cma_dev->device, &gid, &id_priv->id.port_num, NULL); if (!ret) goto out; } } memcpy(&gid, dev_addr->src_dev_addr + rdma_addr_gid_offset(dev_addr), sizeof gid); list_for_each_entry(cma_dev, &dev_list, list) { ret = ib_find_cached_gid(cma_dev->device, &gid, &id_priv->id.port_num, NULL); if (!ret) break; } out: if (!ret) cma_attach_to_dev(id_priv, cma_dev); return ret; } static void cma_deref_id(struct rdma_id_private *id_priv) { if (atomic_dec_and_test(&id_priv->refcount)) complete(&id_priv->comp); } static int cma_disable_callback(struct rdma_id_private *id_priv, enum cma_state state) { mutex_lock(&id_priv->handler_mutex); if (id_priv->state != state) { mutex_unlock(&id_priv->handler_mutex); return -EINVAL; } return 0; } static int cma_has_cm_dev(struct rdma_id_private *id_priv) { return (id_priv->id.device && id_priv->cm_id.ib); } struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler, void *context, enum rdma_port_space ps) { struct rdma_id_private *id_priv; id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL); if (!id_priv) return ERR_PTR(-ENOMEM); id_priv->state = CMA_IDLE; id_priv->id.context = context; id_priv->id.event_handler = event_handler; id_priv->id.ps = ps; spin_lock_init(&id_priv->lock); mutex_init(&id_priv->qp_mutex); init_completion(&id_priv->comp); atomic_set(&id_priv->refcount, 1); mutex_init(&id_priv->handler_mutex); INIT_LIST_HEAD(&id_priv->listen_list); INIT_LIST_HEAD(&id_priv->mc_list); get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num); return &id_priv->id; } EXPORT_SYMBOL(rdma_create_id); static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) { struct ib_qp_attr qp_attr; int qp_attr_mask, ret; qp_attr.qp_state = IB_QPS_INIT; ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); if (ret) return ret; ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); if (ret) return ret; qp_attr.qp_state = IB_QPS_RTR; ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE); if (ret) return ret; qp_attr.qp_state = IB_QPS_RTS; qp_attr.sq_psn = 0; ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN); return ret; } static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) { struct ib_qp_attr qp_attr; int qp_attr_mask, ret; qp_attr.qp_state = IB_QPS_INIT; ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); if (ret) return ret; return ib_modify_qp(qp, &qp_attr, qp_attr_mask); } int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd, struct ib_qp_init_attr *qp_init_attr) { struct rdma_id_private *id_priv; struct ib_qp *qp; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (id->device != pd->device) return -EINVAL; qp = ib_create_qp(pd, qp_init_attr); if (IS_ERR(qp)) return PTR_ERR(qp); if (cma_is_ud_ps(id_priv->id.ps)) ret = cma_init_ud_qp(id_priv, qp); else ret = cma_init_conn_qp(id_priv, qp); if (ret) goto err; id->qp = qp; id_priv->qp_num = qp->qp_num; id_priv->srq = (qp->srq != NULL); return 0; err: ib_destroy_qp(qp); return ret; } EXPORT_SYMBOL(rdma_create_qp); void rdma_destroy_qp(struct rdma_cm_id *id) { struct rdma_id_private *id_priv; id_priv = container_of(id, struct rdma_id_private, id); mutex_lock(&id_priv->qp_mutex); ib_destroy_qp(id_priv->id.qp); id_priv->id.qp = NULL; mutex_unlock(&id_priv->qp_mutex); } EXPORT_SYMBOL(rdma_destroy_qp); static int cma_modify_qp_rtr(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct ib_qp_attr qp_attr; int qp_attr_mask, ret; mutex_lock(&id_priv->qp_mutex); if (!id_priv->id.qp) { ret = 0; goto out; } /* Need to update QP attributes from default values. */ qp_attr.qp_state = IB_QPS_INIT; ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); if (ret) goto out; ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); if (ret) goto out; qp_attr.qp_state = IB_QPS_RTR; ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); if (ret) goto out; if (conn_param) qp_attr.max_dest_rd_atomic = conn_param->responder_resources; ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); out: mutex_unlock(&id_priv->qp_mutex); return ret; } static int cma_modify_qp_rts(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct ib_qp_attr qp_attr; int qp_attr_mask, ret; mutex_lock(&id_priv->qp_mutex); if (!id_priv->id.qp) { ret = 0; goto out; } qp_attr.qp_state = IB_QPS_RTS; ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); if (ret) goto out; if (conn_param) qp_attr.max_rd_atomic = conn_param->initiator_depth; ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); out: mutex_unlock(&id_priv->qp_mutex); return ret; } static int cma_modify_qp_err(struct rdma_id_private *id_priv) { struct ib_qp_attr qp_attr; int ret; mutex_lock(&id_priv->qp_mutex); if (!id_priv->id.qp) { ret = 0; goto out; } qp_attr.qp_state = IB_QPS_ERR; ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE); out: mutex_unlock(&id_priv->qp_mutex); return ret; } static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; int ret; u16 pkey; if (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num) == IB_LINK_LAYER_INFINIBAND) pkey = ib_addr_get_pkey(dev_addr); else pkey = 0xffff; ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num, pkey, &qp_attr->pkey_index); if (ret) return ret; qp_attr->port_num = id_priv->id.port_num; *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT; if (cma_is_ud_ps(id_priv->id.ps)) { ret = cma_set_qkey(id_priv); if (ret) return ret; qp_attr->qkey = id_priv->qkey; *qp_attr_mask |= IB_QP_QKEY; } else { qp_attr->qp_access_flags = 0; *qp_attr_mask |= IB_QP_ACCESS_FLAGS; } return 0; } int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { struct rdma_id_private *id_priv; int ret = 0; id_priv = container_of(id, struct rdma_id_private, id); switch (rdma_node_get_transport(id_priv->id.device->node_type)) { case RDMA_TRANSPORT_IB: if (!id_priv->cm_id.ib || cma_is_ud_ps(id_priv->id.ps)) ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask); else ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr, qp_attr_mask); if (qp_attr->qp_state == IB_QPS_RTR) qp_attr->rq_psn = id_priv->seq_num; break; case RDMA_TRANSPORT_IWARP: if (!id_priv->cm_id.iw) { qp_attr->qp_access_flags = 0; *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS; } else ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr, qp_attr_mask); break; default: ret = -ENOSYS; break; } return ret; } EXPORT_SYMBOL(rdma_init_qp_attr); static inline int cma_zero_addr(struct sockaddr *addr) { struct in6_addr *ip6; if (addr->sa_family == AF_INET) return ipv4_is_zeronet( ((struct sockaddr_in *)addr)->sin_addr.s_addr); else { ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr; return (ip6->s6_addr32[0] | ip6->s6_addr32[1] | ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0; } } static inline int cma_loopback_addr(struct sockaddr *addr) { if (addr->sa_family == AF_INET) return ipv4_is_loopback( ((struct sockaddr_in *) addr)->sin_addr.s_addr); else return ipv6_addr_loopback( &((struct sockaddr_in6 *) addr)->sin6_addr); } static inline int cma_any_addr(struct sockaddr *addr) { return cma_zero_addr(addr) || cma_loopback_addr(addr); } static inline __be16 cma_port(struct sockaddr *addr) { if (addr->sa_family == AF_INET) return ((struct sockaddr_in *) addr)->sin_port; else return ((struct sockaddr_in6 *) addr)->sin6_port; } static inline int cma_any_port(struct sockaddr *addr) { return !cma_port(addr); } static int cma_get_net_info(void *hdr, enum rdma_port_space ps, u8 *ip_ver, __be16 *port, union cma_ip_addr **src, union cma_ip_addr **dst) { switch (ps) { case RDMA_PS_SDP: if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) != SDP_MAJ_VERSION) return -EINVAL; *ip_ver = sdp_get_ip_ver(hdr); *port = ((struct sdp_hh *) hdr)->port; *src = &((struct sdp_hh *) hdr)->src_addr; *dst = &((struct sdp_hh *) hdr)->dst_addr; break; default: if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION) return -EINVAL; *ip_ver = cma_get_ip_ver(hdr); *port = ((struct cma_hdr *) hdr)->port; *src = &((struct cma_hdr *) hdr)->src_addr; *dst = &((struct cma_hdr *) hdr)->dst_addr; break; } if (*ip_ver != 4 && *ip_ver != 6) return -EINVAL; return 0; } static void cma_save_net_info(struct rdma_addr *addr, struct rdma_addr *listen_addr, u8 ip_ver, __be16 port, union cma_ip_addr *src, union cma_ip_addr *dst) { struct sockaddr_in *listen4, *ip4; struct sockaddr_in6 *listen6, *ip6; switch (ip_ver) { case 4: listen4 = (struct sockaddr_in *) &listen_addr->src_addr; ip4 = (struct sockaddr_in *) &addr->src_addr; ip4->sin_family = listen4->sin_family; ip4->sin_addr.s_addr = dst->ip4.addr; ip4->sin_port = listen4->sin_port; ip4 = (struct sockaddr_in *) &addr->dst_addr; ip4->sin_family = listen4->sin_family; ip4->sin_addr.s_addr = src->ip4.addr; ip4->sin_port = port; break; case 6: listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr; ip6 = (struct sockaddr_in6 *) &addr->src_addr; ip6->sin6_family = listen6->sin6_family; ip6->sin6_addr = dst->ip6; ip6->sin6_port = listen6->sin6_port; ip6 = (struct sockaddr_in6 *) &addr->dst_addr; ip6->sin6_family = listen6->sin6_family; ip6->sin6_addr = src->ip6; ip6->sin6_port = port; break; default: break; } } static inline int cma_user_data_offset(enum rdma_port_space ps) { switch (ps) { case RDMA_PS_SDP: return 0; default: return sizeof(struct cma_hdr); } } static void cma_cancel_route(struct rdma_id_private *id_priv) { switch (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)) { case IB_LINK_LAYER_INFINIBAND: if (id_priv->query) ib_sa_cancel_query(id_priv->query_id, id_priv->query); break; default: break; } } static void cma_cancel_listens(struct rdma_id_private *id_priv) { struct rdma_id_private *dev_id_priv; /* * Remove from listen_any_list to prevent added devices from spawning * additional listen requests. */ mutex_lock(&lock); list_del(&id_priv->list); while (!list_empty(&id_priv->listen_list)) { dev_id_priv = list_entry(id_priv->listen_list.next, struct rdma_id_private, listen_list); /* sync with device removal to avoid duplicate destruction */ list_del_init(&dev_id_priv->list); list_del(&dev_id_priv->listen_list); mutex_unlock(&lock); rdma_destroy_id(&dev_id_priv->id); mutex_lock(&lock); } mutex_unlock(&lock); } static void cma_cancel_operation(struct rdma_id_private *id_priv, enum cma_state state) { switch (state) { case CMA_ADDR_QUERY: rdma_addr_cancel(&id_priv->id.route.addr.dev_addr); break; case CMA_ROUTE_QUERY: cma_cancel_route(id_priv); break; case CMA_LISTEN: if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr) && !id_priv->cma_dev) cma_cancel_listens(id_priv); break; default: break; } } static void cma_release_port(struct rdma_id_private *id_priv) { struct rdma_bind_list *bind_list = id_priv->bind_list; if (!bind_list) return; mutex_lock(&lock); hlist_del(&id_priv->node); if (hlist_empty(&bind_list->owners)) { idr_remove(bind_list->ps, bind_list->port); kfree(bind_list); } mutex_unlock(&lock); if (id_priv->sock) sock_release(id_priv->sock); } static void cma_leave_mc_groups(struct rdma_id_private *id_priv) { struct cma_multicast *mc; while (!list_empty(&id_priv->mc_list)) { mc = container_of(id_priv->mc_list.next, struct cma_multicast, list); list_del(&mc->list); switch (rdma_port_get_link_layer(id_priv->cma_dev->device, id_priv->id.port_num)) { case IB_LINK_LAYER_INFINIBAND: ib_sa_free_multicast(mc->multicast.ib); kfree(mc); break; case IB_LINK_LAYER_ETHERNET: kref_put(&mc->mcref, release_mc); break; default: break; } } } void rdma_destroy_id(struct rdma_cm_id *id) { struct rdma_id_private *id_priv; enum cma_state state; id_priv = container_of(id, struct rdma_id_private, id); state = cma_exch(id_priv, CMA_DESTROYING); cma_cancel_operation(id_priv, state); mutex_lock(&lock); if (id_priv->cma_dev) { mutex_unlock(&lock); switch (rdma_node_get_transport(id_priv->id.device->node_type)) { case RDMA_TRANSPORT_IB: if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib)) ib_destroy_cm_id(id_priv->cm_id.ib); break; case RDMA_TRANSPORT_IWARP: if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw)) iw_destroy_cm_id(id_priv->cm_id.iw); break; default: break; } cma_leave_mc_groups(id_priv); mutex_lock(&lock); cma_detach_from_dev(id_priv); } mutex_unlock(&lock); cma_release_port(id_priv); cma_deref_id(id_priv); wait_for_completion(&id_priv->comp); if (id_priv->internal_id) cma_deref_id(id_priv->id.context); kfree(id_priv->id.route.path_rec); kfree(id_priv); } EXPORT_SYMBOL(rdma_destroy_id); static int cma_rep_recv(struct rdma_id_private *id_priv) { int ret; ret = cma_modify_qp_rtr(id_priv, NULL); if (ret) goto reject; ret = cma_modify_qp_rts(id_priv, NULL); if (ret) goto reject; ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0); if (ret) goto reject; return 0; reject: cma_modify_qp_err(id_priv); ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0, NULL, 0); return ret; } static int cma_verify_rep(struct rdma_id_private *id_priv, void *data) { if (id_priv->id.ps == RDMA_PS_SDP && sdp_get_majv(((struct sdp_hah *) data)->sdp_version) != SDP_MAJ_VERSION) return -EINVAL; return 0; } static void cma_set_rep_event_data(struct rdma_cm_event *event, struct ib_cm_rep_event_param *rep_data, void *private_data) { event->param.conn.private_data = private_data; event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE; event->param.conn.responder_resources = rep_data->responder_resources; event->param.conn.initiator_depth = rep_data->initiator_depth; event->param.conn.flow_control = rep_data->flow_control; event->param.conn.rnr_retry_count = rep_data->rnr_retry_count; event->param.conn.srq = rep_data->srq; event->param.conn.qp_num = rep_data->remote_qpn; } static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) { struct rdma_id_private *id_priv = cm_id->context; struct rdma_cm_event event; int ret = 0; if ((ib_event->event != IB_CM_TIMEWAIT_EXIT && cma_disable_callback(id_priv, CMA_CONNECT)) || (ib_event->event == IB_CM_TIMEWAIT_EXIT && cma_disable_callback(id_priv, CMA_DISCONNECT))) return 0; memset(&event, 0, sizeof event); switch (ib_event->event) { case IB_CM_REQ_ERROR: case IB_CM_REP_ERROR: event.event = RDMA_CM_EVENT_UNREACHABLE; event.status = -ETIMEDOUT; break; case IB_CM_REP_RECEIVED: event.status = cma_verify_rep(id_priv, ib_event->private_data); if (event.status) event.event = RDMA_CM_EVENT_CONNECT_ERROR; else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) { event.status = cma_rep_recv(id_priv); event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR : RDMA_CM_EVENT_ESTABLISHED; } else event.event = RDMA_CM_EVENT_CONNECT_RESPONSE; cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd, ib_event->private_data); break; case IB_CM_RTU_RECEIVED: case IB_CM_USER_ESTABLISHED: event.event = RDMA_CM_EVENT_ESTABLISHED; break; case IB_CM_DREQ_ERROR: event.status = -ETIMEDOUT; /* fall through */ case IB_CM_DREQ_RECEIVED: case IB_CM_DREP_RECEIVED: if (!cma_comp_exch(id_priv, CMA_CONNECT, CMA_DISCONNECT)) goto out; event.event = RDMA_CM_EVENT_DISCONNECTED; break; case IB_CM_TIMEWAIT_EXIT: event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT; break; case IB_CM_MRA_RECEIVED: /* ignore event */ goto out; case IB_CM_REJ_RECEIVED: cma_modify_qp_err(id_priv); event.status = ib_event->param.rej_rcvd.reason; event.event = RDMA_CM_EVENT_REJECTED; event.param.conn.private_data = ib_event->private_data; event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE; break; default: printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n", ib_event->event); goto out; } ret = id_priv->id.event_handler(&id_priv->id, &event); if (ret) { /* Destroy the CM ID by returning a non-zero value. */ id_priv->cm_id.ib = NULL; cma_exch(id_priv, CMA_DESTROYING); mutex_unlock(&id_priv->handler_mutex); rdma_destroy_id(&id_priv->id); return ret; } out: mutex_unlock(&id_priv->handler_mutex); return ret; } static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id, struct ib_cm_event *ib_event) { struct rdma_id_private *id_priv; struct rdma_cm_id *id; struct rdma_route *rt; union cma_ip_addr *src, *dst; __be16 port; u8 ip_ver; int ret; if (cma_get_net_info(ib_event->private_data, listen_id->ps, &ip_ver, &port, &src, &dst)) goto err; id = rdma_create_id(listen_id->event_handler, listen_id->context, listen_id->ps); if (IS_ERR(id)) goto err; cma_save_net_info(&id->route.addr, &listen_id->route.addr, ip_ver, port, src, dst); rt = &id->route; rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1; rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths, GFP_KERNEL); if (!rt->path_rec) goto destroy_id; rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path; if (rt->num_paths == 2) rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path; if (cma_any_addr((struct sockaddr *) &rt->addr.src_addr)) { rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND; rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid); ib_addr_set_pkey(&rt->addr.dev_addr, rt->path_rec[0].pkey); } else { ret = rdma_translate_ip((struct sockaddr *) &rt->addr.src_addr, &rt->addr.dev_addr); if (ret) goto destroy_id; } rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid); id_priv = container_of(id, struct rdma_id_private, id); id_priv->state = CMA_CONNECT; return id_priv; destroy_id: rdma_destroy_id(id); err: return NULL; } static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id, struct ib_cm_event *ib_event) { struct rdma_id_private *id_priv; struct rdma_cm_id *id; union cma_ip_addr *src, *dst; __be16 port; u8 ip_ver; int ret; id = rdma_create_id(listen_id->event_handler, listen_id->context, listen_id->ps); if (IS_ERR(id)) return NULL; if (cma_get_net_info(ib_event->private_data, listen_id->ps, &ip_ver, &port, &src, &dst)) goto err; cma_save_net_info(&id->route.addr, &listen_id->route.addr, ip_ver, port, src, dst); if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) { ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr, &id->route.addr.dev_addr); if (ret) goto err; } id_priv = container_of(id, struct rdma_id_private, id); id_priv->state = CMA_CONNECT; return id_priv; err: rdma_destroy_id(id); return NULL; } static void cma_set_req_event_data(struct rdma_cm_event *event, struct ib_cm_req_event_param *req_data, void *private_data, int offset) { event->param.conn.private_data = private_data + offset; event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset; event->param.conn.responder_resources = req_data->responder_resources; event->param.conn.initiator_depth = req_data->initiator_depth; event->param.conn.flow_control = req_data->flow_control; event->param.conn.retry_count = req_data->retry_count; event->param.conn.rnr_retry_count = req_data->rnr_retry_count; event->param.conn.srq = req_data->srq; event->param.conn.qp_num = req_data->remote_qpn; } static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) { struct rdma_id_private *listen_id, *conn_id; struct rdma_cm_event event; int offset, ret; listen_id = cm_id->context; if (cma_disable_callback(listen_id, CMA_LISTEN)) return -ECONNABORTED; memset(&event, 0, sizeof event); offset = cma_user_data_offset(listen_id->id.ps); event.event = RDMA_CM_EVENT_CONNECT_REQUEST; if (cma_is_ud_ps(listen_id->id.ps)) { conn_id = cma_new_udp_id(&listen_id->id, ib_event); event.param.ud.private_data = ib_event->private_data + offset; event.param.ud.private_data_len = IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset; } else { conn_id = cma_new_conn_id(&listen_id->id, ib_event); cma_set_req_event_data(&event, &ib_event->param.req_rcvd, ib_event->private_data, offset); } if (!conn_id) { ret = -ENOMEM; goto out; } mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); mutex_lock(&lock); ret = cma_acquire_dev(conn_id); mutex_unlock(&lock); if (ret) goto release_conn_id; conn_id->cm_id.ib = cm_id; cm_id->context = conn_id; cm_id->cm_handler = cma_ib_handler; ret = conn_id->id.event_handler(&conn_id->id, &event); if (!ret) { /* * Acquire mutex to prevent user executing rdma_destroy_id() * while we're accessing the cm_id. */ mutex_lock(&lock); if (cma_comp(conn_id, CMA_CONNECT) && !cma_is_ud_ps(conn_id->id.ps)) ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); mutex_unlock(&lock); mutex_unlock(&conn_id->handler_mutex); goto out; } /* Destroy the CM ID by returning a non-zero value. */ conn_id->cm_id.ib = NULL; release_conn_id: cma_exch(conn_id, CMA_DESTROYING); mutex_unlock(&conn_id->handler_mutex); rdma_destroy_id(&conn_id->id); out: mutex_unlock(&listen_id->handler_mutex); return ret; } static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr) { return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr))); } static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr, struct ib_cm_compare_data *compare) { struct cma_hdr *cma_data, *cma_mask; struct sdp_hh *sdp_data, *sdp_mask; __be32 ip4_addr; #ifdef INET6 struct in6_addr ip6_addr; #endif memset(compare, 0, sizeof *compare); cma_data = (void *) compare->data; cma_mask = (void *) compare->mask; sdp_data = (void *) compare->data; sdp_mask = (void *) compare->mask; switch (addr->sa_family) { case AF_INET: ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr; if (ps == RDMA_PS_SDP) { sdp_set_ip_ver(sdp_data, 4); sdp_set_ip_ver(sdp_mask, 0xF); sdp_data->dst_addr.ip4.addr = ip4_addr; sdp_mask->dst_addr.ip4.addr = htonl(~0); } else { cma_set_ip_ver(cma_data, 4); cma_set_ip_ver(cma_mask, 0xF); cma_data->dst_addr.ip4.addr = ip4_addr; cma_mask->dst_addr.ip4.addr = htonl(~0); } break; #ifdef INET6 case AF_INET6: ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr; if (ps == RDMA_PS_SDP) { sdp_set_ip_ver(sdp_data, 6); sdp_set_ip_ver(sdp_mask, 0xF); sdp_data->dst_addr.ip6 = ip6_addr; memset(&sdp_mask->dst_addr.ip6, 0xFF, sizeof sdp_mask->dst_addr.ip6); } else { cma_set_ip_ver(cma_data, 6); cma_set_ip_ver(cma_mask, 0xF); cma_data->dst_addr.ip6 = ip6_addr; memset(&cma_mask->dst_addr.ip6, 0xFF, sizeof cma_mask->dst_addr.ip6); } break; #endif default: break; } } static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) { struct rdma_id_private *id_priv = iw_id->context; struct rdma_cm_event event; struct sockaddr_in *sin; int ret = 0; if (cma_disable_callback(id_priv, CMA_CONNECT)) return 0; memset(&event, 0, sizeof event); switch (iw_event->event) { case IW_CM_EVENT_CLOSE: event.event = RDMA_CM_EVENT_DISCONNECTED; break; case IW_CM_EVENT_CONNECT_REPLY: sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; *sin = iw_event->local_addr; sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr; *sin = iw_event->remote_addr; switch ((int)iw_event->status) { case 0: event.event = RDMA_CM_EVENT_ESTABLISHED; break; case -ECONNRESET: case -ECONNREFUSED: event.event = RDMA_CM_EVENT_REJECTED; break; case -ETIMEDOUT: event.event = RDMA_CM_EVENT_UNREACHABLE; break; default: event.event = RDMA_CM_EVENT_CONNECT_ERROR; break; } break; case IW_CM_EVENT_ESTABLISHED: event.event = RDMA_CM_EVENT_ESTABLISHED; break; default: BUG_ON(1); } event.status = iw_event->status; event.param.conn.private_data = iw_event->private_data; event.param.conn.private_data_len = iw_event->private_data_len; ret = id_priv->id.event_handler(&id_priv->id, &event); if (ret) { /* Destroy the CM ID by returning a non-zero value. */ id_priv->cm_id.iw = NULL; cma_exch(id_priv, CMA_DESTROYING); mutex_unlock(&id_priv->handler_mutex); rdma_destroy_id(&id_priv->id); return ret; } mutex_unlock(&id_priv->handler_mutex); return ret; } static int iw_conn_req_handler(struct iw_cm_id *cm_id, struct iw_cm_event *iw_event) { struct rdma_cm_id *new_cm_id; struct rdma_id_private *listen_id, *conn_id; struct sockaddr_in *sin; struct net_device *dev = NULL; struct rdma_cm_event event; int ret; struct ib_device_attr attr; listen_id = cm_id->context; if (cma_disable_callback(listen_id, CMA_LISTEN)) return -ECONNABORTED; /* Create a new RDMA id for the new IW CM ID */ new_cm_id = rdma_create_id(listen_id->id.event_handler, listen_id->id.context, RDMA_PS_TCP); if (IS_ERR(new_cm_id)) { ret = -ENOMEM; goto out; } conn_id = container_of(new_cm_id, struct rdma_id_private, id); mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); conn_id->state = CMA_CONNECT; dev = ip_dev_find(NULL, iw_event->local_addr.sin_addr.s_addr); if (!dev) { ret = -EADDRNOTAVAIL; mutex_unlock(&conn_id->handler_mutex); rdma_destroy_id(new_cm_id); goto out; } ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL); if (ret) { mutex_unlock(&conn_id->handler_mutex); rdma_destroy_id(new_cm_id); goto out; } mutex_lock(&lock); ret = cma_acquire_dev(conn_id); mutex_unlock(&lock); if (ret) { mutex_unlock(&conn_id->handler_mutex); rdma_destroy_id(new_cm_id); goto out; } conn_id->cm_id.iw = cm_id; cm_id->context = conn_id; cm_id->cm_handler = cma_iw_handler; sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr; *sin = iw_event->local_addr; sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr; *sin = iw_event->remote_addr; ret = ib_query_device(conn_id->id.device, &attr); if (ret) { mutex_unlock(&conn_id->handler_mutex); rdma_destroy_id(new_cm_id); goto out; } memset(&event, 0, sizeof event); event.event = RDMA_CM_EVENT_CONNECT_REQUEST; event.param.conn.private_data = iw_event->private_data; event.param.conn.private_data_len = iw_event->private_data_len; event.param.conn.initiator_depth = attr.max_qp_init_rd_atom; event.param.conn.responder_resources = attr.max_qp_rd_atom; ret = conn_id->id.event_handler(&conn_id->id, &event); if (ret) { /* User wants to destroy the CM ID */ conn_id->cm_id.iw = NULL; cma_exch(conn_id, CMA_DESTROYING); mutex_unlock(&conn_id->handler_mutex); rdma_destroy_id(&conn_id->id); goto out; } mutex_unlock(&conn_id->handler_mutex); out: if (dev) dev_put(dev); mutex_unlock(&listen_id->handler_mutex); return ret; } static int cma_ib_listen(struct rdma_id_private *id_priv) { struct ib_cm_compare_data compare_data; struct sockaddr *addr; __be64 svc_id; int ret; id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_req_handler, id_priv); if (IS_ERR(id_priv->cm_id.ib)) return PTR_ERR(id_priv->cm_id.ib); addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr; svc_id = cma_get_service_id(id_priv->id.ps, addr); if (cma_any_addr(addr)) ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL); else { cma_set_compare_data(id_priv->id.ps, addr, &compare_data); ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data); } if (ret) { ib_destroy_cm_id(id_priv->cm_id.ib); id_priv->cm_id.ib = NULL; } return ret; } static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog) { int ret; struct sockaddr_in *sin; id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device, id_priv->sock, iw_conn_req_handler, id_priv); if (IS_ERR(id_priv->cm_id.iw)) return PTR_ERR(id_priv->cm_id.iw); sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; id_priv->cm_id.iw->local_addr = *sin; ret = iw_cm_listen(id_priv->cm_id.iw, backlog); if (ret) { iw_destroy_cm_id(id_priv->cm_id.iw); id_priv->cm_id.iw = NULL; } return ret; } static int cma_listen_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) { struct rdma_id_private *id_priv = id->context; id->context = id_priv->id.context; id->event_handler = id_priv->id.event_handler; return id_priv->id.event_handler(id, event); } static void cma_listen_on_dev(struct rdma_id_private *id_priv, struct cma_device *cma_dev) { struct rdma_id_private *dev_id_priv; struct rdma_cm_id *id; int ret; id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps); if (IS_ERR(id)) return; dev_id_priv = container_of(id, struct rdma_id_private, id); dev_id_priv->state = CMA_ADDR_BOUND; memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr, ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr)); cma_attach_to_dev(dev_id_priv, cma_dev); list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list); atomic_inc(&id_priv->refcount); dev_id_priv->internal_id = 1; ret = rdma_listen(id, id_priv->backlog); if (ret) printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, " "listening on device %s\n", ret, cma_dev->device->name); } static void cma_listen_on_all(struct rdma_id_private *id_priv) { struct cma_device *cma_dev; mutex_lock(&lock); list_add_tail(&id_priv->list, &listen_any_list); list_for_each_entry(cma_dev, &dev_list, list) cma_listen_on_dev(id_priv, cma_dev); mutex_unlock(&lock); } int rdma_listen(struct rdma_cm_id *id, int backlog) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (id_priv->state == CMA_IDLE) { ((struct sockaddr *) &id->route.addr.src_addr)->sa_family = AF_INET; ret = rdma_bind_addr(id, (struct sockaddr *) &id->route.addr.src_addr); if (ret) return ret; } if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_LISTEN)) return -EINVAL; id_priv->backlog = backlog; if (id->device) { switch (rdma_node_get_transport(id->device->node_type)) { case RDMA_TRANSPORT_IB: ret = cma_ib_listen(id_priv); if (ret) goto err; break; case RDMA_TRANSPORT_IWARP: ret = cma_iw_listen(id_priv, backlog); if (ret) goto err; break; default: ret = -ENOSYS; goto err; } } else cma_listen_on_all(id_priv); return 0; err: id_priv->backlog = 0; cma_comp_exch(id_priv, CMA_LISTEN, CMA_ADDR_BOUND); return ret; } EXPORT_SYMBOL(rdma_listen); void rdma_set_service_type(struct rdma_cm_id *id, int tos) { struct rdma_id_private *id_priv; id_priv = container_of(id, struct rdma_id_private, id); id_priv->tos = (u8) tos; } EXPORT_SYMBOL(rdma_set_service_type); static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec, void *context) { struct cma_work *work = context; struct rdma_route *route; route = &work->id->id.route; if (!status) { route->num_paths = 1; *route->path_rec = *path_rec; } else { work->old_state = CMA_ROUTE_QUERY; work->new_state = CMA_ADDR_RESOLVED; work->event.event = RDMA_CM_EVENT_ROUTE_ERROR; work->event.status = status; } queue_work(cma_wq, &work->work); } static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms, struct cma_work *work) { struct rdma_addr *addr = &id_priv->id.route.addr; struct ib_sa_path_rec path_rec; ib_sa_comp_mask comp_mask; struct sockaddr_in6 *sin6; memset(&path_rec, 0, sizeof path_rec); rdma_addr_get_sgid(&addr->dev_addr, &path_rec.sgid); rdma_addr_get_dgid(&addr->dev_addr, &path_rec.dgid); path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr)); path_rec.numb_path = 1; path_rec.reversible = 1; path_rec.service_id = cma_get_service_id(id_priv->id.ps, (struct sockaddr *) &addr->dst_addr); comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID | IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH | IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID; if (addr->src_addr.ss_family == AF_INET) { path_rec.qos_class = cpu_to_be16((u16) id_priv->tos); comp_mask |= IB_SA_PATH_REC_QOS_CLASS; } else { sin6 = (struct sockaddr_in6 *) &addr->src_addr; path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20); comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS; } if (tavor_quirk) { path_rec.mtu_selector = IB_SA_LT; path_rec.mtu = IB_MTU_2048; } id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device, id_priv->id.port_num, &path_rec, comp_mask, timeout_ms, GFP_KERNEL, cma_query_handler, work, &id_priv->query); return (id_priv->query_id < 0) ? id_priv->query_id : 0; } static void cma_work_handler(struct work_struct *_work) { struct cma_work *work = container_of(_work, struct cma_work, work); struct rdma_id_private *id_priv = work->id; int destroy = 0; mutex_lock(&id_priv->handler_mutex); if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) goto out; if (id_priv->id.event_handler(&id_priv->id, &work->event)) { cma_exch(id_priv, CMA_DESTROYING); destroy = 1; } out: mutex_unlock(&id_priv->handler_mutex); cma_deref_id(id_priv); if (destroy) rdma_destroy_id(&id_priv->id); kfree(work); } static void cma_ndev_work_handler(struct work_struct *_work) { struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work); struct rdma_id_private *id_priv = work->id; int destroy = 0; mutex_lock(&id_priv->handler_mutex); if (id_priv->state == CMA_DESTROYING || id_priv->state == CMA_DEVICE_REMOVAL) goto out; if (id_priv->id.event_handler(&id_priv->id, &work->event)) { cma_exch(id_priv, CMA_DESTROYING); destroy = 1; } out: mutex_unlock(&id_priv->handler_mutex); cma_deref_id(id_priv); if (destroy) rdma_destroy_id(&id_priv->id); kfree(work); } static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms) { struct rdma_route *route = &id_priv->id.route; struct cma_work *work; int ret; work = kzalloc(sizeof *work, GFP_KERNEL); if (!work) return -ENOMEM; work->id = id_priv; INIT_WORK(&work->work, cma_work_handler); work->old_state = CMA_ROUTE_QUERY; work->new_state = CMA_ROUTE_RESOLVED; work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL); if (!route->path_rec) { ret = -ENOMEM; goto err1; } ret = cma_query_ib_route(id_priv, timeout_ms, work); if (ret) goto err2; return 0; err2: kfree(route->path_rec); route->path_rec = NULL; err1: kfree(work); return ret; } int rdma_set_ib_paths(struct rdma_cm_id *id, struct ib_sa_path_rec *path_rec, int num_paths) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_RESOLVED)) return -EINVAL; id->route.path_rec = kmalloc(sizeof *path_rec * num_paths, GFP_KERNEL); if (!id->route.path_rec) { ret = -ENOMEM; goto err; } memcpy(id->route.path_rec, path_rec, sizeof *path_rec * num_paths); return 0; err: cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_ADDR_RESOLVED); return ret; } EXPORT_SYMBOL(rdma_set_ib_paths); static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms) { struct cma_work *work; work = kzalloc(sizeof *work, GFP_KERNEL); if (!work) return -ENOMEM; work->id = id_priv; INIT_WORK(&work->work, cma_work_handler); work->old_state = CMA_ROUTE_QUERY; work->new_state = CMA_ROUTE_RESOLVED; work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; queue_work(cma_wq, &work->work); return 0; } static u8 tos_to_sl(u8 tos) { return def_prec2sl & 7; } static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) { struct rdma_route *route = &id_priv->id.route; struct rdma_addr *addr = &route->addr; struct cma_work *work; int ret; struct sockaddr_in *src_addr = (struct sockaddr_in *)&route->addr.src_addr; struct sockaddr_in *dst_addr = (struct sockaddr_in *)&route->addr.dst_addr; struct net_device *ndev = NULL; u16 vid; if (src_addr->sin_family != dst_addr->sin_family) return -EINVAL; work = kzalloc(sizeof *work, GFP_KERNEL); if (!work) return -ENOMEM; work->id = id_priv; INIT_WORK(&work->work, cma_work_handler); route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL); if (!route->path_rec) { ret = -ENOMEM; goto err1; } route->num_paths = 1; if (addr->dev_addr.bound_dev_if) ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if); if (!ndev) { ret = -ENODEV; goto err2; } vid = rdma_vlan_dev_vlan_id(ndev); iboe_mac_vlan_to_ll(&route->path_rec->sgid, addr->dev_addr.src_dev_addr, vid); iboe_mac_vlan_to_ll(&route->path_rec->dgid, addr->dev_addr.dst_dev_addr, vid); route->path_rec->hop_limit = 1; route->path_rec->reversible = 1; route->path_rec->pkey = cpu_to_be16(0xffff); route->path_rec->mtu_selector = IB_SA_EQ; route->path_rec->sl = tos_to_sl(id_priv->tos); #ifdef __linux__ route->path_rec->mtu = iboe_get_mtu(ndev->mtu); #else route->path_rec->mtu = iboe_get_mtu(ndev->if_mtu); #endif route->path_rec->rate_selector = IB_SA_EQ; route->path_rec->rate = iboe_get_rate(ndev); dev_put(ndev); route->path_rec->packet_life_time_selector = IB_SA_EQ; route->path_rec->packet_life_time = IBOE_PACKET_LIFETIME; if (!route->path_rec->mtu) { ret = -EINVAL; goto err2; } work->old_state = CMA_ROUTE_QUERY; work->new_state = CMA_ROUTE_RESOLVED; work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; work->event.status = 0; queue_work(cma_wq, &work->work); return 0; err2: kfree(route->path_rec); route->path_rec = NULL; err1: kfree(work); return ret; } int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY)) return -EINVAL; atomic_inc(&id_priv->refcount); switch (rdma_node_get_transport(id->device->node_type)) { case RDMA_TRANSPORT_IB: switch (rdma_port_get_link_layer(id->device, id->port_num)) { case IB_LINK_LAYER_INFINIBAND: ret = cma_resolve_ib_route(id_priv, timeout_ms); break; case IB_LINK_LAYER_ETHERNET: ret = cma_resolve_iboe_route(id_priv); break; default: ret = -ENOSYS; } break; case RDMA_TRANSPORT_IWARP: ret = cma_resolve_iw_route(id_priv, timeout_ms); break; default: ret = -ENOSYS; break; } if (ret) goto err; return 0; err: cma_comp_exch(id_priv, CMA_ROUTE_QUERY, CMA_ADDR_RESOLVED); cma_deref_id(id_priv); return ret; } EXPORT_SYMBOL(rdma_resolve_route); static int cma_bind_loopback(struct rdma_id_private *id_priv) { struct cma_device *cma_dev; struct ib_port_attr port_attr; union ib_gid gid; u16 pkey; int ret; u8 p; mutex_lock(&lock); if (list_empty(&dev_list)) { ret = -ENODEV; goto out; } list_for_each_entry(cma_dev, &dev_list, list) for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p) if (!ib_query_port(cma_dev->device, p, &port_attr) && port_attr.state == IB_PORT_ACTIVE) goto port_found; p = 1; cma_dev = list_entry(dev_list.next, struct cma_device, list); port_found: ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid); if (ret) goto out; ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey); if (ret) goto out; id_priv->id.route.addr.dev_addr.dev_type = (rdma_port_get_link_layer(cma_dev->device, p) == IB_LINK_LAYER_INFINIBAND) ? ARPHRD_INFINIBAND : ARPHRD_ETHER; rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid); ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey); id_priv->id.port_num = p; cma_attach_to_dev(id_priv, cma_dev); out: mutex_unlock(&lock); return ret; } static void addr_handler(int status, struct sockaddr *src_addr, struct rdma_dev_addr *dev_addr, void *context) { struct rdma_id_private *id_priv = context; struct rdma_cm_event event; memset(&event, 0, sizeof event); mutex_lock(&id_priv->handler_mutex); /* * Grab mutex to block rdma_destroy_id() from removing the device while * we're trying to acquire it. */ mutex_lock(&lock); if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED)) { mutex_unlock(&lock); goto out; } if (!status && !id_priv->cma_dev) status = cma_acquire_dev(id_priv); mutex_unlock(&lock); if (status) { if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ADDR_BOUND)) goto out; event.event = RDMA_CM_EVENT_ADDR_ERROR; event.status = status; } else { memcpy(&id_priv->id.route.addr.src_addr, src_addr, ip_addr_size(src_addr)); event.event = RDMA_CM_EVENT_ADDR_RESOLVED; } if (id_priv->id.event_handler(&id_priv->id, &event)) { cma_exch(id_priv, CMA_DESTROYING); mutex_unlock(&id_priv->handler_mutex); cma_deref_id(id_priv); rdma_destroy_id(&id_priv->id); return; } out: mutex_unlock(&id_priv->handler_mutex); cma_deref_id(id_priv); } static int cma_resolve_loopback(struct rdma_id_private *id_priv) { struct cma_work *work; struct sockaddr *src, *dst; union ib_gid gid; int ret; work = kzalloc(sizeof *work, GFP_KERNEL); if (!work) return -ENOMEM; if (!id_priv->cma_dev) { ret = cma_bind_loopback(id_priv); if (ret) goto err; } rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid); rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid); src = (struct sockaddr *) &id_priv->id.route.addr.src_addr; if (cma_zero_addr(src)) { dst = (struct sockaddr *) &id_priv->id.route.addr.dst_addr; if ((src->sa_family = dst->sa_family) == AF_INET) { ((struct sockaddr_in *) src)->sin_addr.s_addr = ((struct sockaddr_in *) dst)->sin_addr.s_addr; } else { ipv6_addr_copy(&((struct sockaddr_in6 *) src)->sin6_addr, &((struct sockaddr_in6 *) dst)->sin6_addr); } } work->id = id_priv; INIT_WORK(&work->work, cma_work_handler); work->old_state = CMA_ADDR_QUERY; work->new_state = CMA_ADDR_RESOLVED; work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; queue_work(cma_wq, &work->work); return 0; err: kfree(work); return ret; } static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, struct sockaddr *dst_addr) { if (!src_addr || !src_addr->sa_family) { src_addr = (struct sockaddr *) &id->route.addr.src_addr; if ((src_addr->sa_family = dst_addr->sa_family) == AF_INET6) { ((struct sockaddr_in6 *) src_addr)->sin6_scope_id = ((struct sockaddr_in6 *) dst_addr)->sin6_scope_id; } } if (!cma_any_addr(src_addr)) return rdma_bind_addr(id, src_addr); else { struct sockaddr_in addr_in; memset(&addr_in, 0, sizeof addr_in); addr_in.sin_family = dst_addr->sa_family; addr_in.sin_len = sizeof addr_in; return rdma_bind_addr(id, (struct sockaddr *) &addr_in); } } int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, struct sockaddr *dst_addr, int timeout_ms) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (id_priv->state == CMA_IDLE) { ret = cma_bind_addr(id, src_addr, dst_addr); if (ret) return ret; } if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_ADDR_QUERY)) return -EINVAL; atomic_inc(&id_priv->refcount); memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr)); if (cma_any_addr(dst_addr)) ret = cma_resolve_loopback(id_priv); else ret = rdma_resolve_ip(&addr_client, (struct sockaddr *) &id->route.addr.src_addr, dst_addr, &id->route.addr.dev_addr, timeout_ms, addr_handler, id_priv); if (ret) goto err; return 0; err: cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_BOUND); cma_deref_id(id_priv); return ret; } EXPORT_SYMBOL(rdma_resolve_addr); static void cma_bind_port(struct rdma_bind_list *bind_list, struct rdma_id_private *id_priv) { struct sockaddr_in *sin; sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; sin->sin_port = htons(bind_list->port); id_priv->bind_list = bind_list; hlist_add_head(&id_priv->node, &bind_list->owners); } static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv, unsigned short snum) { struct rdma_bind_list *bind_list; int port, ret; bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL); if (!bind_list) return -ENOMEM; do { ret = idr_get_new_above(ps, bind_list, snum, &port); } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL)); if (ret) goto err1; if (port != snum) { ret = -EADDRNOTAVAIL; goto err2; } bind_list->ps = ps; bind_list->port = (unsigned short) port; cma_bind_port(bind_list, id_priv); return 0; err2: idr_remove(ps, port); err1: kfree(bind_list); return ret; } static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv) { #if defined(INET) struct rdma_bind_list *bind_list; int port, ret, low, high; bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL); if (!bind_list) return -ENOMEM; retry: /* FIXME: add proper port randomization per like inet_csk_get_port */ do { ret = idr_get_new_above(ps, bind_list, next_port, &port); } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL)); if (ret) goto err1; inet_get_local_port_range(&low, &high); if (port > high) { if (next_port != low) { idr_remove(ps, port); next_port = low; goto retry; } ret = -EADDRNOTAVAIL; goto err2; } if (port == high) next_port = low; else next_port = port + 1; bind_list->ps = ps; bind_list->port = (unsigned short) port; cma_bind_port(bind_list, id_priv); return 0; err2: idr_remove(ps, port); err1: kfree(bind_list); return ret; #else return -ENOSPC; #endif } static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv) { struct rdma_id_private *cur_id; struct sockaddr_in *sin, *cur_sin; struct rdma_bind_list *bind_list; struct hlist_node *node; unsigned short snum; sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; snum = ntohs(sin->sin_port); #ifdef __linux__ if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE)) return -EACCES; #endif bind_list = idr_find(ps, snum); if (!bind_list) return cma_alloc_port(ps, id_priv, snum); /* * We don't support binding to any address if anyone is bound to * a specific address on the same port. */ if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr)) return -EADDRNOTAVAIL; hlist_for_each_entry(cur_id, node, &bind_list->owners, node) { if (cma_any_addr((struct sockaddr *) &cur_id->id.route.addr.src_addr)) return -EADDRNOTAVAIL; cur_sin = (struct sockaddr_in *) &cur_id->id.route.addr.src_addr; if (sin->sin_addr.s_addr == cur_sin->sin_addr.s_addr) return -EADDRINUSE; } cma_bind_port(bind_list, id_priv); return 0; } static int cma_get_tcp_port(struct rdma_id_private *id_priv) { int ret; int size; struct socket *sock; ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); if (ret) return ret; #ifdef __linux__ ret = sock->ops->bind(sock, (struct sockaddr *) &id_priv->id.route.addr.src_addr, ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr)); #else ret = -sobind(sock, (struct sockaddr *)&id_priv->id.route.addr.src_addr, curthread); #endif if (ret) { sock_release(sock); return ret; } size = ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr); ret = sock_getname(sock, (struct sockaddr *) &id_priv->id.route.addr.src_addr, &size, 0); if (ret) { sock_release(sock); return ret; } id_priv->sock = sock; return 0; } static int cma_get_port(struct rdma_id_private *id_priv) { struct idr *ps; int ret; switch (id_priv->id.ps) { case RDMA_PS_SDP: ps = &sdp_ps; break; case RDMA_PS_TCP: ps = &tcp_ps; if (unify_tcp_port_space) { ret = cma_get_tcp_port(id_priv); if (ret) goto out; } break; case RDMA_PS_UDP: ps = &udp_ps; break; case RDMA_PS_IPOIB: ps = &ipoib_ps; break; default: return -EPROTONOSUPPORT; } mutex_lock(&lock); if (cma_any_port((struct sockaddr *) &id_priv->id.route.addr.src_addr)) ret = cma_alloc_any_port(ps, id_priv); else ret = cma_use_port(ps, id_priv); mutex_unlock(&lock); out: return ret; } static int cma_check_linklocal(struct rdma_dev_addr *dev_addr, struct sockaddr *addr) { #if defined(INET6) struct sockaddr_in6 *sin6; if (addr->sa_family != AF_INET6) return 0; sin6 = (struct sockaddr_in6 *) addr; #ifdef __linux__ if ((ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL) && #else if (IN6_IS_SCOPE_LINKLOCAL(&sin6->sin6_addr) && #endif !sin6->sin6_scope_id) return -EINVAL; dev_addr->bound_dev_if = sin6->sin6_scope_id; #endif return 0; } int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) { struct rdma_id_private *id_priv; int ret; if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6) return -EAFNOSUPPORT; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_comp_exch(id_priv, CMA_IDLE, CMA_ADDR_BOUND)) return -EINVAL; ret = cma_check_linklocal(&id->route.addr.dev_addr, addr); if (ret) goto err1; if (!cma_any_addr(addr)) { ret = rdma_translate_ip(addr, &id->route.addr.dev_addr); if (ret) goto err1; mutex_lock(&lock); ret = cma_acquire_dev(id_priv); mutex_unlock(&lock); if (ret) goto err1; } memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr)); ret = cma_get_port(id_priv); if (ret) goto err2; return 0; err2: if (id_priv->cma_dev) { mutex_lock(&lock); cma_detach_from_dev(id_priv); mutex_unlock(&lock); } err1: cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_IDLE); return ret; } EXPORT_SYMBOL(rdma_bind_addr); static int cma_format_hdr(void *hdr, enum rdma_port_space ps, struct rdma_route *route) { struct cma_hdr *cma_hdr; struct sdp_hh *sdp_hdr; if (route->addr.src_addr.ss_family == AF_INET) { struct sockaddr_in *src4, *dst4; src4 = (struct sockaddr_in *) &route->addr.src_addr; dst4 = (struct sockaddr_in *) &route->addr.dst_addr; switch (ps) { case RDMA_PS_SDP: sdp_hdr = hdr; if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION) return -EINVAL; sdp_set_ip_ver(sdp_hdr, 4); sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; sdp_hdr->port = src4->sin_port; break; default: cma_hdr = hdr; cma_hdr->cma_version = CMA_VERSION; cma_set_ip_ver(cma_hdr, 4); cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; cma_hdr->port = src4->sin_port; break; } } else { struct sockaddr_in6 *src6, *dst6; src6 = (struct sockaddr_in6 *) &route->addr.src_addr; dst6 = (struct sockaddr_in6 *) &route->addr.dst_addr; switch (ps) { case RDMA_PS_SDP: sdp_hdr = hdr; if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION) return -EINVAL; sdp_set_ip_ver(sdp_hdr, 6); sdp_hdr->src_addr.ip6 = src6->sin6_addr; sdp_hdr->dst_addr.ip6 = dst6->sin6_addr; sdp_hdr->port = src6->sin6_port; break; default: cma_hdr = hdr; cma_hdr->cma_version = CMA_VERSION; cma_set_ip_ver(cma_hdr, 6); cma_hdr->src_addr.ip6 = src6->sin6_addr; cma_hdr->dst_addr.ip6 = dst6->sin6_addr; cma_hdr->port = src6->sin6_port; break; } } return 0; } static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) { struct rdma_id_private *id_priv = cm_id->context; struct rdma_cm_event event; struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd; int ret = 0; if (cma_disable_callback(id_priv, CMA_CONNECT)) return 0; memset(&event, 0, sizeof event); switch (ib_event->event) { case IB_CM_SIDR_REQ_ERROR: event.event = RDMA_CM_EVENT_UNREACHABLE; event.status = -ETIMEDOUT; break; case IB_CM_SIDR_REP_RECEIVED: event.param.ud.private_data = ib_event->private_data; event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE; if (rep->status != IB_SIDR_SUCCESS) { event.event = RDMA_CM_EVENT_UNREACHABLE; event.status = ib_event->param.sidr_rep_rcvd.status; break; } ret = cma_set_qkey(id_priv); if (ret) { event.event = RDMA_CM_EVENT_ADDR_ERROR; event.status = -EINVAL; break; } if (id_priv->qkey != rep->qkey) { event.event = RDMA_CM_EVENT_UNREACHABLE; event.status = -EINVAL; break; } ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num, id_priv->id.route.path_rec, &event.param.ud.ah_attr); event.param.ud.qp_num = rep->qpn; event.param.ud.qkey = rep->qkey; event.event = RDMA_CM_EVENT_ESTABLISHED; event.status = 0; break; default: printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n", ib_event->event); goto out; } ret = id_priv->id.event_handler(&id_priv->id, &event); if (ret) { /* Destroy the CM ID by returning a non-zero value. */ id_priv->cm_id.ib = NULL; cma_exch(id_priv, CMA_DESTROYING); mutex_unlock(&id_priv->handler_mutex); rdma_destroy_id(&id_priv->id); return ret; } out: mutex_unlock(&id_priv->handler_mutex); return ret; } static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct ib_cm_sidr_req_param req; struct rdma_route *route; int ret; req.private_data_len = sizeof(struct cma_hdr) + conn_param->private_data_len; req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC); if (!req.private_data) return -ENOMEM; if (conn_param->private_data && conn_param->private_data_len) memcpy((void *) req.private_data + sizeof(struct cma_hdr), conn_param->private_data, conn_param->private_data_len); route = &id_priv->id.route; ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route); if (ret) goto out; id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler, id_priv); if (IS_ERR(id_priv->cm_id.ib)) { ret = PTR_ERR(id_priv->cm_id.ib); goto out; } req.path = route->path_rec; req.service_id = cma_get_service_id(id_priv->id.ps, (struct sockaddr *) &route->addr.dst_addr); req.timeout_ms = 1 << (cma_response_timeout - 8); req.max_cm_retries = CMA_MAX_CM_RETRIES; ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req); if (ret) { ib_destroy_cm_id(id_priv->cm_id.ib); id_priv->cm_id.ib = NULL; } out: kfree(req.private_data); return ret; } static int cma_connect_ib(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct ib_cm_req_param req; struct rdma_route *route; void *private_data; int offset, ret; memset(&req, 0, sizeof req); offset = cma_user_data_offset(id_priv->id.ps); req.private_data_len = offset + conn_param->private_data_len; private_data = kzalloc(req.private_data_len, GFP_ATOMIC); if (!private_data) return -ENOMEM; if (conn_param->private_data && conn_param->private_data_len) memcpy(private_data + offset, conn_param->private_data, conn_param->private_data_len); id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv); if (IS_ERR(id_priv->cm_id.ib)) { ret = PTR_ERR(id_priv->cm_id.ib); goto out; } route = &id_priv->id.route; ret = cma_format_hdr(private_data, id_priv->id.ps, route); if (ret) goto out; req.private_data = private_data; req.primary_path = &route->path_rec[0]; if (route->num_paths == 2) req.alternate_path = &route->path_rec[1]; req.service_id = cma_get_service_id(id_priv->id.ps, (struct sockaddr *) &route->addr.dst_addr); req.qp_num = id_priv->qp_num; req.qp_type = IB_QPT_RC; req.starting_psn = id_priv->seq_num; req.responder_resources = conn_param->responder_resources; req.initiator_depth = conn_param->initiator_depth; req.flow_control = conn_param->flow_control; req.retry_count = conn_param->retry_count; req.rnr_retry_count = conn_param->rnr_retry_count; req.remote_cm_response_timeout = cma_response_timeout; req.local_cm_response_timeout = cma_response_timeout; req.max_cm_retries = CMA_MAX_CM_RETRIES; req.srq = id_priv->srq ? 1 : 0; ret = ib_send_cm_req(id_priv->cm_id.ib, &req); out: if (ret && !IS_ERR(id_priv->cm_id.ib)) { ib_destroy_cm_id(id_priv->cm_id.ib); id_priv->cm_id.ib = NULL; } kfree(private_data); return ret; } static int cma_connect_iw(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct iw_cm_id *cm_id; struct sockaddr_in* sin; int ret; struct iw_cm_conn_param iw_param; cm_id = iw_create_cm_id(id_priv->id.device, id_priv->sock, cma_iw_handler, id_priv); if (IS_ERR(cm_id)) { ret = PTR_ERR(cm_id); goto out; } id_priv->cm_id.iw = cm_id; sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr; cm_id->local_addr = *sin; sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr; cm_id->remote_addr = *sin; ret = cma_modify_qp_rtr(id_priv, conn_param); if (ret) goto out; iw_param.ord = conn_param->initiator_depth; iw_param.ird = conn_param->responder_resources; iw_param.private_data = conn_param->private_data; iw_param.private_data_len = conn_param->private_data_len; if (id_priv->id.qp) iw_param.qpn = id_priv->qp_num; else iw_param.qpn = conn_param->qp_num; ret = iw_cm_connect(cm_id, &iw_param); out: if (ret && !IS_ERR(cm_id)) { iw_destroy_cm_id(cm_id); id_priv->cm_id.iw = NULL; } return ret; } int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_CONNECT)) return -EINVAL; if (!id->qp) { id_priv->qp_num = conn_param->qp_num; id_priv->srq = conn_param->srq; } switch (rdma_node_get_transport(id->device->node_type)) { case RDMA_TRANSPORT_IB: if (cma_is_ud_ps(id->ps)) ret = cma_resolve_ib_udp(id_priv, conn_param); else ret = cma_connect_ib(id_priv, conn_param); break; case RDMA_TRANSPORT_IWARP: ret = cma_connect_iw(id_priv, conn_param); break; default: ret = -ENOSYS; break; } if (ret) goto err; return 0; err: cma_comp_exch(id_priv, CMA_CONNECT, CMA_ROUTE_RESOLVED); return ret; } EXPORT_SYMBOL(rdma_connect); static int cma_accept_ib(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct ib_cm_rep_param rep; int ret; ret = cma_modify_qp_rtr(id_priv, conn_param); if (ret) goto out; ret = cma_modify_qp_rts(id_priv, conn_param); if (ret) goto out; memset(&rep, 0, sizeof rep); rep.qp_num = id_priv->qp_num; rep.starting_psn = id_priv->seq_num; rep.private_data = conn_param->private_data; rep.private_data_len = conn_param->private_data_len; rep.responder_resources = conn_param->responder_resources; rep.initiator_depth = conn_param->initiator_depth; rep.failover_accepted = 0; rep.flow_control = conn_param->flow_control; rep.rnr_retry_count = conn_param->rnr_retry_count; rep.srq = id_priv->srq ? 1 : 0; ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep); out: return ret; } static int cma_accept_iw(struct rdma_id_private *id_priv, struct rdma_conn_param *conn_param) { struct iw_cm_conn_param iw_param; int ret; ret = cma_modify_qp_rtr(id_priv, conn_param); if (ret) return ret; iw_param.ord = conn_param->initiator_depth; iw_param.ird = conn_param->responder_resources; iw_param.private_data = conn_param->private_data; iw_param.private_data_len = conn_param->private_data_len; if (id_priv->id.qp) { iw_param.qpn = id_priv->qp_num; } else iw_param.qpn = conn_param->qp_num; return iw_cm_accept(id_priv->cm_id.iw, &iw_param); } static int cma_send_sidr_rep(struct rdma_id_private *id_priv, enum ib_cm_sidr_status status, const void *private_data, int private_data_len) { struct ib_cm_sidr_rep_param rep; int ret; memset(&rep, 0, sizeof rep); rep.status = status; if (status == IB_SIDR_SUCCESS) { ret = cma_set_qkey(id_priv); if (ret) return ret; rep.qp_num = id_priv->qp_num; rep.qkey = id_priv->qkey; } rep.private_data = private_data; rep.private_data_len = private_data_len; return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep); } int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_comp(id_priv, CMA_CONNECT)) return -EINVAL; if (!id->qp && conn_param) { id_priv->qp_num = conn_param->qp_num; id_priv->srq = conn_param->srq; } switch (rdma_node_get_transport(id->device->node_type)) { case RDMA_TRANSPORT_IB: if (cma_is_ud_ps(id->ps)) ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS, conn_param->private_data, conn_param->private_data_len); else if (conn_param) ret = cma_accept_ib(id_priv, conn_param); else ret = cma_rep_recv(id_priv); break; case RDMA_TRANSPORT_IWARP: ret = cma_accept_iw(id_priv, conn_param); break; default: ret = -ENOSYS; break; } if (ret) goto reject; return 0; reject: cma_modify_qp_err(id_priv); rdma_reject(id, NULL, 0); return ret; } EXPORT_SYMBOL(rdma_accept); int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_has_cm_dev(id_priv)) return -EINVAL; switch (id->device->node_type) { case RDMA_NODE_IB_CA: ret = ib_cm_notify(id_priv->cm_id.ib, event); break; default: ret = 0; break; } return ret; } EXPORT_SYMBOL(rdma_notify); int rdma_reject(struct rdma_cm_id *id, const void *private_data, u8 private_data_len) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_has_cm_dev(id_priv)) return -EINVAL; switch (rdma_node_get_transport(id->device->node_type)) { case RDMA_TRANSPORT_IB: if (cma_is_ud_ps(id->ps)) ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, private_data, private_data_len); else ret = ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0, private_data, private_data_len); break; case RDMA_TRANSPORT_IWARP: ret = iw_cm_reject(id_priv->cm_id.iw, private_data, private_data_len); break; default: ret = -ENOSYS; break; } return ret; } EXPORT_SYMBOL(rdma_reject); int rdma_disconnect(struct rdma_cm_id *id) { struct rdma_id_private *id_priv; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_has_cm_dev(id_priv)) return -EINVAL; switch (rdma_node_get_transport(id->device->node_type)) { case RDMA_TRANSPORT_IB: ret = cma_modify_qp_err(id_priv); if (ret) goto out; /* Initiate or respond to a disconnect. */ if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0); break; case RDMA_TRANSPORT_IWARP: ret = iw_cm_disconnect(id_priv->cm_id.iw, 0); break; default: ret = -EINVAL; break; } out: return ret; } EXPORT_SYMBOL(rdma_disconnect); static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) { struct rdma_id_private *id_priv; struct cma_multicast *mc = multicast->context; struct rdma_cm_event event; int ret; id_priv = mc->id_priv; if (cma_disable_callback(id_priv, CMA_ADDR_BOUND) && cma_disable_callback(id_priv, CMA_ADDR_RESOLVED)) return 0; mutex_lock(&id_priv->qp_mutex); if (!status && id_priv->id.qp) status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid, multicast->rec.mlid); mutex_unlock(&id_priv->qp_mutex); memset(&event, 0, sizeof event); event.status = status; event.param.ud.private_data = mc->context; if (!status) { event.event = RDMA_CM_EVENT_MULTICAST_JOIN; ib_init_ah_from_mcmember(id_priv->id.device, id_priv->id.port_num, &multicast->rec, &event.param.ud.ah_attr); event.param.ud.qp_num = 0xFFFFFF; event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey); } else event.event = RDMA_CM_EVENT_MULTICAST_ERROR; ret = id_priv->id.event_handler(&id_priv->id, &event); if (ret) { cma_exch(id_priv, CMA_DESTROYING); mutex_unlock(&id_priv->handler_mutex); rdma_destroy_id(&id_priv->id); return 0; } mutex_unlock(&id_priv->handler_mutex); return 0; } static void cma_set_mgid(struct rdma_id_private *id_priv, struct sockaddr *addr, union ib_gid *mgid) { #if defined(INET) || defined(INET6) unsigned char mc_map[MAX_ADDR_LEN]; struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; #endif #ifdef INET struct sockaddr_in *sin = (struct sockaddr_in *) addr; #endif #ifdef INET6 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr; #endif if (cma_any_addr(addr)) { memset(mgid, 0, sizeof *mgid); #ifdef INET6 } else if ((addr->sa_family == AF_INET6) && ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) == 0xFF10A01B)) { /* IPv6 address is an SA assigned MGID. */ memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); - } else if ((addr->sa_family == AF_INET6)) { + } else if (addr->sa_family == AF_INET6) { ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map); if (id_priv->id.ps == RDMA_PS_UDP) mc_map[7] = 0x01; /* Use RDMA CM signature */ *mgid = *(union ib_gid *) (mc_map + 4); #endif #ifdef INET } else { ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map); if (id_priv->id.ps == RDMA_PS_UDP) mc_map[7] = 0x01; /* Use RDMA CM signature */ *mgid = *(union ib_gid *) (mc_map + 4); #endif } } static int cma_join_ib_multicast(struct rdma_id_private *id_priv, struct cma_multicast *mc) { struct ib_sa_mcmember_rec rec; struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; ib_sa_comp_mask comp_mask; int ret; ib_addr_get_mgid(dev_addr, &rec.mgid); ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, &rec.mgid, &rec); if (ret) return ret; cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid); if (id_priv->id.ps == RDMA_PS_UDP) rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); rdma_addr_get_sgid(dev_addr, &rec.port_gid); rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); rec.join_state = 1; comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID | IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE | IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL | IB_SA_MCMEMBER_REC_FLOW_LABEL | IB_SA_MCMEMBER_REC_TRAFFIC_CLASS; if (id_priv->id.ps == RDMA_PS_IPOIB) comp_mask |= IB_SA_MCMEMBER_REC_RATE | IB_SA_MCMEMBER_REC_RATE_SELECTOR; mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device, id_priv->id.port_num, &rec, comp_mask, GFP_KERNEL, cma_ib_mc_handler, mc); if (IS_ERR(mc->multicast.ib)) return PTR_ERR(mc->multicast.ib); return 0; } static void iboe_mcast_work_handler(struct work_struct *work) { struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work); struct cma_multicast *mc = mw->mc; struct ib_sa_multicast *m = mc->multicast.ib; mc->multicast.ib->context = mc; cma_ib_mc_handler(0, m); kref_put(&mc->mcref, release_mc); kfree(mw); } static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid) { struct sockaddr_in *sin = (struct sockaddr_in *)addr; struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; if (cma_any_addr(addr)) { memset(mgid, 0, sizeof *mgid); } else if (addr->sa_family == AF_INET6) memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); else { mgid->raw[0] = 0xff; mgid->raw[1] = 0x0e; mgid->raw[2] = 0; mgid->raw[3] = 0; mgid->raw[4] = 0; mgid->raw[5] = 0; mgid->raw[6] = 0; mgid->raw[7] = 0; mgid->raw[8] = 0; mgid->raw[9] = 0; mgid->raw[10] = 0xff; mgid->raw[11] = 0xff; *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr; } } static int cma_iboe_join_multicast(struct rdma_id_private *id_priv, struct cma_multicast *mc) { struct iboe_mcast_work *work; struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; int err; struct sockaddr *addr = (struct sockaddr *)&mc->addr; struct net_device *ndev = NULL; if (cma_zero_addr((struct sockaddr *)&mc->addr)) return -EINVAL; work = kzalloc(sizeof *work, GFP_KERNEL); if (!work) return -ENOMEM; mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL); if (!mc->multicast.ib) { err = -ENOMEM; goto out1; } cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid); mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff); if (id_priv->id.ps == RDMA_PS_UDP) mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); if (dev_addr->bound_dev_if) ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if); if (!ndev) { err = -ENODEV; goto out2; } mc->multicast.ib->rec.rate = iboe_get_rate(ndev); mc->multicast.ib->rec.hop_limit = 1; #ifdef __linux__ mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu); #else mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->if_mtu); #endif dev_put(ndev); if (!mc->multicast.ib->rec.mtu) { err = -EINVAL; goto out2; } iboe_addr_get_sgid(dev_addr, &mc->multicast.ib->rec.port_gid); work->id = id_priv; work->mc = mc; INIT_WORK(&work->work, iboe_mcast_work_handler); kref_get(&mc->mcref); queue_work(cma_wq, &work->work); return 0; out2: kfree(mc->multicast.ib); out1: kfree(work); return err; } int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, void *context) { struct rdma_id_private *id_priv; struct cma_multicast *mc; int ret; id_priv = container_of(id, struct rdma_id_private, id); if (!cma_comp(id_priv, CMA_ADDR_BOUND) && !cma_comp(id_priv, CMA_ADDR_RESOLVED)) return -EINVAL; mc = kmalloc(sizeof *mc, GFP_KERNEL); if (!mc) return -ENOMEM; memcpy(&mc->addr, addr, ip_addr_size(addr)); mc->context = context; mc->id_priv = id_priv; spin_lock(&id_priv->lock); list_add(&mc->list, &id_priv->mc_list); spin_unlock(&id_priv->lock); switch (rdma_node_get_transport(id->device->node_type)) { case RDMA_TRANSPORT_IB: switch (rdma_port_get_link_layer(id->device, id->port_num)) { case IB_LINK_LAYER_INFINIBAND: ret = cma_join_ib_multicast(id_priv, mc); break; case IB_LINK_LAYER_ETHERNET: kref_init(&mc->mcref); ret = cma_iboe_join_multicast(id_priv, mc); break; default: ret = -EINVAL; } break; default: ret = -ENOSYS; break; } if (ret) { spin_lock_irq(&id_priv->lock); list_del(&mc->list); spin_unlock_irq(&id_priv->lock); kfree(mc); } return ret; } EXPORT_SYMBOL(rdma_join_multicast); void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) { struct rdma_id_private *id_priv; struct cma_multicast *mc; id_priv = container_of(id, struct rdma_id_private, id); spin_lock_irq(&id_priv->lock); list_for_each_entry(mc, &id_priv->mc_list, list) { if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) { list_del(&mc->list); spin_unlock_irq(&id_priv->lock); if (id->qp) ib_detach_mcast(id->qp, &mc->multicast.ib->rec.mgid, mc->multicast.ib->rec.mlid); if (rdma_node_get_transport(id_priv->cma_dev->device->node_type) == RDMA_TRANSPORT_IB) { switch (rdma_port_get_link_layer(id->device, id->port_num)) { case IB_LINK_LAYER_INFINIBAND: ib_sa_free_multicast(mc->multicast.ib); kfree(mc); break; case IB_LINK_LAYER_ETHERNET: kref_put(&mc->mcref, release_mc); break; default: break; } } return; } } spin_unlock_irq(&id_priv->lock); } EXPORT_SYMBOL(rdma_leave_multicast); static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv) { struct rdma_dev_addr *dev_addr; struct cma_ndev_work *work; dev_addr = &id_priv->id.route.addr.dev_addr; #ifdef __linux__ if ((dev_addr->bound_dev_if == ndev->ifindex) && memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) { printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n", ndev->name, &id_priv->id); #else if ((dev_addr->bound_dev_if == ndev->if_index) && memcmp(dev_addr->src_dev_addr, IF_LLADDR(ndev), ndev->if_addrlen)) { printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n", ndev->if_xname, &id_priv->id); #endif work = kzalloc(sizeof *work, GFP_KERNEL); if (!work) return -ENOMEM; INIT_WORK(&work->work, cma_ndev_work_handler); work->id = id_priv; work->event.event = RDMA_CM_EVENT_ADDR_CHANGE; atomic_inc(&id_priv->refcount); queue_work(cma_wq, &work->work); } return 0; } static int cma_netdev_callback(struct notifier_block *self, unsigned long event, void *ctx) { struct net_device *ndev = (struct net_device *)ctx; struct cma_device *cma_dev; struct rdma_id_private *id_priv; int ret = NOTIFY_DONE; #ifdef __linux__ if (dev_net(ndev) != &init_net) return NOTIFY_DONE; if (event != NETDEV_BONDING_FAILOVER) return NOTIFY_DONE; if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING)) return NOTIFY_DONE; #else if (event != NETDEV_DOWN && event != NETDEV_UNREGISTER) return NOTIFY_DONE; #endif mutex_lock(&lock); list_for_each_entry(cma_dev, &dev_list, list) list_for_each_entry(id_priv, &cma_dev->id_list, list) { ret = cma_netdev_change(ndev, id_priv); if (ret) goto out; } out: mutex_unlock(&lock); return ret; } static struct notifier_block cma_nb = { .notifier_call = cma_netdev_callback }; static void cma_add_one(struct ib_device *device) { struct cma_device *cma_dev; struct rdma_id_private *id_priv; cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL); if (!cma_dev) return; cma_dev->device = device; init_completion(&cma_dev->comp); atomic_set(&cma_dev->refcount, 1); INIT_LIST_HEAD(&cma_dev->id_list); ib_set_client_data(device, &cma_client, cma_dev); mutex_lock(&lock); list_add_tail(&cma_dev->list, &dev_list); list_for_each_entry(id_priv, &listen_any_list, list) cma_listen_on_dev(id_priv, cma_dev); mutex_unlock(&lock); } static int cma_remove_id_dev(struct rdma_id_private *id_priv) { struct rdma_cm_event event; enum cma_state state; int ret = 0; /* Record that we want to remove the device */ state = cma_exch(id_priv, CMA_DEVICE_REMOVAL); if (state == CMA_DESTROYING) return 0; cma_cancel_operation(id_priv, state); mutex_lock(&id_priv->handler_mutex); /* Check for destruction from another callback. */ if (!cma_comp(id_priv, CMA_DEVICE_REMOVAL)) goto out; memset(&event, 0, sizeof event); event.event = RDMA_CM_EVENT_DEVICE_REMOVAL; ret = id_priv->id.event_handler(&id_priv->id, &event); out: mutex_unlock(&id_priv->handler_mutex); return ret; } static void cma_process_remove(struct cma_device *cma_dev) { struct rdma_id_private *id_priv; int ret; mutex_lock(&lock); while (!list_empty(&cma_dev->id_list)) { id_priv = list_entry(cma_dev->id_list.next, struct rdma_id_private, list); list_del(&id_priv->listen_list); list_del_init(&id_priv->list); atomic_inc(&id_priv->refcount); mutex_unlock(&lock); ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv); cma_deref_id(id_priv); if (ret) rdma_destroy_id(&id_priv->id); mutex_lock(&lock); } mutex_unlock(&lock); cma_deref_dev(cma_dev); wait_for_completion(&cma_dev->comp); } static void cma_remove_one(struct ib_device *device) { struct cma_device *cma_dev; cma_dev = ib_get_client_data(device, &cma_client); if (!cma_dev) return; mutex_lock(&lock); list_del(&cma_dev->list); mutex_unlock(&lock); cma_process_remove(cma_dev); kfree(cma_dev); } static int cma_init(void) { int ret; #if defined(INET) int low, high, remaining; get_random_bytes(&next_port, sizeof next_port); inet_get_local_port_range(&low, &high); remaining = (high - low) + 1; next_port = ((unsigned int) next_port % remaining) + low; #endif cma_wq = create_singlethread_workqueue("rdma_cm"); if (!cma_wq) return -ENOMEM; ib_sa_register_client(&sa_client); rdma_addr_register_client(&addr_client); register_netdevice_notifier(&cma_nb); ret = ib_register_client(&cma_client); if (ret) goto err; return 0; err: unregister_netdevice_notifier(&cma_nb); rdma_addr_unregister_client(&addr_client); ib_sa_unregister_client(&sa_client); destroy_workqueue(cma_wq); return ret; } static void cma_cleanup(void) { ib_unregister_client(&cma_client); unregister_netdevice_notifier(&cma_nb); rdma_addr_unregister_client(&addr_client); ib_sa_unregister_client(&sa_client); destroy_workqueue(cma_wq); idr_destroy(&sdp_ps); idr_destroy(&tcp_ps); idr_destroy(&udp_ps); idr_destroy(&ipoib_ps); } module_init(cma_init); module_exit(cma_cleanup); Index: stable/9/sys/ofed/drivers/infiniband/core/core_priv.h =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/core_priv.h (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/core_priv.h (revision 273864) @@ -1,50 +1,51 @@ /* * Copyright (c) 2004 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef _CORE_PRIV_H #define _CORE_PRIV_H #include #include #include -int ib_device_register_sysfs(struct ib_device *device); +int ib_device_register_sysfs(struct ib_device *device, int (*port_callback)(struct ib_device *, + u8, struct kobject *)); void ib_device_unregister_sysfs(struct ib_device *device); int ib_sysfs_setup(void); void ib_sysfs_cleanup(void); int ib_cache_setup(void); void ib_cache_cleanup(void); #endif /* _CORE_PRIV_H */ Index: stable/9/sys/ofed/drivers/infiniband/core/device.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/device.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/device.c (revision 273864) @@ -1,754 +1,771 @@ /* * Copyright (c) 2004 Topspin Communications. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #include -#include #include #include #include "core_priv.h" MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("core kernel InfiniBand API"); MODULE_LICENSE("Dual BSD/GPL"); #ifdef __ia64__ /* workaround for a bug in hp chipset that would cause kernel panic when dma resources are exhaused */ int dma_map_sg_hp_wa = 0; #endif struct ib_client_data { struct list_head list; struct ib_client *client; void * data; }; static LIST_HEAD(device_list); static LIST_HEAD(client_list); /* * device_mutex protects access to both device_list and client_list. * There's no real point to using multiple locks or something fancier * like an rwsem: we always access both lists, and we're always * modifying one list or the other list. In any case this is not a * hot path so there's no point in trying to optimize. */ static DEFINE_MUTEX(device_mutex); static int ib_device_check_mandatory(struct ib_device *device) { #define IB_MANDATORY_FUNC(x) { offsetof(struct ib_device, x), #x } static const struct { size_t offset; char *name; } mandatory_table[] = { IB_MANDATORY_FUNC(query_device), IB_MANDATORY_FUNC(query_port), IB_MANDATORY_FUNC(query_pkey), IB_MANDATORY_FUNC(query_gid), IB_MANDATORY_FUNC(alloc_pd), IB_MANDATORY_FUNC(dealloc_pd), IB_MANDATORY_FUNC(create_ah), IB_MANDATORY_FUNC(destroy_ah), IB_MANDATORY_FUNC(create_qp), IB_MANDATORY_FUNC(modify_qp), IB_MANDATORY_FUNC(destroy_qp), IB_MANDATORY_FUNC(post_send), IB_MANDATORY_FUNC(post_recv), IB_MANDATORY_FUNC(create_cq), IB_MANDATORY_FUNC(destroy_cq), IB_MANDATORY_FUNC(poll_cq), IB_MANDATORY_FUNC(req_notify_cq), IB_MANDATORY_FUNC(get_dma_mr), IB_MANDATORY_FUNC(dereg_mr) }; int i; for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) { if (!*(void **) ((u_char *) device + mandatory_table[i].offset)) { printk(KERN_WARNING "Device %s is missing mandatory function %s\n", device->name, mandatory_table[i].name); return -EINVAL; } } return 0; } static struct ib_device *__ib_device_get_by_name(const char *name) { struct ib_device *device; list_for_each_entry(device, &device_list, core_list) if (!strncmp(name, device->name, IB_DEVICE_NAME_MAX)) return device; return NULL; } static int alloc_name(char *name) { unsigned long *inuse; char buf[IB_DEVICE_NAME_MAX]; struct ib_device *device; int i; inuse = (unsigned long *) get_zeroed_page(GFP_KERNEL); if (!inuse) return -ENOMEM; list_for_each_entry(device, &device_list, core_list) { if (!sscanf(device->name, name, &i)) continue; if (i < 0 || i >= PAGE_SIZE * 8) continue; snprintf(buf, sizeof buf, name, i); if (!strncmp(buf, device->name, IB_DEVICE_NAME_MAX)) set_bit(i, inuse); } i = find_first_zero_bit(inuse, PAGE_SIZE * 8); free_page((unsigned long) inuse); snprintf(buf, sizeof buf, name, i); if (__ib_device_get_by_name(buf)) return -ENFILE; strlcpy(name, buf, IB_DEVICE_NAME_MAX); return 0; } static int start_port(struct ib_device *device) { return (device->node_type == RDMA_NODE_IB_SWITCH) ? 0 : 1; } static int end_port(struct ib_device *device) { return (device->node_type == RDMA_NODE_IB_SWITCH) ? 0 : device->phys_port_cnt; } /** * ib_alloc_device - allocate an IB device struct * @size:size of structure to allocate * * Low-level drivers should use ib_alloc_device() to allocate &struct * ib_device. @size is the size of the structure to be allocated, * including any private data used by the low-level driver. * ib_dealloc_device() must be used to free structures allocated with * ib_alloc_device(). */ struct ib_device *ib_alloc_device(size_t size) { BUG_ON(size < sizeof (struct ib_device)); return kzalloc(size, GFP_KERNEL); } EXPORT_SYMBOL(ib_alloc_device); /** * ib_dealloc_device - free an IB device struct * @device:structure to free * * Free a structure allocated with ib_alloc_device(). */ void ib_dealloc_device(struct ib_device *device) { if (device->reg_state == IB_DEV_UNINITIALIZED) { kfree(device); return; } BUG_ON(device->reg_state != IB_DEV_UNREGISTERED); kobject_put(&device->dev.kobj); } EXPORT_SYMBOL(ib_dealloc_device); static int add_client_context(struct ib_device *device, struct ib_client *client) { struct ib_client_data *context; unsigned long flags; context = kmalloc(sizeof *context, GFP_KERNEL); if (!context) { printk(KERN_WARNING "Couldn't allocate client context for %s/%s\n", device->name, client->name); return -ENOMEM; } context->client = client; context->data = NULL; spin_lock_irqsave(&device->client_data_lock, flags); list_add(&context->list, &device->client_data_list); spin_unlock_irqrestore(&device->client_data_lock, flags); return 0; } static int read_port_table_lengths(struct ib_device *device) { struct ib_port_attr *tprops = NULL; int num_ports, ret = -ENOMEM; u8 port_index; tprops = kmalloc(sizeof *tprops, GFP_KERNEL); if (!tprops) goto out; num_ports = end_port(device) - start_port(device) + 1; device->pkey_tbl_len = kmalloc(sizeof *device->pkey_tbl_len * num_ports, GFP_KERNEL); device->gid_tbl_len = kmalloc(sizeof *device->gid_tbl_len * num_ports, GFP_KERNEL); if (!device->pkey_tbl_len || !device->gid_tbl_len) goto err; for (port_index = 0; port_index < num_ports; ++port_index) { ret = ib_query_port(device, port_index + start_port(device), tprops); if (ret) goto err; device->pkey_tbl_len[port_index] = tprops->pkey_tbl_len; device->gid_tbl_len[port_index] = tprops->gid_tbl_len; } ret = 0; goto out; err: kfree(device->gid_tbl_len); kfree(device->pkey_tbl_len); out: kfree(tprops); return ret; } /** * ib_register_device - Register an IB device with IB core * @device:Device to register * * Low-level drivers use ib_register_device() to register their * devices with the IB core. All registered clients will receive a * callback for each device that is added. @device must be allocated * with ib_alloc_device(). */ -int ib_register_device(struct ib_device *device) +int ib_register_device(struct ib_device *device, + int (*port_callback)(struct ib_device *, + u8, struct kobject *)) { int ret; mutex_lock(&device_mutex); if (strchr(device->name, '%')) { ret = alloc_name(device->name); if (ret) goto out; } if (ib_device_check_mandatory(device)) { ret = -EINVAL; goto out; } INIT_LIST_HEAD(&device->event_handler_list); INIT_LIST_HEAD(&device->client_data_list); spin_lock_init(&device->event_handler_lock); spin_lock_init(&device->client_data_lock); device->ib_uverbs_xrcd_table = RB_ROOT; mutex_init(&device->xrcd_table_mutex); ret = read_port_table_lengths(device); if (ret) { printk(KERN_WARNING "Couldn't create table lengths cache for device %s\n", device->name); goto out; } - ret = ib_device_register_sysfs(device); + ret = ib_device_register_sysfs(device, port_callback); if (ret) { printk(KERN_WARNING "Couldn't register device %s with driver model\n", device->name); kfree(device->gid_tbl_len); kfree(device->pkey_tbl_len); goto out; } list_add_tail(&device->core_list, &device_list); device->reg_state = IB_DEV_REGISTERED; { struct ib_client *client; list_for_each_entry(client, &client_list, list) if (client->add && !add_client_context(device, client)) client->add(device); } out: mutex_unlock(&device_mutex); return ret; } EXPORT_SYMBOL(ib_register_device); /** * ib_unregister_device - Unregister an IB device * @device:Device to unregister * * Unregister an IB device. All clients will receive a remove callback. */ void ib_unregister_device(struct ib_device *device) { struct ib_client *client; struct ib_client_data *context, *tmp; unsigned long flags; mutex_lock(&device_mutex); list_for_each_entry_reverse(client, &client_list, list) if (client->remove) client->remove(device); list_del(&device->core_list); kfree(device->gid_tbl_len); kfree(device->pkey_tbl_len); mutex_unlock(&device_mutex); ib_device_unregister_sysfs(device); spin_lock_irqsave(&device->client_data_lock, flags); list_for_each_entry_safe(context, tmp, &device->client_data_list, list) kfree(context); spin_unlock_irqrestore(&device->client_data_lock, flags); device->reg_state = IB_DEV_UNREGISTERED; } EXPORT_SYMBOL(ib_unregister_device); /** * ib_register_client - Register an IB client * @client:Client to register * * Upper level users of the IB drivers can use ib_register_client() to * register callbacks for IB device addition and removal. When an IB * device is added, each registered client's add method will be called * (in the order the clients were registered), and when a device is * removed, each client's remove method will be called (in the reverse * order that clients were registered). In addition, when * ib_register_client() is called, the client will receive an add * callback for all devices already registered. */ int ib_register_client(struct ib_client *client) { struct ib_device *device; mutex_lock(&device_mutex); list_add_tail(&client->list, &client_list); list_for_each_entry(device, &device_list, core_list) if (client->add && !add_client_context(device, client)) client->add(device); mutex_unlock(&device_mutex); return 0; } EXPORT_SYMBOL(ib_register_client); /** * ib_unregister_client - Unregister an IB client * @client:Client to unregister * * Upper level users use ib_unregister_client() to remove their client * registration. When ib_unregister_client() is called, the client * will receive a remove callback for each IB device still registered. */ void ib_unregister_client(struct ib_client *client) { struct ib_client_data *context, *tmp; struct ib_device *device; unsigned long flags; mutex_lock(&device_mutex); list_for_each_entry(device, &device_list, core_list) { if (client->remove) client->remove(device); spin_lock_irqsave(&device->client_data_lock, flags); list_for_each_entry_safe(context, tmp, &device->client_data_list, list) if (context->client == client) { list_del(&context->list); kfree(context); } spin_unlock_irqrestore(&device->client_data_lock, flags); } list_del(&client->list); mutex_unlock(&device_mutex); } EXPORT_SYMBOL(ib_unregister_client); /** * ib_get_client_data - Get IB client context * @device:Device to get context for * @client:Client to get context for * * ib_get_client_data() returns client context set with * ib_set_client_data(). */ void *ib_get_client_data(struct ib_device *device, struct ib_client *client) { struct ib_client_data *context; void *ret = NULL; unsigned long flags; spin_lock_irqsave(&device->client_data_lock, flags); list_for_each_entry(context, &device->client_data_list, list) if (context->client == client) { ret = context->data; break; } spin_unlock_irqrestore(&device->client_data_lock, flags); return ret; } EXPORT_SYMBOL(ib_get_client_data); /** * ib_set_client_data - Set IB client context * @device:Device to set context for * @client:Client to set context for * @data:Context to set * * ib_set_client_data() sets client context that can be retrieved with * ib_get_client_data(). */ void ib_set_client_data(struct ib_device *device, struct ib_client *client, void *data) { struct ib_client_data *context; unsigned long flags; spin_lock_irqsave(&device->client_data_lock, flags); list_for_each_entry(context, &device->client_data_list, list) if (context->client == client) { context->data = data; goto out; } printk(KERN_WARNING "No client context found for %s/%s\n", device->name, client->name); out: spin_unlock_irqrestore(&device->client_data_lock, flags); } EXPORT_SYMBOL(ib_set_client_data); /** * ib_register_event_handler - Register an IB event handler * @event_handler:Handler to register * * ib_register_event_handler() registers an event handler that will be * called back when asynchronous IB events occur (as defined in * chapter 11 of the InfiniBand Architecture Specification). This * callback may occur in interrupt context. */ int ib_register_event_handler (struct ib_event_handler *event_handler) { unsigned long flags; spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); list_add_tail(&event_handler->list, &event_handler->device->event_handler_list); spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); return 0; } EXPORT_SYMBOL(ib_register_event_handler); /** * ib_unregister_event_handler - Unregister an event handler * @event_handler:Handler to unregister * * Unregister an event handler registered with * ib_register_event_handler(). */ int ib_unregister_event_handler(struct ib_event_handler *event_handler) { unsigned long flags; spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); list_del(&event_handler->list); spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); return 0; } EXPORT_SYMBOL(ib_unregister_event_handler); /** * ib_dispatch_event - Dispatch an asynchronous event * @event:Event to dispatch * * Low-level drivers must call ib_dispatch_event() to dispatch the * event to all registered event handlers when an asynchronous event * occurs. */ void ib_dispatch_event(struct ib_event *event) { unsigned long flags; struct ib_event_handler *handler; spin_lock_irqsave(&event->device->event_handler_lock, flags); list_for_each_entry(handler, &event->device->event_handler_list, list) handler->handler(handler, event); spin_unlock_irqrestore(&event->device->event_handler_lock, flags); } EXPORT_SYMBOL(ib_dispatch_event); /** * ib_query_device - Query IB device attributes * @device:Device to query * @device_attr:Device attributes * * ib_query_device() returns the attributes of a device through the * @device_attr pointer. */ int ib_query_device(struct ib_device *device, struct ib_device_attr *device_attr) { return device->query_device(device, device_attr); } EXPORT_SYMBOL(ib_query_device); /** * ib_query_port - Query IB port attributes * @device:Device to query * @port_num:Port number to query * @port_attr:Port attributes * * ib_query_port() returns the attributes of a port through the * @port_attr pointer. */ int ib_query_port(struct ib_device *device, u8 port_num, struct ib_port_attr *port_attr) { if (port_num < start_port(device) || port_num > end_port(device)) return -EINVAL; return device->query_port(device, port_num, port_attr); } EXPORT_SYMBOL(ib_query_port); /** * ib_query_gid - Get GID table entry * @device:Device to query * @port_num:Port number to query * @index:GID table index to query * @gid:Returned GID * * ib_query_gid() fetches the specified GID table entry. */ int ib_query_gid(struct ib_device *device, u8 port_num, int index, union ib_gid *gid) { return device->query_gid(device, port_num, index, gid); } EXPORT_SYMBOL(ib_query_gid); /** * ib_query_pkey - Get P_Key table entry * @device:Device to query * @port_num:Port number to query * @index:P_Key table index to query * @pkey:Returned P_Key * * ib_query_pkey() fetches the specified P_Key table entry. */ int ib_query_pkey(struct ib_device *device, u8 port_num, u16 index, u16 *pkey) { return device->query_pkey(device, port_num, index, pkey); } EXPORT_SYMBOL(ib_query_pkey); /** * ib_modify_device - Change IB device attributes * @device:Device to modify * @device_modify_mask:Mask of attributes to change * @device_modify:New attribute values * * ib_modify_device() changes a device's attributes as specified by * the @device_modify_mask and @device_modify structure. */ int ib_modify_device(struct ib_device *device, int device_modify_mask, struct ib_device_modify *device_modify) { return device->modify_device(device, device_modify_mask, device_modify); } EXPORT_SYMBOL(ib_modify_device); /** * ib_modify_port - Modifies the attributes for the specified port. * @device: The device to modify. * @port_num: The number of the port to modify. * @port_modify_mask: Mask used to specify which attributes of the port * to change. * @port_modify: New attribute values for the port. * * ib_modify_port() changes a port's attributes as specified by the * @port_modify_mask and @port_modify structure. */ int ib_modify_port(struct ib_device *device, u8 port_num, int port_modify_mask, struct ib_port_modify *port_modify) { if (port_num < start_port(device) || port_num > end_port(device)) return -EINVAL; return device->modify_port(device, port_num, port_modify_mask, port_modify); } EXPORT_SYMBOL(ib_modify_port); /** * ib_find_gid - Returns the port number and GID table index where * a specified GID value occurs. * @device: The device to query. * @gid: The GID value to search for. * @port_num: The port number of the device where the GID value was found. * @index: The index into the GID table where the GID was found. This * parameter may be NULL. */ int ib_find_gid(struct ib_device *device, union ib_gid *gid, u8 *port_num, u16 *index) { union ib_gid tmp_gid; int ret, port, i; for (port = start_port(device); port <= end_port(device); ++port) { for (i = 0; i < device->gid_tbl_len[port - start_port(device)]; ++i) { ret = ib_query_gid(device, port, i, &tmp_gid); if (ret) return ret; if (!memcmp(&tmp_gid, gid, sizeof *gid)) { *port_num = port; if (index) *index = i; return 0; } } } return -ENOENT; } EXPORT_SYMBOL(ib_find_gid); /** * ib_find_pkey - Returns the PKey table index where a specified * PKey value occurs. * @device: The device to query. * @port_num: The port number of the device to search for the PKey. * @pkey: The PKey value to search for. * @index: The index into the PKey table where the PKey was found. */ int ib_find_pkey(struct ib_device *device, u8 port_num, u16 pkey, u16 *index) { int ret, i; u16 tmp_pkey; for (i = 0; i < device->pkey_tbl_len[port_num - start_port(device)]; ++i) { ret = ib_query_pkey(device, port_num, i, &tmp_pkey); if (ret) return ret; if ((pkey & 0x7fff) == (tmp_pkey & 0x7fff)) { *index = i; return 0; } } return -ENOENT; } EXPORT_SYMBOL(ib_find_pkey); static int __init ib_core_init(void) { int ret; #ifdef __ia64__ if (ia64_platform_is("hpzx1")) dma_map_sg_hp_wa = 1; #endif ret = ib_sysfs_setup(); if (ret) printk(KERN_WARNING "Couldn't create InfiniBand device class\n"); ret = ib_cache_setup(); if (ret) { printk(KERN_WARNING "Couldn't set up InfiniBand P_Key/GID cache\n"); ib_sysfs_cleanup(); } return ret; } static void __exit ib_core_cleanup(void) { ib_cache_cleanup(); ib_sysfs_cleanup(); /* Make sure that any pending umem accounting work is done. */ flush_scheduled_work(); } module_init(ib_core_init); module_exit(ib_core_cleanup); + +#undef MODULE_VERSION +#include +static int +ibcore_evhand(module_t mod, int event, void *arg) +{ + return (0); +} + +static moduledata_t ibcore_mod = { + .name = "ibcore", + .evhand = ibcore_evhand, +}; + +MODULE_VERSION(ibcore, 1); +DECLARE_MODULE(ibcore, ibcore_mod, SI_SUB_SMP, SI_ORDER_ANY); Index: stable/9/sys/ofed/drivers/infiniband/core/iwcm.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/iwcm.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/iwcm.c (revision 273864) @@ -1,1028 +1,1029 @@ /* * Copyright (c) 2004, 2005 Intel Corporation. All rights reserved. * Copyright (c) 2004 Topspin Corporation. All rights reserved. * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. * Copyright (c) 2005 Network Appliance, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ #include #include #include #include #include #include #include #include +#include #include #include #include "iwcm.h" MODULE_AUTHOR("Tom Tucker"); MODULE_DESCRIPTION("iWARP CM"); MODULE_LICENSE("Dual BSD/GPL"); static struct workqueue_struct *iwcm_wq; struct iwcm_work { struct work_struct work; struct iwcm_id_private *cm_id; struct list_head list; struct iw_cm_event event; struct list_head free_list; }; /* * The following services provide a mechanism for pre-allocating iwcm_work * elements. The design pre-allocates them based on the cm_id type: * LISTENING IDS: Get enough elements preallocated to handle the * listen backlog. * ACTIVE IDS: 4: CONNECT_REPLY, ESTABLISHED, DISCONNECT, CLOSE * PASSIVE IDS: 3: ESTABLISHED, DISCONNECT, CLOSE * * Allocating them in connect and listen avoids having to deal * with allocation failures on the event upcall from the provider (which * is called in the interrupt context). * * One exception is when creating the cm_id for incoming connection requests. * There are two cases: * 1) in the event upcall, cm_event_handler(), for a listening cm_id. If * the backlog is exceeded, then no more connection request events will * be processed. cm_event_handler() returns -ENOMEM in this case. Its up * to the provider to reject the connection request. * 2) in the connection request workqueue handler, cm_conn_req_handler(). * If work elements cannot be allocated for the new connect request cm_id, * then IWCM will call the provider reject method. This is ok since * cm_conn_req_handler() runs in the workqueue thread context. */ static struct iwcm_work *get_work(struct iwcm_id_private *cm_id_priv) { struct iwcm_work *work; if (list_empty(&cm_id_priv->work_free_list)) return NULL; work = list_entry(cm_id_priv->work_free_list.next, struct iwcm_work, free_list); list_del_init(&work->free_list); return work; } static void put_work(struct iwcm_work *work) { list_add(&work->free_list, &work->cm_id->work_free_list); } static void dealloc_work_entries(struct iwcm_id_private *cm_id_priv) { struct list_head *e, *tmp; list_for_each_safe(e, tmp, &cm_id_priv->work_free_list) kfree(list_entry(e, struct iwcm_work, free_list)); } static int alloc_work_entries(struct iwcm_id_private *cm_id_priv, int count) { struct iwcm_work *work; BUG_ON(!list_empty(&cm_id_priv->work_free_list)); while (count--) { work = kmalloc(sizeof(struct iwcm_work), GFP_KERNEL); if (!work) { dealloc_work_entries(cm_id_priv); return -ENOMEM; } work->cm_id = cm_id_priv; INIT_LIST_HEAD(&work->list); put_work(work); } return 0; } /* * Save private data from incoming connection requests to * iw_cm_event, so the low level driver doesn't have to. Adjust * the event ptr to point to the local copy. */ static int copy_private_data(struct iw_cm_event *event) { void *p; p = kmemdup(event->private_data, event->private_data_len, GFP_ATOMIC); if (!p) return -ENOMEM; event->private_data = p; return 0; } static void free_cm_id(struct iwcm_id_private *cm_id_priv) { dealloc_work_entries(cm_id_priv); kfree(cm_id_priv); } /* * Release a reference on cm_id. If the last reference is being * released, enable the waiting thread (in iw_destroy_cm_id) to * get woken up, and return 1 if a thread is already waiting. */ static int iwcm_deref_id(struct iwcm_id_private *cm_id_priv) { BUG_ON(atomic_read(&cm_id_priv->refcount)==0); if (atomic_dec_and_test(&cm_id_priv->refcount)) { BUG_ON(!list_empty(&cm_id_priv->work_list)); complete(&cm_id_priv->destroy_comp); return 1; } return 0; } static void add_ref(struct iw_cm_id *cm_id) { struct iwcm_id_private *cm_id_priv; cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); atomic_inc(&cm_id_priv->refcount); } static void rem_ref(struct iw_cm_id *cm_id) { struct iwcm_id_private *cm_id_priv; cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); if (iwcm_deref_id(cm_id_priv) && test_bit(IWCM_F_CALLBACK_DESTROY, &cm_id_priv->flags)) { BUG_ON(!list_empty(&cm_id_priv->work_list)); free_cm_id(cm_id_priv); } } static int cm_event_handler(struct iw_cm_id *cm_id, struct iw_cm_event *event); struct iw_cm_id *iw_create_cm_id(struct ib_device *device, struct socket *so, iw_cm_handler cm_handler, void *context) { struct iwcm_id_private *cm_id_priv; cm_id_priv = kzalloc(sizeof(*cm_id_priv), GFP_KERNEL); if (!cm_id_priv) return ERR_PTR(-ENOMEM); cm_id_priv->state = IW_CM_STATE_IDLE; cm_id_priv->id.device = device; cm_id_priv->id.cm_handler = cm_handler; cm_id_priv->id.context = context; cm_id_priv->id.event_handler = cm_event_handler; cm_id_priv->id.add_ref = add_ref; cm_id_priv->id.rem_ref = rem_ref; cm_id_priv->id.so = so; spin_lock_init(&cm_id_priv->lock); atomic_set(&cm_id_priv->refcount, 1); init_waitqueue_head(&cm_id_priv->connect_wait); init_completion(&cm_id_priv->destroy_comp); INIT_LIST_HEAD(&cm_id_priv->work_list); INIT_LIST_HEAD(&cm_id_priv->work_free_list); return &cm_id_priv->id; } EXPORT_SYMBOL(iw_create_cm_id); static int iwcm_modify_qp_err(struct ib_qp *qp) { struct ib_qp_attr qp_attr; if (!qp) return -EINVAL; qp_attr.qp_state = IB_QPS_ERR; return ib_modify_qp(qp, &qp_attr, IB_QP_STATE); } /* * This is really the RDMAC CLOSING state. It is most similar to the * IB SQD QP state. */ static int iwcm_modify_qp_sqd(struct ib_qp *qp) { struct ib_qp_attr qp_attr; BUG_ON(qp == NULL); qp_attr.qp_state = IB_QPS_SQD; return ib_modify_qp(qp, &qp_attr, IB_QP_STATE); } /* * CM_ID <-- CLOSING * * Block if a passive or active connection is currently being processed. Then * process the event as follows: * - If we are ESTABLISHED, move to CLOSING and modify the QP state * based on the abrupt flag * - If the connection is already in the CLOSING or IDLE state, the peer is * disconnecting concurrently with us and we've already seen the * DISCONNECT event -- ignore the request and return 0 * - Disconnect on a listening endpoint returns -EINVAL */ int iw_cm_disconnect(struct iw_cm_id *cm_id, int abrupt) { struct iwcm_id_private *cm_id_priv; unsigned long flags; int ret = 0; struct ib_qp *qp = NULL; cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); /* Wait if we're currently in a connect or accept downcall */ wait_event(cm_id_priv->connect_wait, !test_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags)); spin_lock_irqsave(&cm_id_priv->lock, flags); switch (cm_id_priv->state) { case IW_CM_STATE_ESTABLISHED: cm_id_priv->state = IW_CM_STATE_CLOSING; /* QP could be for user-mode client */ if (cm_id_priv->qp) qp = cm_id_priv->qp; else ret = -EINVAL; break; case IW_CM_STATE_LISTEN: ret = -EINVAL; break; case IW_CM_STATE_CLOSING: /* remote peer closed first */ case IW_CM_STATE_IDLE: /* accept or connect returned !0 */ break; case IW_CM_STATE_CONN_RECV: /* * App called disconnect before/without calling accept after * connect_request event delivered. */ break; case IW_CM_STATE_CONN_SENT: /* Can only get here if wait above fails */ default: BUG(); } spin_unlock_irqrestore(&cm_id_priv->lock, flags); if (qp) { if (abrupt) ret = iwcm_modify_qp_err(qp); else ret = iwcm_modify_qp_sqd(qp); /* * If both sides are disconnecting the QP could * already be in ERR or SQD states */ ret = 0; } return ret; } EXPORT_SYMBOL(iw_cm_disconnect); /* * CM_ID <-- DESTROYING * * Clean up all resources associated with the connection and release * the initial reference taken by iw_create_cm_id. */ static void destroy_cm_id(struct iw_cm_id *cm_id) { struct iwcm_id_private *cm_id_priv; unsigned long flags; int ret; cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); /* * Wait if we're currently in a connect or accept downcall. A * listening endpoint should never block here. */ wait_event(cm_id_priv->connect_wait, !test_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags)); spin_lock_irqsave(&cm_id_priv->lock, flags); switch (cm_id_priv->state) { case IW_CM_STATE_LISTEN: cm_id_priv->state = IW_CM_STATE_DESTROYING; spin_unlock_irqrestore(&cm_id_priv->lock, flags); /* destroy the listening endpoint */ ret = cm_id->device->iwcm->destroy_listen(cm_id); spin_lock_irqsave(&cm_id_priv->lock, flags); break; case IW_CM_STATE_ESTABLISHED: cm_id_priv->state = IW_CM_STATE_DESTROYING; spin_unlock_irqrestore(&cm_id_priv->lock, flags); /* Abrupt close of the connection */ (void)iwcm_modify_qp_err(cm_id_priv->qp); spin_lock_irqsave(&cm_id_priv->lock, flags); break; case IW_CM_STATE_IDLE: case IW_CM_STATE_CLOSING: cm_id_priv->state = IW_CM_STATE_DESTROYING; break; case IW_CM_STATE_CONN_RECV: /* * App called destroy before/without calling accept after * receiving connection request event notification or * returned non zero from the event callback function. * In either case, must tell the provider to reject. */ cm_id_priv->state = IW_CM_STATE_DESTROYING; spin_unlock_irqrestore(&cm_id_priv->lock, flags); cm_id->device->iwcm->reject(cm_id, NULL, 0); spin_lock_irqsave(&cm_id_priv->lock, flags); break; case IW_CM_STATE_CONN_SENT: case IW_CM_STATE_DESTROYING: default: BUG(); break; } if (cm_id_priv->qp) { cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp); cm_id_priv->qp = NULL; } spin_unlock_irqrestore(&cm_id_priv->lock, flags); (void)iwcm_deref_id(cm_id_priv); } /* * This function is only called by the application thread and cannot * be called by the event thread. The function will wait for all * references to be released on the cm_id and then kfree the cm_id * object. */ void iw_destroy_cm_id(struct iw_cm_id *cm_id) { struct iwcm_id_private *cm_id_priv; cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); BUG_ON(test_bit(IWCM_F_CALLBACK_DESTROY, &cm_id_priv->flags)); destroy_cm_id(cm_id); wait_for_completion(&cm_id_priv->destroy_comp); free_cm_id(cm_id_priv); } EXPORT_SYMBOL(iw_destroy_cm_id); /* * CM_ID <-- LISTEN * * Start listening for connect requests. Generates one CONNECT_REQUEST * event for each inbound connect request. */ int iw_cm_listen(struct iw_cm_id *cm_id, int backlog) { struct iwcm_id_private *cm_id_priv; unsigned long flags; int ret; cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); ret = alloc_work_entries(cm_id_priv, backlog); if (ret) return ret; spin_lock_irqsave(&cm_id_priv->lock, flags); switch (cm_id_priv->state) { case IW_CM_STATE_IDLE: cm_id_priv->state = IW_CM_STATE_LISTEN; spin_unlock_irqrestore(&cm_id_priv->lock, flags); ret = cm_id->device->iwcm->create_listen(cm_id, backlog); if (ret) cm_id_priv->state = IW_CM_STATE_IDLE; spin_lock_irqsave(&cm_id_priv->lock, flags); break; default: ret = -EINVAL; } spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } EXPORT_SYMBOL(iw_cm_listen); /* * CM_ID <-- IDLE * * Rejects an inbound connection request. No events are generated. */ int iw_cm_reject(struct iw_cm_id *cm_id, const void *private_data, u8 private_data_len) { struct iwcm_id_private *cm_id_priv; unsigned long flags; int ret; cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); set_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id_priv->state != IW_CM_STATE_CONN_RECV) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); wake_up_all(&cm_id_priv->connect_wait); return -EINVAL; } cm_id_priv->state = IW_CM_STATE_IDLE; spin_unlock_irqrestore(&cm_id_priv->lock, flags); ret = cm_id->device->iwcm->reject(cm_id, private_data, private_data_len); clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); wake_up_all(&cm_id_priv->connect_wait); return ret; } EXPORT_SYMBOL(iw_cm_reject); /* * CM_ID <-- ESTABLISHED * * Accepts an inbound connection request and generates an ESTABLISHED * event. Callers of iw_cm_disconnect and iw_destroy_cm_id will block * until the ESTABLISHED event is received from the provider. */ int iw_cm_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param) { struct iwcm_id_private *cm_id_priv; struct ib_qp *qp; unsigned long flags; int ret; cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); set_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id_priv->state != IW_CM_STATE_CONN_RECV) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); wake_up_all(&cm_id_priv->connect_wait); return -EINVAL; } /* Get the ib_qp given the QPN */ qp = cm_id->device->iwcm->get_qp(cm_id->device, iw_param->qpn); if (!qp) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); return -EINVAL; } cm_id->device->iwcm->add_ref(qp); cm_id_priv->qp = qp; spin_unlock_irqrestore(&cm_id_priv->lock, flags); ret = cm_id->device->iwcm->accept(cm_id, iw_param); if (ret) { /* An error on accept precludes provider events */ BUG_ON(cm_id_priv->state != IW_CM_STATE_CONN_RECV); cm_id_priv->state = IW_CM_STATE_IDLE; spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id_priv->qp) { cm_id->device->iwcm->rem_ref(qp); cm_id_priv->qp = NULL; } spin_unlock_irqrestore(&cm_id_priv->lock, flags); clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); wake_up_all(&cm_id_priv->connect_wait); } return ret; } EXPORT_SYMBOL(iw_cm_accept); /* * Active Side: CM_ID <-- CONN_SENT * * If successful, results in the generation of a CONNECT_REPLY * event. iw_cm_disconnect and iw_cm_destroy will block until the * CONNECT_REPLY event is received from the provider. */ int iw_cm_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param) { struct iwcm_id_private *cm_id_priv; int ret; unsigned long flags; struct ib_qp *qp; cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); ret = alloc_work_entries(cm_id_priv, 4); if (ret) return ret; set_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id_priv->state != IW_CM_STATE_IDLE) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); wake_up_all(&cm_id_priv->connect_wait); return -EINVAL; } /* Get the ib_qp given the QPN */ qp = cm_id->device->iwcm->get_qp(cm_id->device, iw_param->qpn); if (!qp) { spin_unlock_irqrestore(&cm_id_priv->lock, flags); return -EINVAL; } cm_id->device->iwcm->add_ref(qp); cm_id_priv->qp = qp; cm_id_priv->state = IW_CM_STATE_CONN_SENT; spin_unlock_irqrestore(&cm_id_priv->lock, flags); ret = cm_id->device->iwcm->connect(cm_id, iw_param); if (ret) { spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id_priv->qp) { cm_id->device->iwcm->rem_ref(qp); cm_id_priv->qp = NULL; } spin_unlock_irqrestore(&cm_id_priv->lock, flags); BUG_ON(cm_id_priv->state != IW_CM_STATE_CONN_SENT); cm_id_priv->state = IW_CM_STATE_IDLE; clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); wake_up_all(&cm_id_priv->connect_wait); } return ret; } EXPORT_SYMBOL(iw_cm_connect); /* * Passive Side: new CM_ID <-- CONN_RECV * * Handles an inbound connect request. The function creates a new * iw_cm_id to represent the new connection and inherits the client * callback function and other attributes from the listening parent. * * The work item contains a pointer to the listen_cm_id and the event. The * listen_cm_id contains the client cm_handler, context and * device. These are copied when the device is cloned. The event * contains the new four tuple. * * An error on the child should not affect the parent, so this * function does not return a value. */ static void cm_conn_req_handler(struct iwcm_id_private *listen_id_priv, struct iw_cm_event *iw_event) { unsigned long flags; struct iw_cm_id *cm_id; struct iwcm_id_private *cm_id_priv; int ret; /* * The provider should never generate a connection request * event with a bad status. */ BUG_ON(iw_event->status); /* * We could be destroying the listening id. If so, ignore this * upcall. */ spin_lock_irqsave(&listen_id_priv->lock, flags); if (listen_id_priv->state != IW_CM_STATE_LISTEN) { spin_unlock_irqrestore(&listen_id_priv->lock, flags); goto out; } spin_unlock_irqrestore(&listen_id_priv->lock, flags); cm_id = iw_create_cm_id(listen_id_priv->id.device, iw_event->so, listen_id_priv->id.cm_handler, listen_id_priv->id.context); /* If the cm_id could not be created, ignore the request */ if (IS_ERR(cm_id)) goto out; cm_id->provider_data = iw_event->provider_data; cm_id->local_addr = iw_event->local_addr; cm_id->remote_addr = iw_event->remote_addr; cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); cm_id_priv->state = IW_CM_STATE_CONN_RECV; ret = alloc_work_entries(cm_id_priv, 3); if (ret) { iw_cm_reject(cm_id, NULL, 0); iw_destroy_cm_id(cm_id); goto out; } /* Call the client CM handler */ ret = cm_id->cm_handler(cm_id, iw_event); if (ret) { iw_cm_reject(cm_id, NULL, 0); set_bit(IWCM_F_CALLBACK_DESTROY, &cm_id_priv->flags); destroy_cm_id(cm_id); if (atomic_read(&cm_id_priv->refcount)==0) free_cm_id(cm_id_priv); } out: if (iw_event->private_data_len) kfree(iw_event->private_data); } /* * Passive Side: CM_ID <-- ESTABLISHED * * The provider generated an ESTABLISHED event which means that * the MPA negotion has completed successfully and we are now in MPA * FPDU mode. * * This event can only be received in the CONN_RECV state. If the * remote peer closed, the ESTABLISHED event would be received followed * by the CLOSE event. If the app closes, it will block until we wake * it up after processing this event. */ static int cm_conn_est_handler(struct iwcm_id_private *cm_id_priv, struct iw_cm_event *iw_event) { unsigned long flags; int ret; spin_lock_irqsave(&cm_id_priv->lock, flags); /* * We clear the CONNECT_WAIT bit here to allow the callback * function to call iw_cm_disconnect. Calling iw_destroy_cm_id * from a callback handler is not allowed. */ clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); BUG_ON(cm_id_priv->state != IW_CM_STATE_CONN_RECV); cm_id_priv->state = IW_CM_STATE_ESTABLISHED; spin_unlock_irqrestore(&cm_id_priv->lock, flags); ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, iw_event); wake_up_all(&cm_id_priv->connect_wait); return ret; } /* * Active Side: CM_ID <-- ESTABLISHED * * The app has called connect and is waiting for the established event to * post it's requests to the server. This event will wake up anyone * blocked in iw_cm_disconnect or iw_destroy_id. */ static int cm_conn_rep_handler(struct iwcm_id_private *cm_id_priv, struct iw_cm_event *iw_event) { unsigned long flags; int ret; spin_lock_irqsave(&cm_id_priv->lock, flags); /* * Clear the connect wait bit so a callback function calling * iw_cm_disconnect will not wait and deadlock this thread */ clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); BUG_ON(cm_id_priv->state != IW_CM_STATE_CONN_SENT); if (iw_event->status == IW_CM_EVENT_STATUS_ACCEPTED) { cm_id_priv->id.local_addr = iw_event->local_addr; cm_id_priv->id.remote_addr = iw_event->remote_addr; cm_id_priv->state = IW_CM_STATE_ESTABLISHED; } else { /* REJECTED or RESET */ cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp); cm_id_priv->qp = NULL; cm_id_priv->state = IW_CM_STATE_IDLE; } spin_unlock_irqrestore(&cm_id_priv->lock, flags); ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, iw_event); if (iw_event->private_data_len) kfree(iw_event->private_data); /* Wake up waiters on connect complete */ wake_up_all(&cm_id_priv->connect_wait); return ret; } /* * CM_ID <-- CLOSING * * If in the ESTABLISHED state, move to CLOSING. */ static void cm_disconnect_handler(struct iwcm_id_private *cm_id_priv, struct iw_cm_event *iw_event) { unsigned long flags; spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id_priv->state == IW_CM_STATE_ESTABLISHED) cm_id_priv->state = IW_CM_STATE_CLOSING; spin_unlock_irqrestore(&cm_id_priv->lock, flags); } /* * CM_ID <-- IDLE * * If in the ESTBLISHED or CLOSING states, the QP will have have been * moved by the provider to the ERR state. Disassociate the CM_ID from * the QP, move to IDLE, and remove the 'connected' reference. * * If in some other state, the cm_id was destroyed asynchronously. * This is the last reference that will result in waking up * the app thread blocked in iw_destroy_cm_id. */ static int cm_close_handler(struct iwcm_id_private *cm_id_priv, struct iw_cm_event *iw_event) { unsigned long flags; int ret = 0; spin_lock_irqsave(&cm_id_priv->lock, flags); if (cm_id_priv->qp) { cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp); cm_id_priv->qp = NULL; } switch (cm_id_priv->state) { case IW_CM_STATE_ESTABLISHED: case IW_CM_STATE_CLOSING: cm_id_priv->state = IW_CM_STATE_IDLE; spin_unlock_irqrestore(&cm_id_priv->lock, flags); ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, iw_event); spin_lock_irqsave(&cm_id_priv->lock, flags); break; case IW_CM_STATE_DESTROYING: break; default: BUG(); } spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } static int process_event(struct iwcm_id_private *cm_id_priv, struct iw_cm_event *iw_event) { int ret = 0; switch (iw_event->event) { case IW_CM_EVENT_CONNECT_REQUEST: cm_conn_req_handler(cm_id_priv, iw_event); break; case IW_CM_EVENT_CONNECT_REPLY: ret = cm_conn_rep_handler(cm_id_priv, iw_event); break; case IW_CM_EVENT_ESTABLISHED: ret = cm_conn_est_handler(cm_id_priv, iw_event); break; case IW_CM_EVENT_DISCONNECT: cm_disconnect_handler(cm_id_priv, iw_event); break; case IW_CM_EVENT_CLOSE: ret = cm_close_handler(cm_id_priv, iw_event); break; default: BUG(); } return ret; } /* * Process events on the work_list for the cm_id. If the callback * function requests that the cm_id be deleted, a flag is set in the * cm_id flags to indicate that when the last reference is * removed, the cm_id is to be destroyed. This is necessary to * distinguish between an object that will be destroyed by the app * thread asleep on the destroy_comp list vs. an object destroyed * here synchronously when the last reference is removed. */ static void cm_work_handler(struct work_struct *_work) { struct iwcm_work *work = container_of(_work, struct iwcm_work, work); struct iw_cm_event levent; struct iwcm_id_private *cm_id_priv = work->cm_id; unsigned long flags; int empty; int ret = 0; int destroy_id; spin_lock_irqsave(&cm_id_priv->lock, flags); empty = list_empty(&cm_id_priv->work_list); while (!empty) { work = list_entry(cm_id_priv->work_list.next, struct iwcm_work, list); list_del_init(&work->list); empty = list_empty(&cm_id_priv->work_list); levent = work->event; put_work(work); spin_unlock_irqrestore(&cm_id_priv->lock, flags); ret = process_event(cm_id_priv, &levent); if (ret) { set_bit(IWCM_F_CALLBACK_DESTROY, &cm_id_priv->flags); destroy_cm_id(&cm_id_priv->id); } BUG_ON(atomic_read(&cm_id_priv->refcount)==0); destroy_id = test_bit(IWCM_F_CALLBACK_DESTROY, &cm_id_priv->flags); if (iwcm_deref_id(cm_id_priv)) { if (destroy_id) { BUG_ON(!list_empty(&cm_id_priv->work_list)); free_cm_id(cm_id_priv); } return; } spin_lock_irqsave(&cm_id_priv->lock, flags); } spin_unlock_irqrestore(&cm_id_priv->lock, flags); } /* * This function is called on interrupt context. Schedule events on * the iwcm_wq thread to allow callback functions to downcall into * the CM and/or block. Events are queued to a per-CM_ID * work_list. If this is the first event on the work_list, the work * element is also queued on the iwcm_wq thread. * * Each event holds a reference on the cm_id. Until the last posted * event has been delivered and processed, the cm_id cannot be * deleted. * * Returns: * 0 - the event was handled. * -ENOMEM - the event was not handled due to lack of resources. */ static int cm_event_handler(struct iw_cm_id *cm_id, struct iw_cm_event *iw_event) { struct iwcm_work *work; struct iwcm_id_private *cm_id_priv; unsigned long flags; int ret = 0; cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); spin_lock_irqsave(&cm_id_priv->lock, flags); work = get_work(cm_id_priv); if (!work) { ret = -ENOMEM; goto out; } INIT_WORK(&work->work, cm_work_handler); work->cm_id = cm_id_priv; work->event = *iw_event; if ((work->event.event == IW_CM_EVENT_CONNECT_REQUEST || work->event.event == IW_CM_EVENT_CONNECT_REPLY) && work->event.private_data_len) { ret = copy_private_data(&work->event); if (ret) { put_work(work); goto out; } } atomic_inc(&cm_id_priv->refcount); if (list_empty(&cm_id_priv->work_list)) { list_add_tail(&work->list, &cm_id_priv->work_list); queue_work(iwcm_wq, &work->work); } else list_add_tail(&work->list, &cm_id_priv->work_list); out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } static int iwcm_init_qp_init_attr(struct iwcm_id_private *cm_id_priv, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { unsigned long flags; int ret; spin_lock_irqsave(&cm_id_priv->lock, flags); switch (cm_id_priv->state) { case IW_CM_STATE_IDLE: case IW_CM_STATE_CONN_SENT: case IW_CM_STATE_CONN_RECV: case IW_CM_STATE_ESTABLISHED: *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS; qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE| IB_ACCESS_REMOTE_READ; ret = 0; break; default: ret = -EINVAL; break; } spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } static int iwcm_init_qp_rts_attr(struct iwcm_id_private *cm_id_priv, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { unsigned long flags; int ret; spin_lock_irqsave(&cm_id_priv->lock, flags); switch (cm_id_priv->state) { case IW_CM_STATE_IDLE: case IW_CM_STATE_CONN_SENT: case IW_CM_STATE_CONN_RECV: case IW_CM_STATE_ESTABLISHED: *qp_attr_mask = 0; ret = 0; break; default: ret = -EINVAL; break; } spin_unlock_irqrestore(&cm_id_priv->lock, flags); return ret; } int iw_cm_init_qp_attr(struct iw_cm_id *cm_id, struct ib_qp_attr *qp_attr, int *qp_attr_mask) { struct iwcm_id_private *cm_id_priv; int ret; cm_id_priv = container_of(cm_id, struct iwcm_id_private, id); switch (qp_attr->qp_state) { case IB_QPS_INIT: case IB_QPS_RTR: ret = iwcm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask); break; case IB_QPS_RTS: ret = iwcm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask); break; default: ret = -EINVAL; break; } return ret; } EXPORT_SYMBOL(iw_cm_init_qp_attr); static int __init iw_cm_init(void) { iwcm_wq = create_singlethread_workqueue("iw_cm_wq"); if (!iwcm_wq) return -ENOMEM; return 0; } static void __exit iw_cm_cleanup(void) { destroy_workqueue(iwcm_wq); } module_init(iw_cm_init); module_exit(iw_cm_cleanup); Index: stable/9/sys/ofed/drivers/infiniband/core/local_sa.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/local_sa.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/local_sa.c (revision 273864) @@ -1,1273 +1,1273 @@ /* - * Copyright (c) 2006 Intel Corporation.  All rights reserved. + * Copyright (c) 2006 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include "sa.h" MODULE_AUTHOR("Sean Hefty"); MODULE_DESCRIPTION("InfiniBand subnet administration caching"); MODULE_LICENSE("Dual BSD/GPL"); enum { SA_DB_MAX_PATHS_PER_DEST = 0x7F, SA_DB_MIN_RETRY_TIMER = 4000, /* 4 sec */ SA_DB_MAX_RETRY_TIMER = 256000 /* 256 sec */ }; static int set_paths_per_dest(const char *val, struct kernel_param *kp); static unsigned long paths_per_dest = 0; module_param_call(paths_per_dest, set_paths_per_dest, param_get_ulong, &paths_per_dest, 0644); MODULE_PARM_DESC(paths_per_dest, "Maximum number of paths to retrieve " "to each destination (DGID). Set to 0 " "to disable cache."); static int set_subscribe_inform_info(const char *val, struct kernel_param *kp); static char subscribe_inform_info = 1; module_param_call(subscribe_inform_info, set_subscribe_inform_info, param_get_bool, &subscribe_inform_info, 0644); MODULE_PARM_DESC(subscribe_inform_info, "Subscribe for SA InformInfo/Notice events."); static int do_refresh(const char *val, struct kernel_param *kp); module_param_call(refresh, do_refresh, NULL, NULL, 0200); static unsigned long retry_timer = SA_DB_MIN_RETRY_TIMER; enum sa_db_lookup_method { SA_DB_LOOKUP_LEAST_USED, SA_DB_LOOKUP_RANDOM }; static int set_lookup_method(const char *val, struct kernel_param *kp); static int get_lookup_method(char *buf, struct kernel_param *kp); static unsigned long lookup_method; module_param_call(lookup_method, set_lookup_method, get_lookup_method, &lookup_method, 0644); MODULE_PARM_DESC(lookup_method, "Method used to return path records when " "multiple paths exist to a given destination."); static void sa_db_add_dev(struct ib_device *device); static void sa_db_remove_dev(struct ib_device *device); static struct ib_client sa_db_client = { .name = "local_sa", .add = sa_db_add_dev, .remove = sa_db_remove_dev }; static LIST_HEAD(dev_list); static DEFINE_MUTEX(lock); static rwlock_t rwlock; static struct workqueue_struct *sa_wq; static struct ib_sa_client sa_client; enum sa_db_state { SA_DB_IDLE, SA_DB_REFRESH, SA_DB_DESTROY }; struct sa_db_port { struct sa_db_device *dev; struct ib_mad_agent *agent; /* Limit number of outstanding MADs to SA to reduce SA flooding */ struct ib_mad_send_buf *msg; u16 sm_lid; u8 sm_sl; struct ib_inform_info *in_info; struct ib_inform_info *out_info; struct rb_root paths; struct list_head update_list; unsigned long update_id; enum sa_db_state state; struct work_struct work; union ib_gid gid; int port_num; }; struct sa_db_device { struct list_head list; struct ib_device *device; struct ib_event_handler event_handler; int start_port; int port_count; struct sa_db_port port[0]; }; struct ib_sa_iterator { struct ib_sa_iterator *next; }; struct ib_sa_attr_iter { struct ib_sa_iterator *iter; unsigned long flags; }; struct ib_sa_attr_list { struct ib_sa_iterator iter; struct ib_sa_iterator *tail; int update_id; union ib_gid gid; struct rb_node node; }; struct ib_path_rec_info { struct ib_sa_iterator iter; /* keep first */ struct ib_sa_path_rec rec; unsigned long lookups; }; struct ib_sa_mad_iter { struct ib_mad_recv_wc *recv_wc; struct ib_mad_recv_buf *recv_buf; int attr_size; int attr_offset; int data_offset; int data_left; void *attr; u8 attr_data[0]; }; enum sa_update_type { SA_UPDATE_FULL, SA_UPDATE_ADD, SA_UPDATE_REMOVE }; struct update_info { struct list_head list; union ib_gid gid; enum sa_update_type type; }; struct sa_path_request { struct work_struct work; struct ib_sa_client *client; void (*callback)(int, struct ib_sa_path_rec *, void *); void *context; struct ib_sa_path_rec path_rec; }; static void process_updates(struct sa_db_port *port); static void free_attr_list(struct ib_sa_attr_list *attr_list) { struct ib_sa_iterator *cur; for (cur = attr_list->iter.next; cur; cur = attr_list->iter.next) { attr_list->iter.next = cur->next; kfree(cur); } attr_list->tail = &attr_list->iter; } static void remove_attr(struct rb_root *root, struct ib_sa_attr_list *attr_list) { rb_erase(&attr_list->node, root); free_attr_list(attr_list); kfree(attr_list); } static void remove_all_attrs(struct rb_root *root) { struct rb_node *node, *next_node; struct ib_sa_attr_list *attr_list; write_lock_irq(&rwlock); for (node = rb_first(root); node; node = next_node) { next_node = rb_next(node); attr_list = rb_entry(node, struct ib_sa_attr_list, node); remove_attr(root, attr_list); } write_unlock_irq(&rwlock); } static void remove_old_attrs(struct rb_root *root, unsigned long update_id) { struct rb_node *node, *next_node; struct ib_sa_attr_list *attr_list; write_lock_irq(&rwlock); for (node = rb_first(root); node; node = next_node) { next_node = rb_next(node); attr_list = rb_entry(node, struct ib_sa_attr_list, node); if (attr_list->update_id != update_id) remove_attr(root, attr_list); } write_unlock_irq(&rwlock); } static struct ib_sa_attr_list *insert_attr_list(struct rb_root *root, struct ib_sa_attr_list *attr_list) { struct rb_node **link = &root->rb_node; struct rb_node *parent = NULL; struct ib_sa_attr_list *cur_attr_list; int cmp; while (*link) { parent = *link; cur_attr_list = rb_entry(parent, struct ib_sa_attr_list, node); cmp = memcmp(&cur_attr_list->gid, &attr_list->gid, sizeof attr_list->gid); if (cmp < 0) link = &(*link)->rb_left; else if (cmp > 0) link = &(*link)->rb_right; else return cur_attr_list; } rb_link_node(&attr_list->node, parent, link); rb_insert_color(&attr_list->node, root); return NULL; } static struct ib_sa_attr_list *find_attr_list(struct rb_root *root, u8 *gid) { struct rb_node *node = root->rb_node; struct ib_sa_attr_list *attr_list; int cmp; while (node) { attr_list = rb_entry(node, struct ib_sa_attr_list, node); cmp = memcmp(&attr_list->gid, gid, sizeof attr_list->gid); if (cmp < 0) node = node->rb_left; else if (cmp > 0) node = node->rb_right; else return attr_list; } return NULL; } static int insert_attr(struct rb_root *root, unsigned long update_id, void *key, struct ib_sa_iterator *iter) { struct ib_sa_attr_list *attr_list; void *err; write_lock_irq(&rwlock); attr_list = find_attr_list(root, key); if (!attr_list) { write_unlock_irq(&rwlock); attr_list = kmalloc(sizeof *attr_list, GFP_KERNEL); if (!attr_list) return -ENOMEM; attr_list->iter.next = NULL; attr_list->tail = &attr_list->iter; attr_list->update_id = update_id; memcpy(attr_list->gid.raw, key, sizeof attr_list->gid); write_lock_irq(&rwlock); err = insert_attr_list(root, attr_list); if (err) { write_unlock_irq(&rwlock); kfree(attr_list); return PTR_ERR(err); } } else if (attr_list->update_id != update_id) { free_attr_list(attr_list); attr_list->update_id = update_id; } attr_list->tail->next = iter; iter->next = NULL; attr_list->tail = iter; write_unlock_irq(&rwlock); return 0; } static struct ib_sa_mad_iter *ib_sa_iter_create(struct ib_mad_recv_wc *mad_recv_wc) { struct ib_sa_mad_iter *iter; struct ib_sa_mad *mad = (struct ib_sa_mad *) mad_recv_wc->recv_buf.mad; int attr_size, attr_offset; attr_offset = be16_to_cpu(mad->sa_hdr.attr_offset) * 8; attr_size = 64; /* path record length */ if (attr_offset < attr_size) return ERR_PTR(-EINVAL); iter = kzalloc(sizeof *iter + attr_size, GFP_KERNEL); if (!iter) return ERR_PTR(-ENOMEM); iter->data_left = mad_recv_wc->mad_len - IB_MGMT_SA_HDR; iter->recv_wc = mad_recv_wc; iter->recv_buf = &mad_recv_wc->recv_buf; iter->attr_offset = attr_offset; iter->attr_size = attr_size; return iter; } static void ib_sa_iter_free(struct ib_sa_mad_iter *iter) { kfree(iter); } static void *ib_sa_iter_next(struct ib_sa_mad_iter *iter) { struct ib_sa_mad *mad; int left, offset = 0; while (iter->data_left >= iter->attr_offset) { while (iter->data_offset < IB_MGMT_SA_DATA) { mad = (struct ib_sa_mad *) iter->recv_buf->mad; left = IB_MGMT_SA_DATA - iter->data_offset; if (left < iter->attr_size) { /* copy first piece of the attribute */ iter->attr = &iter->attr_data; memcpy(iter->attr, &mad->data[iter->data_offset], left); offset = left; break; } else if (offset) { /* copy the second piece of the attribute */ memcpy(iter->attr + offset, &mad->data[0], iter->attr_size - offset); iter->data_offset = iter->attr_size - offset; offset = 0; } else { iter->attr = &mad->data[iter->data_offset]; iter->data_offset += iter->attr_size; } iter->data_left -= iter->attr_offset; goto out; } iter->data_offset = 0; iter->recv_buf = list_entry(iter->recv_buf->list.next, struct ib_mad_recv_buf, list); } iter->attr = NULL; out: return iter->attr; } /* * Copy path records from a received response and insert them into our cache. * A path record in the MADs are in network order, packed, and may * span multiple MAD buffers, just to make our life hard. */ static void update_path_db(struct sa_db_port *port, struct ib_mad_recv_wc *mad_recv_wc, enum sa_update_type type) { struct ib_sa_mad_iter *iter; struct ib_path_rec_info *path_info; void *attr; int ret; iter = ib_sa_iter_create(mad_recv_wc); if (IS_ERR(iter)) return; port->update_id += (type == SA_UPDATE_FULL); while ((attr = ib_sa_iter_next(iter)) && (path_info = kmalloc(sizeof *path_info, GFP_KERNEL))) { ib_sa_unpack_attr(&path_info->rec, attr, IB_SA_ATTR_PATH_REC); ret = insert_attr(&port->paths, port->update_id, path_info->rec.dgid.raw, &path_info->iter); if (ret) { kfree(path_info); break; } } ib_sa_iter_free(iter); if (type == SA_UPDATE_FULL) remove_old_attrs(&port->paths, port->update_id); } static struct ib_mad_send_buf *get_sa_msg(struct sa_db_port *port, struct update_info *update) { struct ib_ah_attr ah_attr; struct ib_mad_send_buf *msg; msg = ib_create_send_mad(port->agent, 1, 0, 0, IB_MGMT_SA_HDR, IB_MGMT_SA_DATA, GFP_KERNEL); if (IS_ERR(msg)) return NULL; memset(&ah_attr, 0, sizeof ah_attr); ah_attr.dlid = port->sm_lid; ah_attr.sl = port->sm_sl; ah_attr.port_num = port->port_num; msg->ah = ib_create_ah(port->agent->qp->pd, &ah_attr); if (IS_ERR(msg->ah)) { ib_free_send_mad(msg); return NULL; } msg->timeout_ms = retry_timer; msg->retries = 0; msg->context[0] = port; msg->context[1] = update; return msg; } static __be64 form_tid(u32 hi_tid) { static atomic_t tid; return cpu_to_be64((((u64) hi_tid) << 32) | ((u32) atomic_inc_return(&tid))); } static void format_path_req(struct sa_db_port *port, struct update_info *update, struct ib_mad_send_buf *msg) { struct ib_sa_mad *mad = msg->mad; struct ib_sa_path_rec path_rec; mad->mad_hdr.base_version = IB_MGMT_BASE_VERSION; mad->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; mad->mad_hdr.class_version = IB_SA_CLASS_VERSION; mad->mad_hdr.method = IB_SA_METHOD_GET_TABLE; mad->mad_hdr.attr_id = cpu_to_be16(IB_SA_ATTR_PATH_REC); mad->mad_hdr.tid = form_tid(msg->mad_agent->hi_tid); mad->sa_hdr.comp_mask = IB_SA_PATH_REC_SGID | IB_SA_PATH_REC_NUMB_PATH; path_rec.sgid = port->gid; path_rec.numb_path = (u8) paths_per_dest; if (update->type == SA_UPDATE_ADD) { mad->sa_hdr.comp_mask |= IB_SA_PATH_REC_DGID; memcpy(&path_rec.dgid, &update->gid, sizeof path_rec.dgid); } ib_sa_pack_attr(mad->data, &path_rec, IB_SA_ATTR_PATH_REC); } static int send_query(struct sa_db_port *port, struct update_info *update) { int ret; port->msg = get_sa_msg(port, update); if (!port->msg) return -ENOMEM; format_path_req(port, update, port->msg); ret = ib_post_send_mad(port->msg, NULL); if (ret) goto err; return 0; err: ib_destroy_ah(port->msg->ah); ib_free_send_mad(port->msg); return ret; } static void add_update(struct sa_db_port *port, u8 *gid, enum sa_update_type type) { struct update_info *update; update = kmalloc(sizeof *update, GFP_KERNEL); if (update) { if (gid) memcpy(&update->gid, gid, sizeof update->gid); update->type = type; list_add(&update->list, &port->update_list); } if (port->state == SA_DB_IDLE) { port->state = SA_DB_REFRESH; process_updates(port); } } static void clean_update_list(struct sa_db_port *port) { struct update_info *update; while (!list_empty(&port->update_list)) { update = list_entry(port->update_list.next, struct update_info, list); list_del(&update->list); kfree(update); } } static int notice_handler(int status, struct ib_inform_info *info, struct ib_sa_notice *notice) { struct sa_db_port *port = info->context; struct ib_sa_notice_data_gid *gid_data; struct ib_inform_info **pinfo; enum sa_update_type type; if (info->trap_number == IB_SA_SM_TRAP_GID_IN_SERVICE) { pinfo = &port->in_info; type = SA_UPDATE_ADD; } else { pinfo = &port->out_info; type = SA_UPDATE_REMOVE; } mutex_lock(&lock); if (port->state == SA_DB_DESTROY || !*pinfo) { mutex_unlock(&lock); return 0; } if (notice) { gid_data = (struct ib_sa_notice_data_gid *) ¬ice->data_details; add_update(port, gid_data->gid, type); mutex_unlock(&lock); } else if (status == -ENETRESET) { *pinfo = NULL; mutex_unlock(&lock); } else { if (status) *pinfo = ERR_PTR(-EINVAL); port->state = SA_DB_IDLE; clean_update_list(port); mutex_unlock(&lock); queue_work(sa_wq, &port->work); } return status; } static int reg_in_info(struct sa_db_port *port) { int ret = 0; port->in_info = ib_sa_register_inform_info(&sa_client, port->dev->device, port->port_num, IB_SA_SM_TRAP_GID_IN_SERVICE, GFP_KERNEL, notice_handler, port); if (IS_ERR(port->in_info)) ret = PTR_ERR(port->in_info); return ret; } static int reg_out_info(struct sa_db_port *port) { int ret = 0; port->out_info = ib_sa_register_inform_info(&sa_client, port->dev->device, port->port_num, IB_SA_SM_TRAP_GID_OUT_OF_SERVICE, GFP_KERNEL, notice_handler, port); if (IS_ERR(port->out_info)) ret = PTR_ERR(port->out_info); return ret; } static void unsubscribe_port(struct sa_db_port *port) { if (port->in_info && !IS_ERR(port->in_info)) ib_sa_unregister_inform_info(port->in_info); if (port->out_info && !IS_ERR(port->out_info)) ib_sa_unregister_inform_info(port->out_info); port->out_info = NULL; port->in_info = NULL; } static void cleanup_port(struct sa_db_port *port) { unsubscribe_port(port); clean_update_list(port); remove_all_attrs(&port->paths); } static int update_port_info(struct sa_db_port *port) { struct ib_port_attr port_attr; int ret; ret = ib_query_port(port->dev->device, port->port_num, &port_attr); if (ret) return ret; if (port_attr.state != IB_PORT_ACTIVE) return -ENODATA; port->sm_lid = port_attr.sm_lid; port->sm_sl = port_attr.sm_sl; return 0; } static void process_updates(struct sa_db_port *port) { struct update_info *update; struct ib_sa_attr_list *attr_list; int ret; if (!paths_per_dest || update_port_info(port)) { cleanup_port(port); goto out; } /* Event registration is an optimization, so ignore failures. */ if (subscribe_inform_info) { if (!port->out_info) { ret = reg_out_info(port); if (!ret) return; } if (!port->in_info) { ret = reg_in_info(port); if (!ret) return; } } else unsubscribe_port(port); while (!list_empty(&port->update_list)) { update = list_entry(port->update_list.next, struct update_info, list); if (update->type == SA_UPDATE_REMOVE) { write_lock_irq(&rwlock); attr_list = find_attr_list(&port->paths, update->gid.raw); if (attr_list) remove_attr(&port->paths, attr_list); write_unlock_irq(&rwlock); } else { ret = send_query(port, update); if (!ret) return; } list_del(&update->list); kfree(update); } out: port->state = SA_DB_IDLE; } static void refresh_port_db(struct sa_db_port *port) { if (port->state == SA_DB_DESTROY) return; if (port->state == SA_DB_REFRESH) { clean_update_list(port); ib_cancel_mad(port->agent, port->msg); } add_update(port, NULL, SA_UPDATE_FULL); } static void refresh_dev_db(struct sa_db_device *dev) { int i; for (i = 0; i < dev->port_count; i++) refresh_port_db(&dev->port[i]); } static void refresh_db(void) { struct sa_db_device *dev; list_for_each_entry(dev, &dev_list, list) refresh_dev_db(dev); } static int do_refresh(const char *val, struct kernel_param *kp) { mutex_lock(&lock); refresh_db(); mutex_unlock(&lock); return 0; } static int get_lookup_method(char *buf, struct kernel_param *kp) { return sprintf(buf, "%c %d round robin\n" "%c %d random", (lookup_method == SA_DB_LOOKUP_LEAST_USED) ? '*' : ' ', SA_DB_LOOKUP_LEAST_USED, (lookup_method == SA_DB_LOOKUP_RANDOM) ? '*' : ' ', SA_DB_LOOKUP_RANDOM); } static int set_lookup_method(const char *val, struct kernel_param *kp) { unsigned long method; int ret = 0; method = simple_strtoul(val, NULL, 0); switch (method) { case SA_DB_LOOKUP_LEAST_USED: case SA_DB_LOOKUP_RANDOM: lookup_method = method; break; default: ret = -EINVAL; break; } return ret; } static int set_paths_per_dest(const char *val, struct kernel_param *kp) { int ret; mutex_lock(&lock); ret = param_set_ulong(val, kp); if (ret) goto out; if (paths_per_dest > SA_DB_MAX_PATHS_PER_DEST) paths_per_dest = SA_DB_MAX_PATHS_PER_DEST; refresh_db(); out: mutex_unlock(&lock); return ret; } static int set_subscribe_inform_info(const char *val, struct kernel_param *kp) { int ret; ret = param_set_bool(val, kp); if (ret) return ret; return do_refresh(val, kp); } static void port_work_handler(struct work_struct *work) { struct sa_db_port *port; port = container_of(work, typeof(*port), work); mutex_lock(&lock); refresh_port_db(port); mutex_unlock(&lock); } static void handle_event(struct ib_event_handler *event_handler, struct ib_event *event) { struct sa_db_device *dev; struct sa_db_port *port; dev = container_of(event_handler, typeof(*dev), event_handler); port = &dev->port[event->element.port_num - dev->start_port]; switch (event->event) { case IB_EVENT_PORT_ERR: case IB_EVENT_LID_CHANGE: case IB_EVENT_SM_CHANGE: case IB_EVENT_CLIENT_REREGISTER: case IB_EVENT_PKEY_CHANGE: case IB_EVENT_PORT_ACTIVE: queue_work(sa_wq, &port->work); break; default: break; } } static void ib_free_path_iter(struct ib_sa_attr_iter *iter) { read_unlock_irqrestore(&rwlock, iter->flags); } static int ib_create_path_iter(struct ib_device *device, u8 port_num, union ib_gid *dgid, struct ib_sa_attr_iter *iter) { struct sa_db_device *dev; struct sa_db_port *port; struct ib_sa_attr_list *list; dev = ib_get_client_data(device, &sa_db_client); if (!dev) return -ENODEV; port = &dev->port[port_num - dev->start_port]; read_lock_irqsave(&rwlock, iter->flags); list = find_attr_list(&port->paths, dgid->raw); if (!list) { ib_free_path_iter(iter); return -ENODATA; } iter->iter = &list->iter; return 0; } static struct ib_sa_path_rec *ib_get_next_path(struct ib_sa_attr_iter *iter) { struct ib_path_rec_info *next_path; iter->iter = iter->iter->next; if (iter->iter) { next_path = container_of(iter->iter, struct ib_path_rec_info, iter); return &next_path->rec; } else return NULL; } static int cmp_rec(struct ib_sa_path_rec *src, struct ib_sa_path_rec *dst, ib_sa_comp_mask comp_mask) { /* DGID check already done */ if (comp_mask & IB_SA_PATH_REC_SGID && memcmp(&src->sgid, &dst->sgid, sizeof src->sgid)) return -EINVAL; if (comp_mask & IB_SA_PATH_REC_DLID && src->dlid != dst->dlid) return -EINVAL; if (comp_mask & IB_SA_PATH_REC_SLID && src->slid != dst->slid) return -EINVAL; if (comp_mask & IB_SA_PATH_REC_RAW_TRAFFIC && src->raw_traffic != dst->raw_traffic) return -EINVAL; if (comp_mask & IB_SA_PATH_REC_FLOW_LABEL && src->flow_label != dst->flow_label) return -EINVAL; if (comp_mask & IB_SA_PATH_REC_HOP_LIMIT && src->hop_limit != dst->hop_limit) return -EINVAL; if (comp_mask & IB_SA_PATH_REC_TRAFFIC_CLASS && src->traffic_class != dst->traffic_class) return -EINVAL; if (comp_mask & IB_SA_PATH_REC_REVERSIBLE && dst->reversible && !src->reversible) return -EINVAL; /* Numb path check already done */ if (comp_mask & IB_SA_PATH_REC_PKEY && src->pkey != dst->pkey) return -EINVAL; if (comp_mask & IB_SA_PATH_REC_SL && src->sl != dst->sl) return -EINVAL; if (ib_sa_check_selector(comp_mask, IB_SA_PATH_REC_MTU_SELECTOR, IB_SA_PATH_REC_MTU, dst->mtu_selector, src->mtu, dst->mtu)) return -EINVAL; if (ib_sa_check_selector(comp_mask, IB_SA_PATH_REC_RATE_SELECTOR, IB_SA_PATH_REC_RATE, dst->rate_selector, src->rate, dst->rate)) return -EINVAL; if (ib_sa_check_selector(comp_mask, IB_SA_PATH_REC_PACKET_LIFE_TIME_SELECTOR, IB_SA_PATH_REC_PACKET_LIFE_TIME, dst->packet_life_time_selector, src->packet_life_time, dst->packet_life_time)) return -EINVAL; return 0; } static struct ib_sa_path_rec *get_random_path(struct ib_sa_attr_iter *iter, struct ib_sa_path_rec *req_path, ib_sa_comp_mask comp_mask) { struct ib_sa_path_rec *path, *rand_path = NULL; int num, count = 0; for (path = ib_get_next_path(iter); path; path = ib_get_next_path(iter)) { if (!cmp_rec(path, req_path, comp_mask)) { get_random_bytes(&num, sizeof num); if ((num % ++count) == 0) rand_path = path; } } return rand_path; } static struct ib_sa_path_rec *get_next_path(struct ib_sa_attr_iter *iter, struct ib_sa_path_rec *req_path, ib_sa_comp_mask comp_mask) { struct ib_path_rec_info *cur_path, *next_path = NULL; struct ib_sa_path_rec *path; unsigned long lookups = ~0; for (path = ib_get_next_path(iter); path; path = ib_get_next_path(iter)) { if (!cmp_rec(path, req_path, comp_mask)) { cur_path = container_of(iter->iter, struct ib_path_rec_info, iter); if (cur_path->lookups < lookups) { lookups = cur_path->lookups; next_path = cur_path; } } } if (next_path) { next_path->lookups++; return &next_path->rec; } else return NULL; } static void report_path(struct work_struct *work) { struct sa_path_request *req; req = container_of(work, struct sa_path_request, work); req->callback(0, &req->path_rec, req->context); ib_sa_client_put(req->client); kfree(req); } /** * ib_sa_path_rec_get - Start a Path get query * @client:SA client * @device:device to send query on * @port_num: port number to send query on * @rec:Path Record to send in query * @comp_mask:component mask to send in query * @timeout_ms:time to wait for response * @gfp_mask:GFP mask to use for internal allocations * @callback:function called when query completes, times out or is * canceled * @context:opaque user context passed to callback * @sa_query:query context, used to cancel query * * Send a Path Record Get query to the SA to look up a path. The * callback function will be called when the query completes (or * fails); status is 0 for a successful response, -EINTR if the query * is canceled, -ETIMEDOUT is the query timed out, or -EIO if an error * occurred sending the query. The resp parameter of the callback is * only valid if status is 0. * * If the return value of ib_sa_path_rec_get() is negative, it is an * error code. Otherwise it is a query ID that can be used to cancel * the query. */ int ib_sa_path_rec_get(struct ib_sa_client *client, struct ib_device *device, u8 port_num, struct ib_sa_path_rec *rec, ib_sa_comp_mask comp_mask, int timeout_ms, gfp_t gfp_mask, void (*callback)(int status, struct ib_sa_path_rec *resp, void *context), void *context, struct ib_sa_query **sa_query) { struct sa_path_request *req; struct ib_sa_attr_iter iter; struct ib_sa_path_rec *path_rec; int ret; if (!paths_per_dest) goto query_sa; if (!(comp_mask & IB_SA_PATH_REC_DGID) || !(comp_mask & IB_SA_PATH_REC_NUMB_PATH) || rec->numb_path != 1) goto query_sa; req = kmalloc(sizeof *req, gfp_mask); if (!req) goto query_sa; ret = ib_create_path_iter(device, port_num, &rec->dgid, &iter); if (ret) goto free_req; if (lookup_method == SA_DB_LOOKUP_RANDOM) path_rec = get_random_path(&iter, rec, comp_mask); else path_rec = get_next_path(&iter, rec, comp_mask); if (!path_rec) goto free_iter; memcpy(&req->path_rec, path_rec, sizeof *path_rec); ib_free_path_iter(&iter); INIT_WORK(&req->work, report_path); req->client = client; req->callback = callback; req->context = context; ib_sa_client_get(client); queue_work(sa_wq, &req->work); *sa_query = ERR_PTR(-EEXIST); return 0; free_iter: ib_free_path_iter(&iter); free_req: kfree(req); query_sa: return ib_sa_path_rec_query(client, device, port_num, rec, comp_mask, timeout_ms, gfp_mask, callback, context, sa_query); } EXPORT_SYMBOL(ib_sa_path_rec_get); static void recv_handler(struct ib_mad_agent *mad_agent, struct ib_mad_recv_wc *mad_recv_wc) { struct sa_db_port *port; struct update_info *update; struct ib_mad_send_buf *msg; enum sa_update_type type; msg = (struct ib_mad_send_buf *) (unsigned long) mad_recv_wc->wc->wr_id; port = msg->context[0]; update = msg->context[1]; mutex_lock(&lock); if (port->state == SA_DB_DESTROY || update != list_entry(port->update_list.next, struct update_info, list)) { mutex_unlock(&lock); } else { type = update->type; mutex_unlock(&lock); update_path_db(mad_agent->context, mad_recv_wc, type); } ib_free_recv_mad(mad_recv_wc); } static void send_handler(struct ib_mad_agent *agent, struct ib_mad_send_wc *mad_send_wc) { struct ib_mad_send_buf *msg; struct sa_db_port *port; struct update_info *update; int ret; msg = mad_send_wc->send_buf; port = msg->context[0]; update = msg->context[1]; mutex_lock(&lock); if (port->state == SA_DB_DESTROY) goto unlock; if (update == list_entry(port->update_list.next, struct update_info, list)) { if (mad_send_wc->status == IB_WC_RESP_TIMEOUT_ERR && msg->timeout_ms < SA_DB_MAX_RETRY_TIMER) { msg->timeout_ms <<= 1; ret = ib_post_send_mad(msg, NULL); if (!ret) { mutex_unlock(&lock); return; } } list_del(&update->list); kfree(update); } process_updates(port); unlock: mutex_unlock(&lock); ib_destroy_ah(msg->ah); ib_free_send_mad(msg); } static int init_port(struct sa_db_device *dev, int port_num) { struct sa_db_port *port; int ret; port = &dev->port[port_num - dev->start_port]; port->dev = dev; port->port_num = port_num; INIT_WORK(&port->work, port_work_handler); port->paths = RB_ROOT; INIT_LIST_HEAD(&port->update_list); ret = ib_get_cached_gid(dev->device, port_num, 0, &port->gid); if (ret) return ret; port->agent = ib_register_mad_agent(dev->device, port_num, IB_QPT_GSI, NULL, IB_MGMT_RMPP_VERSION, send_handler, recv_handler, port); if (IS_ERR(port->agent)) ret = PTR_ERR(port->agent); return ret; } static void destroy_port(struct sa_db_port *port) { mutex_lock(&lock); port->state = SA_DB_DESTROY; mutex_unlock(&lock); ib_unregister_mad_agent(port->agent); cleanup_port(port); flush_workqueue(sa_wq); } static void sa_db_add_dev(struct ib_device *device) { struct sa_db_device *dev; struct sa_db_port *port; int s, e, i, ret; if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB) return; if (device->node_type == RDMA_NODE_IB_SWITCH) { s = e = 0; } else { s = 1; e = device->phys_port_cnt; } dev = kzalloc(sizeof *dev + (e - s + 1) * sizeof *port, GFP_KERNEL); if (!dev) return; dev->start_port = s; dev->port_count = e - s + 1; dev->device = device; for (i = 0; i < dev->port_count; i++) { ret = init_port(dev, s + i); if (ret) goto err; } ib_set_client_data(device, &sa_db_client, dev); INIT_IB_EVENT_HANDLER(&dev->event_handler, device, handle_event); mutex_lock(&lock); list_add_tail(&dev->list, &dev_list); refresh_dev_db(dev); mutex_unlock(&lock); ib_register_event_handler(&dev->event_handler); return; err: while (i--) destroy_port(&dev->port[i]); kfree(dev); } static void sa_db_remove_dev(struct ib_device *device) { struct sa_db_device *dev; int i; dev = ib_get_client_data(device, &sa_db_client); if (!dev) return; ib_unregister_event_handler(&dev->event_handler); flush_workqueue(sa_wq); for (i = 0; i < dev->port_count; i++) destroy_port(&dev->port[i]); mutex_lock(&lock); list_del(&dev->list); mutex_unlock(&lock); kfree(dev); } int sa_db_init(void) { int ret; rwlock_init(&rwlock); sa_wq = create_singlethread_workqueue("local_sa"); if (!sa_wq) return -ENOMEM; ib_sa_register_client(&sa_client); ret = ib_register_client(&sa_db_client); if (ret) goto err; return 0; err: ib_sa_unregister_client(&sa_client); destroy_workqueue(sa_wq); return ret; } void sa_db_cleanup(void) { ib_unregister_client(&sa_db_client); ib_sa_unregister_client(&sa_client); destroy_workqueue(sa_wq); } Index: stable/9/sys/ofed/drivers/infiniband/core/notice.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/notice.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/notice.c (revision 273864) @@ -1,749 +1,749 @@ /* - * Copyright (c) 2006 Intel Corporation.  All rights reserved. + * Copyright (c) 2006 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #include #include #include #include "sa.h" MODULE_AUTHOR("Sean Hefty"); MODULE_DESCRIPTION("InfiniBand InformInfo & Notice event handling"); MODULE_LICENSE("Dual BSD/GPL"); static void inform_add_one(struct ib_device *device); static void inform_remove_one(struct ib_device *device); static struct ib_client inform_client = { .name = "ib_notice", .add = inform_add_one, .remove = inform_remove_one }; static struct ib_sa_client sa_client; static struct workqueue_struct *inform_wq; struct inform_device; struct inform_port { struct inform_device *dev; spinlock_t lock; struct rb_root table; atomic_t refcount; struct completion comp; u8 port_num; }; struct inform_device { struct ib_device *device; struct ib_event_handler event_handler; int start_port; int end_port; struct inform_port port[0]; }; enum inform_state { INFORM_IDLE, INFORM_REGISTERING, INFORM_MEMBER, INFORM_BUSY, INFORM_ERROR }; struct inform_member; struct inform_group { u16 trap_number; struct rb_node node; struct inform_port *port; spinlock_t lock; struct work_struct work; struct list_head pending_list; struct list_head active_list; struct list_head notice_list; struct inform_member *last_join; int members; enum inform_state join_state; /* State relative to SA */ atomic_t refcount; enum inform_state state; struct ib_sa_query *query; int query_id; }; struct inform_member { struct ib_inform_info info; struct ib_sa_client *client; struct inform_group *group; struct list_head list; enum inform_state state; atomic_t refcount; struct completion comp; }; struct inform_notice { struct list_head list; struct ib_sa_notice notice; }; static void reg_handler(int status, struct ib_sa_inform *inform, void *context); static void unreg_handler(int status, struct ib_sa_inform *inform, void *context); static struct inform_group *inform_find(struct inform_port *port, u16 trap_number) { struct rb_node *node = port->table.rb_node; struct inform_group *group; while (node) { group = rb_entry(node, struct inform_group, node); if (trap_number < group->trap_number) node = node->rb_left; else if (trap_number > group->trap_number) node = node->rb_right; else return group; } return NULL; } static struct inform_group *inform_insert(struct inform_port *port, struct inform_group *group) { struct rb_node **link = &port->table.rb_node; struct rb_node *parent = NULL; struct inform_group *cur_group; while (*link) { parent = *link; cur_group = rb_entry(parent, struct inform_group, node); if (group->trap_number < cur_group->trap_number) link = &(*link)->rb_left; else if (group->trap_number > cur_group->trap_number) link = &(*link)->rb_right; else return cur_group; } rb_link_node(&group->node, parent, link); rb_insert_color(&group->node, &port->table); return NULL; } static void deref_port(struct inform_port *port) { if (atomic_dec_and_test(&port->refcount)) complete(&port->comp); } static void release_group(struct inform_group *group) { struct inform_port *port = group->port; unsigned long flags; spin_lock_irqsave(&port->lock, flags); if (atomic_dec_and_test(&group->refcount)) { rb_erase(&group->node, &port->table); spin_unlock_irqrestore(&port->lock, flags); kfree(group); deref_port(port); } else spin_unlock_irqrestore(&port->lock, flags); } static void deref_member(struct inform_member *member) { if (atomic_dec_and_test(&member->refcount)) complete(&member->comp); } static void queue_reg(struct inform_member *member) { struct inform_group *group = member->group; unsigned long flags; spin_lock_irqsave(&group->lock, flags); list_add(&member->list, &group->pending_list); if (group->state == INFORM_IDLE) { group->state = INFORM_BUSY; atomic_inc(&group->refcount); queue_work(inform_wq, &group->work); } spin_unlock_irqrestore(&group->lock, flags); } static int send_reg(struct inform_group *group, struct inform_member *member) { struct inform_port *port = group->port; struct ib_sa_inform inform; int ret; memset(&inform, 0, sizeof inform); inform.lid_range_begin = cpu_to_be16(0xFFFF); inform.is_generic = 1; inform.subscribe = 1; inform.type = cpu_to_be16(IB_SA_EVENT_TYPE_ALL); inform.trap.generic.trap_num = cpu_to_be16(member->info.trap_number); inform.trap.generic.resp_time = 19; inform.trap.generic.producer_type = cpu_to_be32(IB_SA_EVENT_PRODUCER_TYPE_ALL); group->last_join = member; ret = ib_sa_informinfo_query(&sa_client, port->dev->device, port->port_num, &inform, 3000, GFP_KERNEL, reg_handler, group,&group->query); if (ret >= 0) { group->query_id = ret; ret = 0; } return ret; } static int send_unreg(struct inform_group *group) { struct inform_port *port = group->port; struct ib_sa_inform inform; int ret; memset(&inform, 0, sizeof inform); inform.lid_range_begin = cpu_to_be16(0xFFFF); inform.is_generic = 1; inform.type = cpu_to_be16(IB_SA_EVENT_TYPE_ALL); inform.trap.generic.trap_num = cpu_to_be16(group->trap_number); inform.trap.generic.qpn = IB_QP1; inform.trap.generic.resp_time = 19; inform.trap.generic.producer_type = cpu_to_be32(IB_SA_EVENT_PRODUCER_TYPE_ALL); ret = ib_sa_informinfo_query(&sa_client, port->dev->device, port->port_num, &inform, 3000, GFP_KERNEL, unreg_handler, group, &group->query); if (ret >= 0) { group->query_id = ret; ret = 0; } return ret; } static void join_group(struct inform_group *group, struct inform_member *member) { member->state = INFORM_MEMBER; group->members++; list_move(&member->list, &group->active_list); } static int fail_join(struct inform_group *group, struct inform_member *member, int status) { spin_lock_irq(&group->lock); list_del_init(&member->list); spin_unlock_irq(&group->lock); return member->info.callback(status, &member->info, NULL); } static void process_group_error(struct inform_group *group) { struct inform_member *member; int ret; spin_lock_irq(&group->lock); while (!list_empty(&group->active_list)) { member = list_entry(group->active_list.next, struct inform_member, list); atomic_inc(&member->refcount); list_del_init(&member->list); group->members--; member->state = INFORM_ERROR; spin_unlock_irq(&group->lock); ret = member->info.callback(-ENETRESET, &member->info, NULL); deref_member(member); if (ret) ib_sa_unregister_inform_info(&member->info); spin_lock_irq(&group->lock); } group->join_state = INFORM_IDLE; group->state = INFORM_BUSY; spin_unlock_irq(&group->lock); } /* * Report a notice to all active subscribers. We use a temporary list to * handle unsubscription requests while the notice is being reported, which * avoids holding the group lock while in the user's callback. */ static void process_notice(struct inform_group *group, struct inform_notice *info_notice) { struct inform_member *member; struct list_head list; int ret; INIT_LIST_HEAD(&list); spin_lock_irq(&group->lock); list_splice_init(&group->active_list, &list); while (!list_empty(&list)) { member = list_entry(list.next, struct inform_member, list); atomic_inc(&member->refcount); list_move(&member->list, &group->active_list); spin_unlock_irq(&group->lock); ret = member->info.callback(0, &member->info, &info_notice->notice); deref_member(member); if (ret) ib_sa_unregister_inform_info(&member->info); spin_lock_irq(&group->lock); } spin_unlock_irq(&group->lock); } static void inform_work_handler(struct work_struct *work) { struct inform_group *group; struct inform_member *member; struct ib_inform_info *info; struct inform_notice *info_notice; int status, ret; group = container_of(work, typeof(*group), work); retest: spin_lock_irq(&group->lock); while (!list_empty(&group->pending_list) || !list_empty(&group->notice_list) || (group->state == INFORM_ERROR)) { if (group->state == INFORM_ERROR) { spin_unlock_irq(&group->lock); process_group_error(group); goto retest; } if (!list_empty(&group->notice_list)) { info_notice = list_entry(group->notice_list.next, struct inform_notice, list); list_del(&info_notice->list); spin_unlock_irq(&group->lock); process_notice(group, info_notice); kfree(info_notice); goto retest; } member = list_entry(group->pending_list.next, struct inform_member, list); info = &member->info; atomic_inc(&member->refcount); if (group->join_state == INFORM_MEMBER) { join_group(group, member); spin_unlock_irq(&group->lock); ret = info->callback(0, info, NULL); } else { spin_unlock_irq(&group->lock); status = send_reg(group, member); if (!status) { deref_member(member); return; } ret = fail_join(group, member, status); } deref_member(member); if (ret) ib_sa_unregister_inform_info(&member->info); spin_lock_irq(&group->lock); } if (!group->members && (group->join_state == INFORM_MEMBER)) { group->join_state = INFORM_IDLE; spin_unlock_irq(&group->lock); if (send_unreg(group)) goto retest; } else { group->state = INFORM_IDLE; spin_unlock_irq(&group->lock); release_group(group); } } /* * Fail a join request if it is still active - at the head of the pending queue. */ static void process_join_error(struct inform_group *group, int status) { struct inform_member *member; int ret; spin_lock_irq(&group->lock); member = list_entry(group->pending_list.next, struct inform_member, list); if (group->last_join == member) { atomic_inc(&member->refcount); list_del_init(&member->list); spin_unlock_irq(&group->lock); ret = member->info.callback(status, &member->info, NULL); deref_member(member); if (ret) ib_sa_unregister_inform_info(&member->info); } else spin_unlock_irq(&group->lock); } static void reg_handler(int status, struct ib_sa_inform *inform, void *context) { struct inform_group *group = context; if (status) process_join_error(group, status); else group->join_state = INFORM_MEMBER; inform_work_handler(&group->work); } static void unreg_handler(int status, struct ib_sa_inform *rec, void *context) { struct inform_group *group = context; inform_work_handler(&group->work); } int notice_dispatch(struct ib_device *device, u8 port_num, struct ib_sa_notice *notice) { struct inform_device *dev; struct inform_port *port; struct inform_group *group; struct inform_notice *info_notice; dev = ib_get_client_data(device, &inform_client); if (!dev) return 0; /* No one to give notice to. */ port = &dev->port[port_num - dev->start_port]; spin_lock_irq(&port->lock); group = inform_find(port, __be16_to_cpu(notice->trap. generic.trap_num)); if (!group) { spin_unlock_irq(&port->lock); return 0; } atomic_inc(&group->refcount); spin_unlock_irq(&port->lock); info_notice = kmalloc(sizeof *info_notice, GFP_KERNEL); if (!info_notice) { release_group(group); return -ENOMEM; } info_notice->notice = *notice; spin_lock_irq(&group->lock); list_add(&info_notice->list, &group->notice_list); if (group->state == INFORM_IDLE) { group->state = INFORM_BUSY; spin_unlock_irq(&group->lock); inform_work_handler(&group->work); } else { spin_unlock_irq(&group->lock); release_group(group); } return 0; } static struct inform_group *acquire_group(struct inform_port *port, u16 trap_number, gfp_t gfp_mask) { struct inform_group *group, *cur_group; unsigned long flags; spin_lock_irqsave(&port->lock, flags); group = inform_find(port, trap_number); if (group) goto found; spin_unlock_irqrestore(&port->lock, flags); group = kzalloc(sizeof *group, gfp_mask); if (!group) return NULL; group->port = port; group->trap_number = trap_number; INIT_LIST_HEAD(&group->pending_list); INIT_LIST_HEAD(&group->active_list); INIT_LIST_HEAD(&group->notice_list); INIT_WORK(&group->work, inform_work_handler); spin_lock_init(&group->lock); spin_lock_irqsave(&port->lock, flags); cur_group = inform_insert(port, group); if (cur_group) { kfree(group); group = cur_group; } else atomic_inc(&port->refcount); found: atomic_inc(&group->refcount); spin_unlock_irqrestore(&port->lock, flags); return group; } /* * We serialize all join requests to a single group to make our lives much * easier. Otherwise, two users could try to join the same group * simultaneously, with different configurations, one could leave while the * join is in progress, etc., which makes locking around error recovery * difficult. */ struct ib_inform_info * ib_sa_register_inform_info(struct ib_sa_client *client, struct ib_device *device, u8 port_num, u16 trap_number, gfp_t gfp_mask, int (*callback)(int status, struct ib_inform_info *info, struct ib_sa_notice *notice), void *context) { struct inform_device *dev; struct inform_member *member; struct ib_inform_info *info; int ret; dev = ib_get_client_data(device, &inform_client); if (!dev) return ERR_PTR(-ENODEV); member = kzalloc(sizeof *member, gfp_mask); if (!member) return ERR_PTR(-ENOMEM); ib_sa_client_get(client); member->client = client; member->info.trap_number = trap_number; member->info.callback = callback; member->info.context = context; init_completion(&member->comp); atomic_set(&member->refcount, 1); member->state = INFORM_REGISTERING; member->group = acquire_group(&dev->port[port_num - dev->start_port], trap_number, gfp_mask); if (!member->group) { ret = -ENOMEM; goto err; } /* * The user will get the info structure in their callback. They * could then free the info structure before we can return from * this routine. So we save the pointer to return before queuing * any callback. */ info = &member->info; queue_reg(member); return info; err: ib_sa_client_put(member->client); kfree(member); return ERR_PTR(ret); } EXPORT_SYMBOL(ib_sa_register_inform_info); void ib_sa_unregister_inform_info(struct ib_inform_info *info) { struct inform_member *member; struct inform_group *group; member = container_of(info, struct inform_member, info); group = member->group; spin_lock_irq(&group->lock); if (member->state == INFORM_MEMBER) group->members--; list_del_init(&member->list); if (group->state == INFORM_IDLE) { group->state = INFORM_BUSY; spin_unlock_irq(&group->lock); /* Continue to hold reference on group until callback */ queue_work(inform_wq, &group->work); } else { spin_unlock_irq(&group->lock); release_group(group); } deref_member(member); wait_for_completion(&member->comp); ib_sa_client_put(member->client); kfree(member); } EXPORT_SYMBOL(ib_sa_unregister_inform_info); static void inform_groups_lost(struct inform_port *port) { struct inform_group *group; struct rb_node *node; unsigned long flags; spin_lock_irqsave(&port->lock, flags); for (node = rb_first(&port->table); node; node = rb_next(node)) { group = rb_entry(node, struct inform_group, node); spin_lock(&group->lock); if (group->state == INFORM_IDLE) { atomic_inc(&group->refcount); queue_work(inform_wq, &group->work); } group->state = INFORM_ERROR; spin_unlock(&group->lock); } spin_unlock_irqrestore(&port->lock, flags); } static void inform_event_handler(struct ib_event_handler *handler, struct ib_event *event) { struct inform_device *dev; dev = container_of(handler, struct inform_device, event_handler); switch (event->event) { case IB_EVENT_PORT_ERR: case IB_EVENT_LID_CHANGE: case IB_EVENT_SM_CHANGE: case IB_EVENT_CLIENT_REREGISTER: inform_groups_lost(&dev->port[event->element.port_num - dev->start_port]); break; default: break; } } static void inform_add_one(struct ib_device *device) { struct inform_device *dev; struct inform_port *port; int i; if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB) return; dev = kmalloc(sizeof *dev + device->phys_port_cnt * sizeof *port, GFP_KERNEL); if (!dev) return; if (device->node_type == RDMA_NODE_IB_SWITCH) dev->start_port = dev->end_port = 0; else { dev->start_port = 1; dev->end_port = device->phys_port_cnt; } for (i = 0; i <= dev->end_port - dev->start_port; i++) { port = &dev->port[i]; port->dev = dev; port->port_num = dev->start_port + i; spin_lock_init(&port->lock); port->table = RB_ROOT; init_completion(&port->comp); atomic_set(&port->refcount, 1); } dev->device = device; ib_set_client_data(device, &inform_client, dev); INIT_IB_EVENT_HANDLER(&dev->event_handler, device, inform_event_handler); ib_register_event_handler(&dev->event_handler); } static void inform_remove_one(struct ib_device *device) { struct inform_device *dev; struct inform_port *port; int i; dev = ib_get_client_data(device, &inform_client); if (!dev) return; ib_unregister_event_handler(&dev->event_handler); flush_workqueue(inform_wq); for (i = 0; i <= dev->end_port - dev->start_port; i++) { port = &dev->port[i]; deref_port(port); wait_for_completion(&port->comp); } kfree(dev); } int notice_init(void) { int ret; inform_wq = create_singlethread_workqueue("ib_inform"); if (!inform_wq) return -ENOMEM; ib_sa_register_client(&sa_client); ret = ib_register_client(&inform_client); if (ret) goto err; return 0; err: ib_sa_unregister_client(&sa_client); destroy_workqueue(inform_wq); return ret; } void notice_cleanup(void) { ib_unregister_client(&inform_client); ib_sa_unregister_client(&sa_client); destroy_workqueue(inform_wq); } Index: stable/9/sys/ofed/drivers/infiniband/core/sa_query.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/sa_query.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/sa_query.c (revision 273864) @@ -1,1480 +1,1500 @@ /* * Copyright (c) 2004 Topspin Communications. All rights reserved. * Copyright (c) 2005 Voltaire, Inc. All rights reserved. * Copyright (c) 2006 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include -#include #include #include #include #include #include #include #include #include #include #include #include "sa.h" MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("InfiniBand subnet administration query support"); MODULE_LICENSE("Dual BSD/GPL"); struct ib_sa_sm_ah { struct ib_ah *ah; struct kref ref; u16 pkey_index; u8 src_path_mask; }; struct ib_sa_port { struct ib_mad_agent *agent; struct ib_mad_agent *notice_agent; struct ib_sa_sm_ah *sm_ah; struct work_struct update_task; spinlock_t ah_lock; u8 port_num; struct ib_device *device; }; struct ib_sa_device { int start_port, end_port; struct ib_event_handler event_handler; struct ib_sa_port port[0]; }; struct ib_sa_query { void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *); void (*release)(struct ib_sa_query *); struct ib_sa_client *client; struct ib_sa_port *port; struct ib_mad_send_buf *mad_buf; struct ib_sa_sm_ah *sm_ah; int id; }; struct ib_sa_service_query { void (*callback)(int, struct ib_sa_service_rec *, void *); void *context; struct ib_sa_query sa_query; }; struct ib_sa_path_query { void (*callback)(int, struct ib_sa_path_rec *, void *); void *context; struct ib_sa_query sa_query; }; struct ib_sa_mcmember_query { void (*callback)(int, struct ib_sa_mcmember_rec *, void *); void *context; struct ib_sa_query sa_query; }; struct ib_sa_inform_query { void (*callback)(int, struct ib_sa_inform *, void *); void *context; struct ib_sa_query sa_query; }; static void ib_sa_add_one(struct ib_device *device); static void ib_sa_remove_one(struct ib_device *device); static struct ib_client sa_client = { .name = "sa", .add = ib_sa_add_one, .remove = ib_sa_remove_one }; static spinlock_t idr_lock; static DEFINE_IDR(query_idr); static spinlock_t tid_lock; static u32 tid; #define PATH_REC_FIELD(field) \ .struct_offset_bytes = offsetof(struct ib_sa_path_rec, field), \ .struct_size_bytes = sizeof ((struct ib_sa_path_rec *) 0)->field, \ .field_name = "sa_path_rec:" #field static const struct ib_field path_rec_table[] = { { PATH_REC_FIELD(service_id), .offset_words = 0, .offset_bits = 0, .size_bits = 64 }, { PATH_REC_FIELD(dgid), .offset_words = 2, .offset_bits = 0, .size_bits = 128 }, { PATH_REC_FIELD(sgid), .offset_words = 6, .offset_bits = 0, .size_bits = 128 }, { PATH_REC_FIELD(dlid), .offset_words = 10, .offset_bits = 0, .size_bits = 16 }, { PATH_REC_FIELD(slid), .offset_words = 10, .offset_bits = 16, .size_bits = 16 }, { PATH_REC_FIELD(raw_traffic), .offset_words = 11, .offset_bits = 0, .size_bits = 1 }, { RESERVED, .offset_words = 11, .offset_bits = 1, .size_bits = 3 }, { PATH_REC_FIELD(flow_label), .offset_words = 11, .offset_bits = 4, .size_bits = 20 }, { PATH_REC_FIELD(hop_limit), .offset_words = 11, .offset_bits = 24, .size_bits = 8 }, { PATH_REC_FIELD(traffic_class), .offset_words = 12, .offset_bits = 0, .size_bits = 8 }, { PATH_REC_FIELD(reversible), .offset_words = 12, .offset_bits = 8, .size_bits = 1 }, { PATH_REC_FIELD(numb_path), .offset_words = 12, .offset_bits = 9, .size_bits = 7 }, { PATH_REC_FIELD(pkey), .offset_words = 12, .offset_bits = 16, .size_bits = 16 }, { PATH_REC_FIELD(qos_class), .offset_words = 13, .offset_bits = 0, .size_bits = 12 }, { PATH_REC_FIELD(sl), .offset_words = 13, .offset_bits = 12, .size_bits = 4 }, { PATH_REC_FIELD(mtu_selector), .offset_words = 13, .offset_bits = 16, .size_bits = 2 }, { PATH_REC_FIELD(mtu), .offset_words = 13, .offset_bits = 18, .size_bits = 6 }, { PATH_REC_FIELD(rate_selector), .offset_words = 13, .offset_bits = 24, .size_bits = 2 }, { PATH_REC_FIELD(rate), .offset_words = 13, .offset_bits = 26, .size_bits = 6 }, { PATH_REC_FIELD(packet_life_time_selector), .offset_words = 14, .offset_bits = 0, .size_bits = 2 }, { PATH_REC_FIELD(packet_life_time), .offset_words = 14, .offset_bits = 2, .size_bits = 6 }, { PATH_REC_FIELD(preference), .offset_words = 14, .offset_bits = 8, .size_bits = 8 }, { RESERVED, .offset_words = 14, .offset_bits = 16, .size_bits = 48 }, }; #define MCMEMBER_REC_FIELD(field) \ .struct_offset_bytes = offsetof(struct ib_sa_mcmember_rec, field), \ .struct_size_bytes = sizeof ((struct ib_sa_mcmember_rec *) 0)->field, \ .field_name = "sa_mcmember_rec:" #field static const struct ib_field mcmember_rec_table[] = { { MCMEMBER_REC_FIELD(mgid), .offset_words = 0, .offset_bits = 0, .size_bits = 128 }, { MCMEMBER_REC_FIELD(port_gid), .offset_words = 4, .offset_bits = 0, .size_bits = 128 }, { MCMEMBER_REC_FIELD(qkey), .offset_words = 8, .offset_bits = 0, .size_bits = 32 }, { MCMEMBER_REC_FIELD(mlid), .offset_words = 9, .offset_bits = 0, .size_bits = 16 }, { MCMEMBER_REC_FIELD(mtu_selector), .offset_words = 9, .offset_bits = 16, .size_bits = 2 }, { MCMEMBER_REC_FIELD(mtu), .offset_words = 9, .offset_bits = 18, .size_bits = 6 }, { MCMEMBER_REC_FIELD(traffic_class), .offset_words = 9, .offset_bits = 24, .size_bits = 8 }, { MCMEMBER_REC_FIELD(pkey), .offset_words = 10, .offset_bits = 0, .size_bits = 16 }, { MCMEMBER_REC_FIELD(rate_selector), .offset_words = 10, .offset_bits = 16, .size_bits = 2 }, { MCMEMBER_REC_FIELD(rate), .offset_words = 10, .offset_bits = 18, .size_bits = 6 }, { MCMEMBER_REC_FIELD(packet_life_time_selector), .offset_words = 10, .offset_bits = 24, .size_bits = 2 }, { MCMEMBER_REC_FIELD(packet_life_time), .offset_words = 10, .offset_bits = 26, .size_bits = 6 }, { MCMEMBER_REC_FIELD(sl), .offset_words = 11, .offset_bits = 0, .size_bits = 4 }, { MCMEMBER_REC_FIELD(flow_label), .offset_words = 11, .offset_bits = 4, .size_bits = 20 }, { MCMEMBER_REC_FIELD(hop_limit), .offset_words = 11, .offset_bits = 24, .size_bits = 8 }, { MCMEMBER_REC_FIELD(scope), .offset_words = 12, .offset_bits = 0, .size_bits = 4 }, { MCMEMBER_REC_FIELD(join_state), .offset_words = 12, .offset_bits = 4, .size_bits = 4 }, { MCMEMBER_REC_FIELD(proxy_join), .offset_words = 12, .offset_bits = 8, .size_bits = 1 }, { RESERVED, .offset_words = 12, .offset_bits = 9, .size_bits = 23 }, }; #define SERVICE_REC_FIELD(field) \ .struct_offset_bytes = offsetof(struct ib_sa_service_rec, field), \ .struct_size_bytes = sizeof ((struct ib_sa_service_rec *) 0)->field, \ .field_name = "sa_service_rec:" #field static const struct ib_field service_rec_table[] = { { SERVICE_REC_FIELD(id), .offset_words = 0, .offset_bits = 0, .size_bits = 64 }, { SERVICE_REC_FIELD(gid), .offset_words = 2, .offset_bits = 0, .size_bits = 128 }, { SERVICE_REC_FIELD(pkey), .offset_words = 6, .offset_bits = 0, .size_bits = 16 }, { SERVICE_REC_FIELD(lease), .offset_words = 7, .offset_bits = 0, .size_bits = 32 }, { SERVICE_REC_FIELD(key), .offset_words = 8, .offset_bits = 0, .size_bits = 128 }, { SERVICE_REC_FIELD(name), .offset_words = 12, .offset_bits = 0, .size_bits = 64*8 }, { SERVICE_REC_FIELD(data8), .offset_words = 28, .offset_bits = 0, .size_bits = 16*8 }, { SERVICE_REC_FIELD(data16), .offset_words = 32, .offset_bits = 0, .size_bits = 8*16 }, { SERVICE_REC_FIELD(data32), .offset_words = 36, .offset_bits = 0, .size_bits = 4*32 }, { SERVICE_REC_FIELD(data64), .offset_words = 40, .offset_bits = 0, .size_bits = 2*64 }, }; #define INFORM_FIELD(field) \ .struct_offset_bytes = offsetof(struct ib_sa_inform, field), \ .struct_size_bytes = sizeof ((struct ib_sa_inform *) 0)->field, \ .field_name = "sa_inform:" #field static const struct ib_field inform_table[] = { { INFORM_FIELD(gid), .offset_words = 0, .offset_bits = 0, .size_bits = 128 }, { INFORM_FIELD(lid_range_begin), .offset_words = 4, .offset_bits = 0, .size_bits = 16 }, { INFORM_FIELD(lid_range_end), .offset_words = 4, .offset_bits = 16, .size_bits = 16 }, { RESERVED, .offset_words = 5, .offset_bits = 0, .size_bits = 16 }, { INFORM_FIELD(is_generic), .offset_words = 5, .offset_bits = 16, .size_bits = 8 }, { INFORM_FIELD(subscribe), .offset_words = 5, .offset_bits = 24, .size_bits = 8 }, { INFORM_FIELD(type), .offset_words = 6, .offset_bits = 0, .size_bits = 16 }, { INFORM_FIELD(trap.generic.trap_num), .offset_words = 6, .offset_bits = 16, .size_bits = 16 }, { INFORM_FIELD(trap.generic.qpn), .offset_words = 7, .offset_bits = 0, .size_bits = 24 }, { RESERVED, .offset_words = 7, .offset_bits = 24, .size_bits = 3 }, { INFORM_FIELD(trap.generic.resp_time), .offset_words = 7, .offset_bits = 27, .size_bits = 5 }, { RESERVED, .offset_words = 8, .offset_bits = 0, .size_bits = 8 }, { INFORM_FIELD(trap.generic.producer_type), .offset_words = 8, .offset_bits = 8, .size_bits = 24 }, }; #define NOTICE_FIELD(field) \ .struct_offset_bytes = offsetof(struct ib_sa_notice, field), \ .struct_size_bytes = sizeof ((struct ib_sa_notice *) 0)->field, \ .field_name = "sa_notice:" #field static const struct ib_field notice_table[] = { { NOTICE_FIELD(is_generic), .offset_words = 0, .offset_bits = 0, .size_bits = 1 }, { NOTICE_FIELD(type), .offset_words = 0, .offset_bits = 1, .size_bits = 7 }, { NOTICE_FIELD(trap.generic.producer_type), .offset_words = 0, .offset_bits = 8, .size_bits = 24 }, { NOTICE_FIELD(trap.generic.trap_num), .offset_words = 1, .offset_bits = 0, .size_bits = 16 }, { NOTICE_FIELD(issuer_lid), .offset_words = 1, .offset_bits = 16, .size_bits = 16 }, { NOTICE_FIELD(notice_toggle), .offset_words = 2, .offset_bits = 0, .size_bits = 1 }, { NOTICE_FIELD(notice_count), .offset_words = 2, .offset_bits = 1, .size_bits = 15 }, { NOTICE_FIELD(data_details), .offset_words = 2, .offset_bits = 16, .size_bits = 432 }, { NOTICE_FIELD(issuer_gid), .offset_words = 16, .offset_bits = 0, .size_bits = 128 }, }; int ib_sa_check_selector(ib_sa_comp_mask comp_mask, ib_sa_comp_mask selector_mask, ib_sa_comp_mask value_mask, u8 selector, u8 src_value, u8 dst_value) { int err; if (!(comp_mask & selector_mask) || !(comp_mask & value_mask)) return 0; switch (selector) { case IB_SA_GT: err = (src_value <= dst_value); break; case IB_SA_LT: err = (src_value >= dst_value); break; case IB_SA_EQ: err = (src_value != dst_value); break; default: err = 0; break; } return err; } int ib_sa_pack_attr(void *dst, void *src, int attr_id) { switch (attr_id) { case IB_SA_ATTR_PATH_REC: ib_pack(path_rec_table, ARRAY_SIZE(path_rec_table), src, dst); break; default: return -EINVAL; } return 0; } int ib_sa_unpack_attr(void *dst, void *src, int attr_id) { switch (attr_id) { case IB_SA_ATTR_PATH_REC: ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table), src, dst); break; default: return -EINVAL; } return 0; } static void free_sm_ah(struct kref *kref) { struct ib_sa_sm_ah *sm_ah = container_of(kref, struct ib_sa_sm_ah, ref); ib_destroy_ah(sm_ah->ah); kfree(sm_ah); } static void update_sm_ah(struct work_struct *work) { struct ib_sa_port *port = container_of(work, struct ib_sa_port, update_task); struct ib_sa_sm_ah *new_ah; struct ib_port_attr port_attr; struct ib_ah_attr ah_attr; if (ib_query_port(port->agent->device, port->port_num, &port_attr)) { printk(KERN_WARNING "Couldn't query port\n"); return; } new_ah = kmalloc(sizeof *new_ah, GFP_KERNEL); if (!new_ah) { printk(KERN_WARNING "Couldn't allocate new SM AH\n"); return; } kref_init(&new_ah->ref); new_ah->src_path_mask = (1 << port_attr.lmc) - 1; new_ah->pkey_index = 0; if (ib_find_pkey(port->agent->device, port->port_num, IB_DEFAULT_PKEY_FULL, &new_ah->pkey_index)) printk(KERN_ERR "Couldn't find index for default PKey\n"); memset(&ah_attr, 0, sizeof ah_attr); ah_attr.dlid = port_attr.sm_lid; ah_attr.sl = port_attr.sm_sl; ah_attr.port_num = port->port_num; new_ah->ah = ib_create_ah(port->agent->qp->pd, &ah_attr); if (IS_ERR(new_ah->ah)) { printk(KERN_WARNING "Couldn't create new SM AH\n"); kfree(new_ah); return; } spin_lock_irq(&port->ah_lock); if (port->sm_ah) kref_put(&port->sm_ah->ref, free_sm_ah); port->sm_ah = new_ah; spin_unlock_irq(&port->ah_lock); } static void ib_sa_event(struct ib_event_handler *handler, struct ib_event *event) { if (event->event == IB_EVENT_PORT_ERR || event->event == IB_EVENT_PORT_ACTIVE || event->event == IB_EVENT_LID_CHANGE || event->event == IB_EVENT_PKEY_CHANGE || event->event == IB_EVENT_SM_CHANGE || event->event == IB_EVENT_CLIENT_REREGISTER) { unsigned long flags; struct ib_sa_device *sa_dev = container_of(handler, typeof(*sa_dev), event_handler); struct ib_sa_port *port = &sa_dev->port[event->element.port_num - sa_dev->start_port]; if (rdma_port_get_link_layer(handler->device, port->port_num) != IB_LINK_LAYER_INFINIBAND) return; spin_lock_irqsave(&port->ah_lock, flags); if (port->sm_ah) kref_put(&port->sm_ah->ref, free_sm_ah); port->sm_ah = NULL; spin_unlock_irqrestore(&port->ah_lock, flags); schedule_work(&sa_dev->port[event->element.port_num - sa_dev->start_port].update_task); } } void ib_sa_register_client(struct ib_sa_client *client) { atomic_set(&client->users, 1); init_completion(&client->comp); } EXPORT_SYMBOL(ib_sa_register_client); void ib_sa_unregister_client(struct ib_sa_client *client) { ib_sa_client_put(client); wait_for_completion(&client->comp); } EXPORT_SYMBOL(ib_sa_unregister_client); /** * ib_sa_cancel_query - try to cancel an SA query * @id:ID of query to cancel * @query:query pointer to cancel * * Try to cancel an SA query. If the id and query don't match up or * the query has already completed, nothing is done. Otherwise the * query is canceled and will complete with a status of -EINTR. */ void ib_sa_cancel_query(int id, struct ib_sa_query *query) { unsigned long flags; struct ib_mad_agent *agent; struct ib_mad_send_buf *mad_buf; spin_lock_irqsave(&idr_lock, flags); if (idr_find(&query_idr, id) != query) { spin_unlock_irqrestore(&idr_lock, flags); return; } agent = query->port->agent; mad_buf = query->mad_buf; spin_unlock_irqrestore(&idr_lock, flags); ib_cancel_mad(agent, mad_buf); } EXPORT_SYMBOL(ib_sa_cancel_query); static u8 get_src_path_mask(struct ib_device *device, u8 port_num) { struct ib_sa_device *sa_dev; struct ib_sa_port *port; unsigned long flags; u8 src_path_mask; sa_dev = ib_get_client_data(device, &sa_client); if (!sa_dev) return 0x7f; port = &sa_dev->port[port_num - sa_dev->start_port]; spin_lock_irqsave(&port->ah_lock, flags); src_path_mask = port->sm_ah ? port->sm_ah->src_path_mask : 0x7f; spin_unlock_irqrestore(&port->ah_lock, flags); return src_path_mask; } int ib_init_ah_from_path(struct ib_device *device, u8 port_num, struct ib_sa_path_rec *rec, struct ib_ah_attr *ah_attr) { int ret; u16 gid_index; int force_grh; memset(ah_attr, 0, sizeof *ah_attr); ah_attr->dlid = be16_to_cpu(rec->dlid); ah_attr->sl = rec->sl; ah_attr->src_path_bits = be16_to_cpu(rec->slid) & get_src_path_mask(device, port_num); ah_attr->port_num = port_num; ah_attr->static_rate = rec->rate; force_grh = rdma_port_get_link_layer(device, port_num) == IB_LINK_LAYER_ETHERNET; if (rec->hop_limit > 1 || force_grh) { ah_attr->ah_flags = IB_AH_GRH; ah_attr->grh.dgid = rec->dgid; ret = ib_find_cached_gid(device, &rec->sgid, &port_num, &gid_index); if (ret) return ret; ah_attr->grh.sgid_index = gid_index; ah_attr->grh.flow_label = be32_to_cpu(rec->flow_label); ah_attr->grh.hop_limit = rec->hop_limit; ah_attr->grh.traffic_class = rec->traffic_class; } return 0; } EXPORT_SYMBOL(ib_init_ah_from_path); static int alloc_mad(struct ib_sa_query *query, gfp_t gfp_mask) { unsigned long flags; spin_lock_irqsave(&query->port->ah_lock, flags); if (!query->port->sm_ah) { spin_unlock_irqrestore(&query->port->ah_lock, flags); return -EAGAIN; } kref_get(&query->port->sm_ah->ref); query->sm_ah = query->port->sm_ah; spin_unlock_irqrestore(&query->port->ah_lock, flags); query->mad_buf = ib_create_send_mad(query->port->agent, 1, query->sm_ah->pkey_index, 0, IB_MGMT_SA_HDR, IB_MGMT_SA_DATA, gfp_mask); if (IS_ERR(query->mad_buf)) { kref_put(&query->sm_ah->ref, free_sm_ah); return -ENOMEM; } query->mad_buf->ah = query->sm_ah->ah; return 0; } static void free_mad(struct ib_sa_query *query) { ib_free_send_mad(query->mad_buf); kref_put(&query->sm_ah->ref, free_sm_ah); } static void init_mad(struct ib_sa_mad *mad, struct ib_mad_agent *agent) { unsigned long flags; memset(mad, 0, sizeof *mad); mad->mad_hdr.base_version = IB_MGMT_BASE_VERSION; mad->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; mad->mad_hdr.class_version = IB_SA_CLASS_VERSION; spin_lock_irqsave(&tid_lock, flags); mad->mad_hdr.tid = cpu_to_be64(((u64) agent->hi_tid) << 32 | tid++); spin_unlock_irqrestore(&tid_lock, flags); } static int send_mad(struct ib_sa_query *query, int timeout_ms, gfp_t gfp_mask) { unsigned long flags; int ret, id; retry: if (!idr_pre_get(&query_idr, gfp_mask)) return -ENOMEM; spin_lock_irqsave(&idr_lock, flags); ret = idr_get_new(&query_idr, query, &id); spin_unlock_irqrestore(&idr_lock, flags); if (ret == -EAGAIN) goto retry; if (ret) return ret; query->mad_buf->timeout_ms = timeout_ms; query->mad_buf->context[0] = query; query->id = id; ret = ib_post_send_mad(query->mad_buf, NULL); if (ret) { spin_lock_irqsave(&idr_lock, flags); idr_remove(&query_idr, id); spin_unlock_irqrestore(&idr_lock, flags); } /* * It's not safe to dereference query any more, because the * send may already have completed and freed the query in * another context. */ return ret ? ret : id; } void ib_sa_unpack_path(void *attribute, struct ib_sa_path_rec *rec) { ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table), attribute, rec); } EXPORT_SYMBOL(ib_sa_unpack_path); static void ib_sa_path_rec_callback(struct ib_sa_query *sa_query, int status, struct ib_sa_mad *mad) { struct ib_sa_path_query *query = container_of(sa_query, struct ib_sa_path_query, sa_query); if (mad) { struct ib_sa_path_rec rec; ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table), mad->data, &rec); query->callback(status, &rec, query->context); } else query->callback(status, NULL, query->context); } static void ib_sa_path_rec_release(struct ib_sa_query *sa_query) { kfree(container_of(sa_query, struct ib_sa_path_query, sa_query)); } int ib_sa_path_rec_query(struct ib_sa_client *client, struct ib_device *device, u8 port_num, struct ib_sa_path_rec *rec, ib_sa_comp_mask comp_mask, int timeout_ms, gfp_t gfp_mask, void (*callback)(int status, struct ib_sa_path_rec *resp, void *context), void *context, struct ib_sa_query **sa_query) { struct ib_sa_path_query *query; struct ib_sa_device *sa_dev = ib_get_client_data(device, &sa_client); struct ib_sa_port *port; struct ib_mad_agent *agent; struct ib_sa_mad *mad; int ret; if (!sa_dev) return -ENODEV; port = &sa_dev->port[port_num - sa_dev->start_port]; agent = port->agent; query = kmalloc(sizeof *query, gfp_mask); if (!query) return -ENOMEM; query->sa_query.port = port; ret = alloc_mad(&query->sa_query, gfp_mask); if (ret) goto err1; ib_sa_client_get(client); query->sa_query.client = client; query->callback = callback; query->context = context; mad = query->sa_query.mad_buf->mad; init_mad(mad, agent); query->sa_query.callback = callback ? ib_sa_path_rec_callback : NULL; query->sa_query.release = ib_sa_path_rec_release; mad->mad_hdr.method = IB_MGMT_METHOD_GET; mad->mad_hdr.attr_id = cpu_to_be16(IB_SA_ATTR_PATH_REC); mad->sa_hdr.comp_mask = comp_mask; ib_pack(path_rec_table, ARRAY_SIZE(path_rec_table), rec, mad->data); *sa_query = &query->sa_query; ret = send_mad(&query->sa_query, timeout_ms, gfp_mask); if (ret < 0) goto err2; return ret; err2: *sa_query = NULL; ib_sa_client_put(query->sa_query.client); free_mad(&query->sa_query); err1: kfree(query); return ret; } static void ib_sa_service_rec_callback(struct ib_sa_query *sa_query, int status, struct ib_sa_mad *mad) { struct ib_sa_service_query *query = container_of(sa_query, struct ib_sa_service_query, sa_query); if (mad) { struct ib_sa_service_rec rec; ib_unpack(service_rec_table, ARRAY_SIZE(service_rec_table), mad->data, &rec); query->callback(status, &rec, query->context); } else query->callback(status, NULL, query->context); } static void ib_sa_service_rec_release(struct ib_sa_query *sa_query) { kfree(container_of(sa_query, struct ib_sa_service_query, sa_query)); } /** * ib_sa_service_rec_query - Start Service Record operation * @client:SA client * @device:device to send request on * @port_num: port number to send request on * @method:SA method - should be get, set, or delete * @rec:Service Record to send in request * @comp_mask:component mask to send in request * @timeout_ms:time to wait for response * @gfp_mask:GFP mask to use for internal allocations * @callback:function called when request completes, times out or is * canceled * @context:opaque user context passed to callback * @sa_query:request context, used to cancel request * * Send a Service Record set/get/delete to the SA to register, * unregister or query a service record. * The callback function will be called when the request completes (or * fails); status is 0 for a successful response, -EINTR if the query * is canceled, -ETIMEDOUT is the query timed out, or -EIO if an error * occurred sending the query. The resp parameter of the callback is * only valid if status is 0. * * If the return value of ib_sa_service_rec_query() is negative, it is an * error code. Otherwise it is a request ID that can be used to cancel * the query. */ int ib_sa_service_rec_query(struct ib_sa_client *client, struct ib_device *device, u8 port_num, u8 method, struct ib_sa_service_rec *rec, ib_sa_comp_mask comp_mask, int timeout_ms, gfp_t gfp_mask, void (*callback)(int status, struct ib_sa_service_rec *resp, void *context), void *context, struct ib_sa_query **sa_query) { struct ib_sa_service_query *query; struct ib_sa_device *sa_dev = ib_get_client_data(device, &sa_client); struct ib_sa_port *port; struct ib_mad_agent *agent; struct ib_sa_mad *mad; int ret; if (!sa_dev) return -ENODEV; port = &sa_dev->port[port_num - sa_dev->start_port]; agent = port->agent; if (method != IB_MGMT_METHOD_GET && method != IB_MGMT_METHOD_SET && method != IB_SA_METHOD_DELETE) return -EINVAL; query = kmalloc(sizeof *query, gfp_mask); if (!query) return -ENOMEM; query->sa_query.port = port; ret = alloc_mad(&query->sa_query, gfp_mask); if (ret) goto err1; ib_sa_client_get(client); query->sa_query.client = client; query->callback = callback; query->context = context; mad = query->sa_query.mad_buf->mad; init_mad(mad, agent); query->sa_query.callback = callback ? ib_sa_service_rec_callback : NULL; query->sa_query.release = ib_sa_service_rec_release; mad->mad_hdr.method = method; mad->mad_hdr.attr_id = cpu_to_be16(IB_SA_ATTR_SERVICE_REC); mad->sa_hdr.comp_mask = comp_mask; ib_pack(service_rec_table, ARRAY_SIZE(service_rec_table), rec, mad->data); *sa_query = &query->sa_query; ret = send_mad(&query->sa_query, timeout_ms, gfp_mask); if (ret < 0) goto err2; return ret; err2: *sa_query = NULL; ib_sa_client_put(query->sa_query.client); free_mad(&query->sa_query); err1: kfree(query); return ret; } EXPORT_SYMBOL(ib_sa_service_rec_query); static void ib_sa_mcmember_rec_callback(struct ib_sa_query *sa_query, int status, struct ib_sa_mad *mad) { struct ib_sa_mcmember_query *query = container_of(sa_query, struct ib_sa_mcmember_query, sa_query); if (mad) { struct ib_sa_mcmember_rec rec; ib_unpack(mcmember_rec_table, ARRAY_SIZE(mcmember_rec_table), mad->data, &rec); query->callback(status, &rec, query->context); } else query->callback(status, NULL, query->context); } static void ib_sa_mcmember_rec_release(struct ib_sa_query *sa_query) { kfree(container_of(sa_query, struct ib_sa_mcmember_query, sa_query)); } int ib_sa_mcmember_rec_query(struct ib_sa_client *client, struct ib_device *device, u8 port_num, u8 method, struct ib_sa_mcmember_rec *rec, ib_sa_comp_mask comp_mask, int timeout_ms, gfp_t gfp_mask, void (*callback)(int status, struct ib_sa_mcmember_rec *resp, void *context), void *context, struct ib_sa_query **sa_query) { struct ib_sa_mcmember_query *query; struct ib_sa_device *sa_dev = ib_get_client_data(device, &sa_client); struct ib_sa_port *port; struct ib_mad_agent *agent; struct ib_sa_mad *mad; int ret; if (!sa_dev) return -ENODEV; port = &sa_dev->port[port_num - sa_dev->start_port]; agent = port->agent; query = kmalloc(sizeof *query, gfp_mask); if (!query) return -ENOMEM; query->sa_query.port = port; ret = alloc_mad(&query->sa_query, gfp_mask); if (ret) goto err1; ib_sa_client_get(client); query->sa_query.client = client; query->callback = callback; query->context = context; mad = query->sa_query.mad_buf->mad; init_mad(mad, agent); query->sa_query.callback = callback ? ib_sa_mcmember_rec_callback : NULL; query->sa_query.release = ib_sa_mcmember_rec_release; mad->mad_hdr.method = method; mad->mad_hdr.attr_id = cpu_to_be16(IB_SA_ATTR_MC_MEMBER_REC); mad->sa_hdr.comp_mask = comp_mask; ib_pack(mcmember_rec_table, ARRAY_SIZE(mcmember_rec_table), rec, mad->data); *sa_query = &query->sa_query; ret = send_mad(&query->sa_query, timeout_ms, gfp_mask); if (ret < 0) goto err2; return ret; err2: *sa_query = NULL; ib_sa_client_put(query->sa_query.client); free_mad(&query->sa_query); err1: kfree(query); return ret; } static void ib_sa_inform_callback(struct ib_sa_query *sa_query, int status, struct ib_sa_mad *mad) { struct ib_sa_inform_query *query = container_of(sa_query, struct ib_sa_inform_query, sa_query); if (mad) { struct ib_sa_inform rec; ib_unpack(inform_table, ARRAY_SIZE(inform_table), mad->data, &rec); query->callback(status, &rec, query->context); } else query->callback(status, NULL, query->context); } static void ib_sa_inform_release(struct ib_sa_query *sa_query) { kfree(container_of(sa_query, struct ib_sa_inform_query, sa_query)); +} + +int ib_sa_guid_info_rec_query(struct ib_sa_client *client, + struct ib_device *device, u8 port_num, + struct ib_sa_guidinfo_rec *rec, + ib_sa_comp_mask comp_mask, u8 method, + int timeout_ms, gfp_t gfp_mask, + void (*callback)(int status, + struct ib_sa_guidinfo_rec *resp, + void *context), + void *context, + struct ib_sa_query **sa_query) +{ + // stub function - + // called originally from mad.c under mlx4_ib_init_sriov() + // which calls mlx4_ib_init_alias_guid_service() in alias_GUID.c + // which goes down to this function + + printk("ERROR: function should be called only in SRIOV flow!!!"); + + return 0; } /** * ib_sa_informinfo_query - Start an InformInfo registration. * @client:SA client * @device:device to send query on * @port_num: port number to send query on * @rec:Inform record to send in query * @timeout_ms:time to wait for response * @gfp_mask:GFP mask to use for internal allocations * @callback:function called when notice handler registration completes, * times out or is canceled * @context:opaque user context passed to callback * @sa_query:query context, used to cancel query * * This function sends inform info to register with SA to receive * in-service notice. * The callback function will be called when the query completes (or * fails); status is 0 for a successful response, -EINTR if the query * is canceled, -ETIMEDOUT is the query timed out, or -EIO if an error * occurred sending the query. The resp parameter of the callback is * only valid if status is 0. * * If the return value of ib_sa_inform_query() is negative, it is an * error code. Otherwise it is a query ID that can be used to cancel * the query. */ int ib_sa_informinfo_query(struct ib_sa_client *client, struct ib_device *device, u8 port_num, struct ib_sa_inform *rec, int timeout_ms, gfp_t gfp_mask, void (*callback)(int status, struct ib_sa_inform *resp, void *context), void *context, struct ib_sa_query **sa_query) { struct ib_sa_inform_query *query; struct ib_sa_device *sa_dev = ib_get_client_data(device, &sa_client); struct ib_sa_port *port; struct ib_mad_agent *agent; struct ib_sa_mad *mad; int ret; if (!sa_dev) return -ENODEV; port = &sa_dev->port[port_num - sa_dev->start_port]; agent = port->agent; query = kmalloc(sizeof *query, gfp_mask); if (!query) return -ENOMEM; query->sa_query.port = port; ret = alloc_mad(&query->sa_query, gfp_mask); if (ret) goto err1; ib_sa_client_get(client); query->sa_query.client = client; query->callback = callback; query->context = context; mad = query->sa_query.mad_buf->mad; init_mad(mad, agent); query->sa_query.callback = callback ? ib_sa_inform_callback : NULL; query->sa_query.release = ib_sa_inform_release; query->sa_query.port = port; mad->mad_hdr.method = IB_MGMT_METHOD_SET; mad->mad_hdr.attr_id = cpu_to_be16(IB_SA_ATTR_INFORM_INFO); ib_pack(inform_table, ARRAY_SIZE(inform_table), rec, mad->data); *sa_query = &query->sa_query; ret = send_mad(&query->sa_query, timeout_ms, gfp_mask); if (ret < 0) goto err2; return ret; err2: *sa_query = NULL; ib_sa_client_put(query->sa_query.client); free_mad(&query->sa_query); err1: kfree(query); return ret; } static void ib_sa_notice_resp(struct ib_sa_port *port, struct ib_mad_recv_wc *mad_recv_wc) { struct ib_mad_send_buf *mad_buf; struct ib_sa_mad *mad; int ret; unsigned long flags; mad_buf = ib_create_send_mad(port->notice_agent, 1, 0, 0, IB_MGMT_SA_HDR, IB_MGMT_SA_DATA, GFP_KERNEL); if (IS_ERR(mad_buf)) return; mad = mad_buf->mad; memcpy(mad, mad_recv_wc->recv_buf.mad, sizeof *mad); mad->mad_hdr.method = IB_MGMT_METHOD_REPORT_RESP; spin_lock_irqsave(&port->ah_lock, flags); if (!port->sm_ah) { spin_unlock_irqrestore(&port->ah_lock, flags); ib_free_send_mad(mad_buf); return; } kref_get(&port->sm_ah->ref); mad_buf->context[0] = &port->sm_ah->ref; mad_buf->ah = port->sm_ah->ah; spin_unlock_irqrestore(&port->ah_lock, flags); ret = ib_post_send_mad(mad_buf, NULL); if (ret) goto err; return; err: kref_put(mad_buf->context[0], free_sm_ah); ib_free_send_mad(mad_buf); } static void send_handler(struct ib_mad_agent *agent, struct ib_mad_send_wc *mad_send_wc) { struct ib_sa_query *query = mad_send_wc->send_buf->context[0]; unsigned long flags; if (query->callback) switch (mad_send_wc->status) { case IB_WC_SUCCESS: /* No callback -- already got recv */ break; case IB_WC_RESP_TIMEOUT_ERR: query->callback(query, -ETIMEDOUT, NULL); break; case IB_WC_WR_FLUSH_ERR: query->callback(query, -EINTR, NULL); break; default: query->callback(query, -EIO, NULL); break; } spin_lock_irqsave(&idr_lock, flags); idr_remove(&query_idr, query->id); spin_unlock_irqrestore(&idr_lock, flags); free_mad(query); ib_sa_client_put(query->client); query->release(query); } static void recv_handler(struct ib_mad_agent *mad_agent, struct ib_mad_recv_wc *mad_recv_wc) { struct ib_sa_query *query; struct ib_mad_send_buf *mad_buf; mad_buf = (void *) (unsigned long) mad_recv_wc->wc->wr_id; query = mad_buf->context[0]; if (query->callback) { if (mad_recv_wc->wc->status == IB_WC_SUCCESS) query->callback(query, mad_recv_wc->recv_buf.mad->mad_hdr.status ? -EINVAL : 0, (struct ib_sa_mad *) mad_recv_wc->recv_buf.mad); else query->callback(query, -EIO, NULL); } ib_free_recv_mad(mad_recv_wc); } static void notice_resp_handler(struct ib_mad_agent *agent, struct ib_mad_send_wc *mad_send_wc) { kref_put(mad_send_wc->send_buf->context[0], free_sm_ah); ib_free_send_mad(mad_send_wc->send_buf); } static void notice_handler(struct ib_mad_agent *mad_agent, struct ib_mad_recv_wc *mad_recv_wc) { struct ib_sa_port *port; struct ib_sa_mad *mad; struct ib_sa_notice notice; port = mad_agent->context; mad = (struct ib_sa_mad *) mad_recv_wc->recv_buf.mad; ib_unpack(notice_table, ARRAY_SIZE(notice_table), mad->data, ¬ice); if (!notice_dispatch(port->device, port->port_num, ¬ice)) ib_sa_notice_resp(port, mad_recv_wc); ib_free_recv_mad(mad_recv_wc); } static void ib_sa_add_one(struct ib_device *device) { struct ib_sa_device *sa_dev; struct ib_mad_reg_req reg_req = { .mgmt_class = IB_MGMT_CLASS_SUBN_ADM, .mgmt_class_version = 2 }; int s, e, i; if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB) return; if (device->node_type == RDMA_NODE_IB_SWITCH) s = e = 0; else { s = 1; e = device->phys_port_cnt; } sa_dev = kzalloc(sizeof *sa_dev + (e - s + 1) * sizeof (struct ib_sa_port), GFP_KERNEL); if (!sa_dev) return; sa_dev->start_port = s; sa_dev->end_port = e; for (i = 0; i <= e - s; ++i) { spin_lock_init(&sa_dev->port[i].ah_lock); if (rdma_port_get_link_layer(device, i + 1) != IB_LINK_LAYER_INFINIBAND) continue; sa_dev->port[i].sm_ah = NULL; sa_dev->port[i].port_num = i + s; sa_dev->port[i].agent = ib_register_mad_agent(device, i + s, IB_QPT_GSI, NULL, 0, send_handler, recv_handler, sa_dev); if (IS_ERR(sa_dev->port[i].agent)) goto err; sa_dev->port[i].device = device; set_bit(IB_MGMT_METHOD_REPORT, reg_req.method_mask); sa_dev->port[i].notice_agent = ib_register_mad_agent(device, i + s, IB_QPT_GSI, ®_req, 0, notice_resp_handler, notice_handler, &sa_dev->port[i]); if (IS_ERR(sa_dev->port[i].notice_agent)) goto err; INIT_WORK(&sa_dev->port[i].update_task, update_sm_ah); } ib_set_client_data(device, &sa_client, sa_dev); /* * We register our event handler after everything is set up, * and then update our cached info after the event handler is * registered to avoid any problems if a port changes state * during our initialization. */ INIT_IB_EVENT_HANDLER(&sa_dev->event_handler, device, ib_sa_event); if (ib_register_event_handler(&sa_dev->event_handler)) goto err; for (i = 0; i <= e - s; ++i) if (rdma_port_get_link_layer(device, i + 1) == IB_LINK_LAYER_INFINIBAND) update_sm_ah(&sa_dev->port[i].update_task); return; err: while (--i >= 0) if (rdma_port_get_link_layer(device, i + 1) == IB_LINK_LAYER_INFINIBAND) { if (!IS_ERR(sa_dev->port[i].notice_agent)) ib_unregister_mad_agent(sa_dev->port[i].notice_agent); if (!IS_ERR(sa_dev->port[i].agent)) ib_unregister_mad_agent(sa_dev->port[i].agent); } kfree(sa_dev); return; } static void ib_sa_remove_one(struct ib_device *device) { struct ib_sa_device *sa_dev = ib_get_client_data(device, &sa_client); int i; if (!sa_dev) return; ib_unregister_event_handler(&sa_dev->event_handler); flush_scheduled_work(); for (i = 0; i <= sa_dev->end_port - sa_dev->start_port; ++i) { if (rdma_port_get_link_layer(device, i + 1) == IB_LINK_LAYER_INFINIBAND) { ib_unregister_mad_agent(sa_dev->port[i].notice_agent); ib_unregister_mad_agent(sa_dev->port[i].agent); if (sa_dev->port[i].sm_ah) kref_put(&sa_dev->port[i].sm_ah->ref, free_sm_ah); } } kfree(sa_dev); } static int __init ib_sa_init(void) { int ret; spin_lock_init(&idr_lock); spin_lock_init(&tid_lock); get_random_bytes(&tid, sizeof tid); ret = ib_register_client(&sa_client); if (ret) { printk(KERN_ERR "Couldn't register ib_sa client\n"); goto err1; } ret = mcast_init(); if (ret) { printk(KERN_ERR "Couldn't initialize multicast handling\n"); goto err2; } ret = notice_init(); if (ret) { printk(KERN_ERR "Couldn't initialize notice handling\n"); goto err3; } ret = sa_db_init(); if (ret) { printk(KERN_ERR "Couldn't initialize local SA\n"); goto err4; } return 0; err4: notice_cleanup(); err3: mcast_cleanup(); err2: ib_unregister_client(&sa_client); err1: return ret; } static void __exit ib_sa_cleanup(void) { sa_db_cleanup(); mcast_cleanup(); notice_cleanup(); ib_unregister_client(&sa_client); idr_destroy(&query_idr); } module_init_order(ib_sa_init, SI_ORDER_SECOND); module_exit(ib_sa_cleanup); Index: stable/9/sys/ofed/drivers/infiniband/core/sysfs.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/sysfs.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/sysfs.c (revision 273864) @@ -1,911 +1,981 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "core_priv.h" #include #include +#include #include +#include struct ib_port { struct kobject kobj; struct ib_device *ibdev; struct attribute_group gid_group; struct attribute_group pkey_group; u8 port_num; }; struct port_attribute { struct attribute attr; ssize_t (*show)(struct ib_port *, struct port_attribute *, char *buf); ssize_t (*store)(struct ib_port *, struct port_attribute *, const char *buf, size_t count); }; #define PORT_ATTR(_name, _mode, _show, _store) \ struct port_attribute port_attr_##_name = __ATTR(_name, _mode, _show, _store) #define PORT_ATTR_RO(_name) \ struct port_attribute port_attr_##_name = __ATTR_RO(_name) struct port_table_attribute { struct port_attribute attr; char name[8]; int index; }; static ssize_t port_attr_show(struct kobject *kobj, struct attribute *attr, char *buf) { struct port_attribute *port_attr = container_of(attr, struct port_attribute, attr); struct ib_port *p = container_of(kobj, struct ib_port, kobj); if (!port_attr->show) return -EIO; return port_attr->show(p, port_attr, buf); } static const struct sysfs_ops port_sysfs_ops = { .show = port_attr_show }; static ssize_t state_show(struct ib_port *p, struct port_attribute *unused, char *buf) { struct ib_port_attr attr; ssize_t ret; static const char *state_name[] = { [IB_PORT_NOP] = "NOP", [IB_PORT_DOWN] = "DOWN", [IB_PORT_INIT] = "INIT", [IB_PORT_ARMED] = "ARMED", [IB_PORT_ACTIVE] = "ACTIVE", [IB_PORT_ACTIVE_DEFER] = "ACTIVE_DEFER" }; ret = ib_query_port(p->ibdev, p->port_num, &attr); if (ret) return ret; return sprintf(buf, "%d: %s\n", attr.state, - attr.state >= 0 && attr.state < ARRAY_SIZE(state_name) ? + attr.state < ARRAY_SIZE(state_name) ? state_name[attr.state] : "UNKNOWN"); } static ssize_t lid_show(struct ib_port *p, struct port_attribute *unused, char *buf) { struct ib_port_attr attr; ssize_t ret; ret = ib_query_port(p->ibdev, p->port_num, &attr); if (ret) return ret; return sprintf(buf, "0x%x\n", attr.lid); } static ssize_t lid_mask_count_show(struct ib_port *p, struct port_attribute *unused, char *buf) { struct ib_port_attr attr; ssize_t ret; ret = ib_query_port(p->ibdev, p->port_num, &attr); if (ret) return ret; return sprintf(buf, "%d\n", attr.lmc); } static ssize_t sm_lid_show(struct ib_port *p, struct port_attribute *unused, char *buf) { struct ib_port_attr attr; ssize_t ret; ret = ib_query_port(p->ibdev, p->port_num, &attr); if (ret) return ret; return sprintf(buf, "0x%x\n", attr.sm_lid); } static ssize_t sm_sl_show(struct ib_port *p, struct port_attribute *unused, char *buf) { struct ib_port_attr attr; ssize_t ret; ret = ib_query_port(p->ibdev, p->port_num, &attr); if (ret) return ret; return sprintf(buf, "%d\n", attr.sm_sl); } static ssize_t cap_mask_show(struct ib_port *p, struct port_attribute *unused, char *buf) { struct ib_port_attr attr; ssize_t ret; ret = ib_query_port(p->ibdev, p->port_num, &attr); if (ret) return ret; return sprintf(buf, "0x%08x\n", attr.port_cap_flags); } static ssize_t rate_show(struct ib_port *p, struct port_attribute *unused, char *buf) { struct ib_port_attr attr; char *speed = ""; int rate; ssize_t ret; ret = ib_query_port(p->ibdev, p->port_num, &attr); if (ret) return ret; switch (attr.active_speed) { case 2: speed = " DDR"; break; case 4: speed = " QDR"; break; } rate = 25 * ib_width_enum_to_int(attr.active_width) * attr.active_speed; if (rate < 0) return -EINVAL; return sprintf(buf, "%d%s Gb/sec (%dX%s)\n", rate / 10, rate % 10 ? ".5" : "", ib_width_enum_to_int(attr.active_width), speed); } static ssize_t phys_state_show(struct ib_port *p, struct port_attribute *unused, char *buf) { struct ib_port_attr attr; ssize_t ret; ret = ib_query_port(p->ibdev, p->port_num, &attr); if (ret) return ret; switch (attr.phys_state) { case 1: return sprintf(buf, "1: Sleep\n"); case 2: return sprintf(buf, "2: Polling\n"); case 3: return sprintf(buf, "3: Disabled\n"); case 4: return sprintf(buf, "4: PortConfigurationTraining\n"); case 5: return sprintf(buf, "5: LinkUp\n"); case 6: return sprintf(buf, "6: LinkErrorRecovery\n"); case 7: return sprintf(buf, "7: Phy Test\n"); default: return sprintf(buf, "%d: \n", attr.phys_state); } } static ssize_t link_layer_show(struct ib_port *p, struct port_attribute *unused, char *buf) { switch (rdma_port_get_link_layer(p->ibdev, p->port_num)) { case IB_LINK_LAYER_INFINIBAND: return sprintf(buf, "%s\n", "IB"); case IB_LINK_LAYER_ETHERNET: return sprintf(buf, "%s\n", "Ethernet"); default: return sprintf(buf, "%s\n", "Unknown"); } } static PORT_ATTR_RO(state); static PORT_ATTR_RO(lid); static PORT_ATTR_RO(lid_mask_count); static PORT_ATTR_RO(sm_lid); static PORT_ATTR_RO(sm_sl); static PORT_ATTR_RO(cap_mask); static PORT_ATTR_RO(rate); static PORT_ATTR_RO(phys_state); static PORT_ATTR_RO(link_layer); static struct attribute *port_default_attrs[] = { &port_attr_state.attr, &port_attr_lid.attr, &port_attr_lid_mask_count.attr, &port_attr_sm_lid.attr, &port_attr_sm_sl.attr, &port_attr_cap_mask.attr, &port_attr_rate.attr, &port_attr_phys_state.attr, &port_attr_link_layer.attr, NULL }; static ssize_t show_port_gid(struct ib_port *p, struct port_attribute *attr, char *buf) { struct port_table_attribute *tab_attr = container_of(attr, struct port_table_attribute, attr); union ib_gid gid; ssize_t ret; u16 *raw; ret = ib_query_gid(p->ibdev, p->port_num, tab_attr->index, &gid); if (ret) return ret; raw = (u16 *)gid.raw; return sprintf(buf, "%.4x:%.4x:%.4x:%.4x:%.4x:%.4x:%.4x:%.4x\n", htons(raw[0]), htons(raw[1]), htons(raw[2]), htons(raw[3]), htons(raw[4]), htons(raw[5]), htons(raw[6]), htons(raw[7])); } static ssize_t show_port_pkey(struct ib_port *p, struct port_attribute *attr, char *buf) { struct port_table_attribute *tab_attr = container_of(attr, struct port_table_attribute, attr); u16 pkey; ssize_t ret; ret = ib_query_pkey(p->ibdev, p->port_num, tab_attr->index, &pkey); if (ret) return ret; return sprintf(buf, "0x%04x\n", pkey); } -#define PORT_PMA_ATTR(_name, _counter, _width, _offset) \ -struct port_table_attribute port_pma_attr_##_name = { \ - .attr = __ATTR(_name, S_IRUGO, show_pma_counter, NULL), \ - .index = (_offset) | ((_width) << 16) | ((_counter) << 24) \ -} - -static ssize_t show_pma_counter(struct ib_port *p, struct port_attribute *attr, - char *buf) +static ssize_t get_pma_counters(struct ib_port *p, struct port_attribute *attr, + char *buf, int c_ext) { - struct port_table_attribute *tab_attr = - container_of(attr, struct port_table_attribute, attr); - int offset = tab_attr->index & 0xffff; - int width = (tab_attr->index >> 16) & 0xff; - struct ib_mad *in_mad = NULL; - struct ib_mad *out_mad = NULL; - ssize_t ret; + struct port_table_attribute *tab_attr = + container_of(attr, struct port_table_attribute, attr); + int offset = tab_attr->index & 0xffff; + int width = (tab_attr->index >> 16) & 0xff; + struct ib_mad *in_mad = NULL; + struct ib_mad *out_mad = NULL; + ssize_t ret; - if (!p->ibdev->process_mad) - return sprintf(buf, "N/A (no PMA)\n"); + if (!p->ibdev->process_mad) + return -ENXIO; - in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) { - ret = -ENOMEM; - goto out; - } + in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); + out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); + if (!in_mad || !out_mad) { + ret = -ENOMEM; + goto out; + } - in_mad->mad_hdr.base_version = 1; - in_mad->mad_hdr.mgmt_class = IB_MGMT_CLASS_PERF_MGMT; - in_mad->mad_hdr.class_version = 1; - in_mad->mad_hdr.method = IB_MGMT_METHOD_GET; - in_mad->mad_hdr.attr_id = cpu_to_be16(0x12); /* PortCounters */ + in_mad->mad_hdr.base_version = 1; + in_mad->mad_hdr.mgmt_class = IB_MGMT_CLASS_PERF_MGMT; + in_mad->mad_hdr.class_version = 1; + in_mad->mad_hdr.method = IB_MGMT_METHOD_GET; + if (c_ext) + in_mad->mad_hdr.attr_id = IB_PMA_PORT_COUNTERS_EXT; + else + in_mad->mad_hdr.attr_id = IB_PMA_PORT_COUNTERS; - in_mad->data[41] = p->port_num; /* PortSelect field */ + in_mad->data[41] = p->port_num; /* PortSelect field */ - if ((p->ibdev->process_mad(p->ibdev, IB_MAD_IGNORE_MKEY, - p->port_num, NULL, NULL, in_mad, out_mad) & - (IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY)) != - (IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY)) { - ret = -EINVAL; - goto out; - } + if ((p->ibdev->process_mad(p->ibdev, IB_MAD_IGNORE_MKEY, + p->port_num, NULL, NULL, in_mad, out_mad) & + (IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY)) != + (IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY)) { + ret = -EINVAL; + goto out; + } - switch (width) { - case 4: - ret = sprintf(buf, "%u\n", (out_mad->data[40 + offset / 8] >> - (4 - (offset % 8))) & 0xf); - break; - case 8: - ret = sprintf(buf, "%u\n", out_mad->data[40 + offset / 8]); - break; - case 16: - ret = sprintf(buf, "%u\n", - be16_to_cpup((__be16 *)(out_mad->data + 40 + offset / 8))); - break; - case 32: - ret = sprintf(buf, "%u\n", - be32_to_cpup((__be32 *)(out_mad->data + 40 + offset / 8))); - break; - default: - ret = 0; - } + switch (width) { + case 4: + ret = sprintf(buf, "%u\n", (out_mad->data[40 + offset / 8] >> + (4 - (offset % 8))) & 0xf); + break; + case 8: + ret = sprintf(buf, "%u\n", out_mad->data[40 + offset / 8]); + break; + case 16: + ret = sprintf(buf, "%u\n", + be16_to_cpup((__be16 *)(out_mad->data + 40 + offset / 8))); + break; + case 32: + ret = sprintf(buf, "%u\n", + be32_to_cpup((__be32 *)(out_mad->data + 40 + offset / 8))); + break; + case 64: + ret = sprintf(buf, "%llu\n", (unsigned long long) + be64_to_cpup((__be64 *)(out_mad->data + 40 + offset / 8))); + break; + default: + ret = 0; + } out: - kfree(in_mad); - kfree(out_mad); + kfree(in_mad); + kfree(out_mad); - return ret; + return ret; } -static PORT_PMA_ATTR(symbol_error , 0, 16, 32); -static PORT_PMA_ATTR(link_error_recovery , 1, 8, 48); -static PORT_PMA_ATTR(link_downed , 2, 8, 56); -static PORT_PMA_ATTR(port_rcv_errors , 3, 16, 64); +#define PORT_PMA_ATTR(_name, _counter, _width, _offset) \ +struct port_table_attribute port_pma_attr_##_name = { \ + .attr = __ATTR(_name, S_IRUGO, show_pma_counter, NULL), \ + .index = (_offset) | ((_width) << 16) | ((_counter) << 24) \ +} + +static ssize_t show_pma_counter(struct ib_port *p, struct port_attribute *attr, + char *buf) +{ + return get_pma_counters(p, attr, buf, 0); +} + +static PORT_PMA_ATTR(symbol_error , 0, 16, 32); +static PORT_PMA_ATTR(link_error_recovery , 1, 8, 48); +static PORT_PMA_ATTR(link_downed , 2, 8, 56); +static PORT_PMA_ATTR(port_rcv_errors , 3, 16, 64); static PORT_PMA_ATTR(port_rcv_remote_physical_errors, 4, 16, 80); static PORT_PMA_ATTR(port_rcv_switch_relay_errors , 5, 16, 96); -static PORT_PMA_ATTR(port_xmit_discards , 6, 16, 112); +static PORT_PMA_ATTR(port_xmit_discards , 6, 16, 112); static PORT_PMA_ATTR(port_xmit_constraint_errors , 7, 8, 128); -static PORT_PMA_ATTR(port_rcv_constraint_errors , 8, 8, 136); +static PORT_PMA_ATTR(port_rcv_constraint_errors , 8, 8, 136); static PORT_PMA_ATTR(local_link_integrity_errors , 9, 4, 152); static PORT_PMA_ATTR(excessive_buffer_overrun_errors, 10, 4, 156); -static PORT_PMA_ATTR(VL15_dropped , 11, 16, 176); -static PORT_PMA_ATTR(port_xmit_data , 12, 32, 192); -static PORT_PMA_ATTR(port_rcv_data , 13, 32, 224); -static PORT_PMA_ATTR(port_xmit_packets , 14, 32, 256); -static PORT_PMA_ATTR(port_rcv_packets , 15, 32, 288); -/* - * There is no bit allocated for port_xmit_wait in the CounterSelect field - * (IB spec). However, since this bit is ignored when reading - * (show_pma_counter), the _counter field of port_xmit_wait can be set to zero. - */ -static PORT_PMA_ATTR(port_xmit_wait , 0, 32, 320); +static PORT_PMA_ATTR(VL15_dropped , 11, 16, 176); +static PORT_PMA_ATTR(port_xmit_data , 12, 32, 192); +static PORT_PMA_ATTR(port_rcv_data , 13, 32, 224); +static PORT_PMA_ATTR(port_xmit_packets , 14, 32, 256); +static PORT_PMA_ATTR(port_rcv_packets , 15, 32, 288); static struct attribute *pma_attrs[] = { - &port_pma_attr_symbol_error.attr.attr, - &port_pma_attr_link_error_recovery.attr.attr, - &port_pma_attr_link_downed.attr.attr, - &port_pma_attr_port_rcv_errors.attr.attr, - &port_pma_attr_port_rcv_remote_physical_errors.attr.attr, - &port_pma_attr_port_rcv_switch_relay_errors.attr.attr, - &port_pma_attr_port_xmit_discards.attr.attr, - &port_pma_attr_port_xmit_constraint_errors.attr.attr, - &port_pma_attr_port_rcv_constraint_errors.attr.attr, - &port_pma_attr_local_link_integrity_errors.attr.attr, - &port_pma_attr_excessive_buffer_overrun_errors.attr.attr, - &port_pma_attr_VL15_dropped.attr.attr, - &port_pma_attr_port_xmit_data.attr.attr, - &port_pma_attr_port_rcv_data.attr.attr, - &port_pma_attr_port_xmit_packets.attr.attr, - &port_pma_attr_port_rcv_packets.attr.attr, - &port_pma_attr_port_xmit_wait.attr.attr, - NULL + &port_pma_attr_symbol_error.attr.attr, + &port_pma_attr_link_error_recovery.attr.attr, + &port_pma_attr_link_downed.attr.attr, + &port_pma_attr_port_rcv_errors.attr.attr, + &port_pma_attr_port_rcv_remote_physical_errors.attr.attr, + &port_pma_attr_port_rcv_switch_relay_errors.attr.attr, + &port_pma_attr_port_xmit_discards.attr.attr, + &port_pma_attr_port_xmit_constraint_errors.attr.attr, + &port_pma_attr_port_rcv_constraint_errors.attr.attr, + &port_pma_attr_local_link_integrity_errors.attr.attr, + &port_pma_attr_excessive_buffer_overrun_errors.attr.attr, + &port_pma_attr_VL15_dropped.attr.attr, + &port_pma_attr_port_xmit_data.attr.attr, + &port_pma_attr_port_rcv_data.attr.attr, + &port_pma_attr_port_xmit_packets.attr.attr, + &port_pma_attr_port_rcv_packets.attr.attr, + NULL }; static struct attribute_group pma_group = { .name = "counters", .attrs = pma_attrs }; +#define PORT_PMA_ATTR_EXT(_name, _counter, _width, _offset) \ +struct port_table_attribute port_pma_attr_ext_##_name = { \ + .attr = __ATTR(_name, S_IRUGO, show_pma_counter_ext, NULL), \ + .index = (_offset) | ((_width) << 16) | ((_counter) << 24) \ +} + +static ssize_t show_pma_counter_ext(struct ib_port *p, + struct port_attribute *attr, char *buf) +{ + return get_pma_counters(p, attr, buf, 1); +} + +static PORT_PMA_ATTR_EXT(port_xmit_data_64 , 0, 64, 64); +static PORT_PMA_ATTR_EXT(port_rcv_data_64 , 0, 64, 128); +static PORT_PMA_ATTR_EXT(port_xmit_packets_64 , 0, 64, 192); +static PORT_PMA_ATTR_EXT(port_rcv_packets_64 , 0, 64, 256); +static PORT_PMA_ATTR_EXT(port_unicast_xmit_packets , 0, 64, 320); +static PORT_PMA_ATTR_EXT(port_unicast_rcv_packets , 0, 64, 384); +static PORT_PMA_ATTR_EXT(port_multicast_xmit_packets , 0, 64, 448); +static PORT_PMA_ATTR_EXT(port_multicast_rcv_packets , 0, 64, 512); + +static struct attribute *pma_attrs_ext[] = { + &port_pma_attr_ext_port_xmit_data_64.attr.attr, + &port_pma_attr_ext_port_rcv_data_64.attr.attr, + &port_pma_attr_ext_port_xmit_packets_64.attr.attr, + &port_pma_attr_ext_port_rcv_packets_64.attr.attr, + &port_pma_attr_ext_port_unicast_xmit_packets.attr.attr, + &port_pma_attr_ext_port_unicast_rcv_packets.attr.attr, + &port_pma_attr_ext_port_multicast_xmit_packets.attr.attr, + &port_pma_attr_ext_port_multicast_rcv_packets.attr.attr, + NULL +}; + +static struct attribute_group pma_ext_group = { + .name = "counters_ext", + .attrs = pma_attrs_ext +}; + static void ib_port_release(struct kobject *kobj) { struct ib_port *p = container_of(kobj, struct ib_port, kobj); struct attribute *a; int i; for (i = 0; (a = p->gid_group.attrs[i]); ++i) kfree(a); kfree(p->gid_group.attrs); for (i = 0; (a = p->pkey_group.attrs[i]); ++i) kfree(a); kfree(p->pkey_group.attrs); kfree(p); } static struct kobj_type port_type = { .release = ib_port_release, .sysfs_ops = &port_sysfs_ops, .default_attrs = port_default_attrs }; static void ib_device_release(struct device *device) { struct ib_device *dev = container_of(device, struct ib_device, dev); kfree(dev); } #ifdef __linux__ /* BSD supports this through devfs(5) and devd(8). */ static int ib_device_uevent(struct device *device, struct kobj_uevent_env *env) { struct ib_device *dev = container_of(device, struct ib_device, dev); if (add_uevent_var(env, "NAME=%s", dev->name)) return -ENOMEM; /* * It would be nice to pass the node GUID with the event... */ return 0; } #endif static struct attribute ** alloc_group_attrs(ssize_t (*show)(struct ib_port *, struct port_attribute *, char *buf), int len) { struct attribute **tab_attr; struct port_table_attribute *element; int i; tab_attr = kcalloc(1 + len, sizeof(struct attribute *), GFP_KERNEL); if (!tab_attr) return NULL; for (i = 0; i < len; i++) { element = kzalloc(sizeof(struct port_table_attribute), GFP_KERNEL); if (!element) goto err; if (snprintf(element->name, sizeof(element->name), "%d", i) >= sizeof(element->name)) { kfree(element); goto err; } element->attr.attr.name = element->name; element->attr.attr.mode = S_IRUGO; element->attr.show = show; element->index = i; tab_attr[i] = &element->attr.attr; } return tab_attr; err: while (--i >= 0) kfree(tab_attr[i]); kfree(tab_attr); return NULL; } -static int add_port(struct ib_device *device, int port_num) +static int add_port(struct ib_device *device, int port_num, + int (*port_callback)(struct ib_device *, + u8, struct kobject *)) { struct ib_port *p; struct ib_port_attr attr; int i; int ret; ret = ib_query_port(device, port_num, &attr); if (ret) return ret; p = kzalloc(sizeof *p, GFP_KERNEL); if (!p) return -ENOMEM; p->ibdev = device; p->port_num = port_num; ret = kobject_init_and_add(&p->kobj, &port_type, - device->ports_parent, + kobject_get(device->ports_parent), "%d", port_num); if (ret) goto err_put; ret = sysfs_create_group(&p->kobj, &pma_group); if (ret) goto err_put; + ret = sysfs_create_group(&p->kobj, &pma_ext_group); + if (ret) + goto err_remove_pma; + p->gid_group.name = "gids"; p->gid_group.attrs = alloc_group_attrs(show_port_gid, attr.gid_tbl_len); if (!p->gid_group.attrs) - goto err_remove_pma; + goto err_remove_pma_ext; ret = sysfs_create_group(&p->kobj, &p->gid_group); if (ret) goto err_free_gid; p->pkey_group.name = "pkeys"; p->pkey_group.attrs = alloc_group_attrs(show_port_pkey, attr.pkey_tbl_len); if (!p->pkey_group.attrs) goto err_remove_gid; ret = sysfs_create_group(&p->kobj, &p->pkey_group); if (ret) goto err_free_pkey; + if (port_callback) { + ret = port_callback(device, port_num, &p->kobj); + if (ret) + goto err_remove_pkey; + } + list_add_tail(&p->kobj.entry, &device->port_list); #ifdef __linux__ kobject_uevent(&p->kobj, KOBJ_ADD); #endif return 0; +err_remove_pkey: + sysfs_remove_group(&p->kobj, &p->pkey_group); + err_free_pkey: for (i = 0; i < attr.pkey_tbl_len; ++i) kfree(p->pkey_group.attrs[i]); kfree(p->pkey_group.attrs); err_remove_gid: sysfs_remove_group(&p->kobj, &p->gid_group); err_free_gid: for (i = 0; i < attr.gid_tbl_len; ++i) kfree(p->gid_group.attrs[i]); kfree(p->gid_group.attrs); +err_remove_pma_ext: + sysfs_remove_group(&p->kobj, &pma_ext_group); + err_remove_pma: sysfs_remove_group(&p->kobj, &pma_group); err_put: kobject_put(device->ports_parent); kfree(p); return ret; } static ssize_t show_node_type(struct device *device, struct device_attribute *attr, char *buf) { struct ib_device *dev = container_of(device, struct ib_device, dev); switch (dev->node_type) { case RDMA_NODE_IB_CA: return sprintf(buf, "%d: CA\n", dev->node_type); case RDMA_NODE_RNIC: return sprintf(buf, "%d: RNIC\n", dev->node_type); case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev->node_type); case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev->node_type); default: return sprintf(buf, "%d: \n", dev->node_type); } } static ssize_t show_sys_image_guid(struct device *device, struct device_attribute *dev_attr, char *buf) { struct ib_device *dev = container_of(device, struct ib_device, dev); struct ib_device_attr attr; ssize_t ret; ret = ib_query_device(dev, &attr); if (ret) return ret; return sprintf(buf, "%04x:%04x:%04x:%04x\n", be16_to_cpu(((__be16 *) &attr.sys_image_guid)[0]), be16_to_cpu(((__be16 *) &attr.sys_image_guid)[1]), be16_to_cpu(((__be16 *) &attr.sys_image_guid)[2]), be16_to_cpu(((__be16 *) &attr.sys_image_guid)[3])); } static ssize_t show_node_guid(struct device *device, struct device_attribute *attr, char *buf) { struct ib_device *dev = container_of(device, struct ib_device, dev); return sprintf(buf, "%04x:%04x:%04x:%04x\n", be16_to_cpu(((__be16 *) &dev->node_guid)[0]), be16_to_cpu(((__be16 *) &dev->node_guid)[1]), be16_to_cpu(((__be16 *) &dev->node_guid)[2]), be16_to_cpu(((__be16 *) &dev->node_guid)[3])); } static ssize_t show_node_desc(struct device *device, struct device_attribute *attr, char *buf) { struct ib_device *dev = container_of(device, struct ib_device, dev); return sprintf(buf, "%.64s\n", dev->node_desc); } static ssize_t set_node_desc(struct device *device, struct device_attribute *attr, const char *buf, size_t count) { struct ib_device *dev = container_of(device, struct ib_device, dev); struct ib_device_modify desc = {}; int ret; if (!dev->modify_device) return -EIO; memcpy(desc.node_desc, buf, min_t(int, count, 64)); ret = ib_modify_device(dev, IB_DEVICE_MODIFY_NODE_DESC, &desc); if (ret) return ret; return count; } static DEVICE_ATTR(node_type, S_IRUGO, show_node_type, NULL); static DEVICE_ATTR(sys_image_guid, S_IRUGO, show_sys_image_guid, NULL); static DEVICE_ATTR(node_guid, S_IRUGO, show_node_guid, NULL); static DEVICE_ATTR(node_desc, S_IRUGO | S_IWUSR, show_node_desc, set_node_desc); static struct device_attribute *ib_class_attributes[] = { &dev_attr_node_type, &dev_attr_sys_image_guid, &dev_attr_node_guid, &dev_attr_node_desc }; static struct class ib_class = { .name = "infiniband", .dev_release = ib_device_release, #ifdef __linux__ .dev_uevent = ib_device_uevent, #endif }; /* Show a given an attribute in the statistics group */ static ssize_t show_protocol_stat(const struct device *device, struct device_attribute *attr, char *buf, unsigned offset) { struct ib_device *dev = container_of(__DECONST(struct device *, device), struct ib_device, dev); union rdma_protocol_stats stats; ssize_t ret; ret = dev->get_protocol_stats(dev, &stats); if (ret) return ret; return sprintf(buf, "%llu\n", (unsigned long long) ((u64 *) &stats)[offset]); } /* generate a read-only iwarp statistics attribute */ #define IW_STATS_ENTRY(name) \ static ssize_t show_##name(struct device *device, \ struct device_attribute *attr, char *buf) \ { \ return show_protocol_stat(device, attr, buf, \ offsetof(struct iw_protocol_stats, name) / \ sizeof (u64)); \ } \ static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL) IW_STATS_ENTRY(ipInReceives); IW_STATS_ENTRY(ipInHdrErrors); IW_STATS_ENTRY(ipInTooBigErrors); IW_STATS_ENTRY(ipInNoRoutes); IW_STATS_ENTRY(ipInAddrErrors); IW_STATS_ENTRY(ipInUnknownProtos); IW_STATS_ENTRY(ipInTruncatedPkts); IW_STATS_ENTRY(ipInDiscards); IW_STATS_ENTRY(ipInDelivers); IW_STATS_ENTRY(ipOutForwDatagrams); IW_STATS_ENTRY(ipOutRequests); IW_STATS_ENTRY(ipOutDiscards); IW_STATS_ENTRY(ipOutNoRoutes); IW_STATS_ENTRY(ipReasmTimeout); IW_STATS_ENTRY(ipReasmReqds); IW_STATS_ENTRY(ipReasmOKs); IW_STATS_ENTRY(ipReasmFails); IW_STATS_ENTRY(ipFragOKs); IW_STATS_ENTRY(ipFragFails); IW_STATS_ENTRY(ipFragCreates); IW_STATS_ENTRY(ipInMcastPkts); IW_STATS_ENTRY(ipOutMcastPkts); IW_STATS_ENTRY(ipInBcastPkts); IW_STATS_ENTRY(ipOutBcastPkts); IW_STATS_ENTRY(tcpRtoAlgorithm); IW_STATS_ENTRY(tcpRtoMin); IW_STATS_ENTRY(tcpRtoMax); IW_STATS_ENTRY(tcpMaxConn); IW_STATS_ENTRY(tcpActiveOpens); IW_STATS_ENTRY(tcpPassiveOpens); IW_STATS_ENTRY(tcpAttemptFails); IW_STATS_ENTRY(tcpEstabResets); IW_STATS_ENTRY(tcpCurrEstab); IW_STATS_ENTRY(tcpInSegs); IW_STATS_ENTRY(tcpOutSegs); IW_STATS_ENTRY(tcpRetransSegs); IW_STATS_ENTRY(tcpInErrs); IW_STATS_ENTRY(tcpOutRsts); static struct attribute *iw_proto_stats_attrs[] = { &dev_attr_ipInReceives.attr, &dev_attr_ipInHdrErrors.attr, &dev_attr_ipInTooBigErrors.attr, &dev_attr_ipInNoRoutes.attr, &dev_attr_ipInAddrErrors.attr, &dev_attr_ipInUnknownProtos.attr, &dev_attr_ipInTruncatedPkts.attr, &dev_attr_ipInDiscards.attr, &dev_attr_ipInDelivers.attr, &dev_attr_ipOutForwDatagrams.attr, &dev_attr_ipOutRequests.attr, &dev_attr_ipOutDiscards.attr, &dev_attr_ipOutNoRoutes.attr, &dev_attr_ipReasmTimeout.attr, &dev_attr_ipReasmReqds.attr, &dev_attr_ipReasmOKs.attr, &dev_attr_ipReasmFails.attr, &dev_attr_ipFragOKs.attr, &dev_attr_ipFragFails.attr, &dev_attr_ipFragCreates.attr, &dev_attr_ipInMcastPkts.attr, &dev_attr_ipOutMcastPkts.attr, &dev_attr_ipInBcastPkts.attr, &dev_attr_ipOutBcastPkts.attr, &dev_attr_tcpRtoAlgorithm.attr, &dev_attr_tcpRtoMin.attr, &dev_attr_tcpRtoMax.attr, &dev_attr_tcpMaxConn.attr, &dev_attr_tcpActiveOpens.attr, &dev_attr_tcpPassiveOpens.attr, &dev_attr_tcpAttemptFails.attr, &dev_attr_tcpEstabResets.attr, &dev_attr_tcpCurrEstab.attr, &dev_attr_tcpInSegs.attr, &dev_attr_tcpOutSegs.attr, &dev_attr_tcpRetransSegs.attr, &dev_attr_tcpInErrs.attr, &dev_attr_tcpOutRsts.attr, NULL }; static struct attribute_group iw_stats_group = { .name = "proto_stats", .attrs = iw_proto_stats_attrs, }; -int ib_device_register_sysfs(struct ib_device *device) +int ib_device_register_sysfs(struct ib_device *device, + int (*port_callback)(struct ib_device *, u8, struct kobject *)) { struct device *class_dev = &device->dev; int ret; int i; class_dev->class = &ib_class; - class_dev->driver_data = device; class_dev->parent = device->dma_device; - dev_set_name(class_dev, device->name); + dev_set_name(class_dev, device->name); + dev_set_drvdata(class_dev, device); INIT_LIST_HEAD(&device->port_list); ret = device_register(class_dev); if (ret) goto err; for (i = 0; i < ARRAY_SIZE(ib_class_attributes); ++i) { ret = device_create_file(class_dev, ib_class_attributes[i]); if (ret) goto err_unregister; } device->ports_parent = kobject_create_and_add("ports", - &class_dev->kobj); - if (!device->ports_parent) { + kobject_get(&class_dev->kobj)); + if (!device->ports_parent) { ret = -ENOMEM; goto err_put; } if (device->node_type == RDMA_NODE_IB_SWITCH) { - ret = add_port(device, 0); + ret = add_port(device, 0, port_callback); if (ret) goto err_put; } else { for (i = 1; i <= device->phys_port_cnt; ++i) { - ret = add_port(device, i); + ret = add_port(device, i, port_callback); if (ret) goto err_put; } } if (device->node_type == RDMA_NODE_RNIC && device->get_protocol_stats) { ret = sysfs_create_group(&class_dev->kobj, &iw_stats_group); if (ret) goto err_put; } return 0; err_put: { struct kobject *p, *t; struct ib_port *port; list_for_each_entry_safe(p, t, &device->port_list, entry) { list_del(&p->entry); port = container_of(p, struct ib_port, kobj); sysfs_remove_group(p, &pma_group); sysfs_remove_group(p, &port->pkey_group); sysfs_remove_group(p, &port->gid_group); kobject_put(p); } } kobject_put(&class_dev->kobj); err_unregister: device_unregister(class_dev); err: return ret; } void ib_device_unregister_sysfs(struct ib_device *device) { struct kobject *p, *t; struct ib_port *port; + int i; /* Hold kobject until ib_dealloc_device() */ kobject_get(&device->dev.kobj); + for (i = 0; i < ARRAY_SIZE(ib_class_attributes); ++i) { + device_remove_file(&device->dev, ib_class_attributes[i]); + } + list_for_each_entry_safe(p, t, &device->port_list, entry) { list_del(&p->entry); port = container_of(p, struct ib_port, kobj); sysfs_remove_group(p, &pma_group); sysfs_remove_group(p, &port->pkey_group); sysfs_remove_group(p, &port->gid_group); kobject_put(p); } kobject_put(device->ports_parent); device_unregister(&device->dev); } int ib_sysfs_setup(void) { return class_register(&ib_class); } void ib_sysfs_cleanup(void) { class_unregister(&ib_class); } -int ib_sysfs_create_port_files(struct ib_device *device, +/*int ib_sysfs_create_port_files(struct ib_device *device, int (*create)(struct ib_device *dev, u8 port_num, struct kobject *kobj)) { struct kobject *p; struct ib_port *port; int ret = 0; list_for_each_entry(p, &device->port_list, entry) { port = container_of(p, struct ib_port, kobj); ret = create(device, port->port_num, &port->kobj); if (ret) break; } return ret; } -EXPORT_SYMBOL(ib_sysfs_create_port_files); +EXPORT_SYMBOL(ib_sysfs_create_port_files);*/ Index: stable/9/sys/ofed/drivers/infiniband/core/ucm.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/ucm.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/ucm.c (revision 273864) @@ -1,1348 +1,1345 @@ /* * Copyright (c) 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include -#include #include #include #include #include #include #include #include #include #include #include +#include #include #include #include #include MODULE_AUTHOR("Libor Michalek"); MODULE_DESCRIPTION("InfiniBand userspace Connection Manager access"); MODULE_LICENSE("Dual BSD/GPL"); struct ib_ucm_device { int devnum; struct cdev cdev; struct device dev; struct ib_device *ib_dev; }; struct ib_ucm_file { struct mutex file_mutex; struct file *filp; struct ib_ucm_device *device; struct list_head ctxs; struct list_head events; wait_queue_head_t poll_wait; }; struct ib_ucm_context { int id; struct completion comp; atomic_t ref; int events_reported; struct ib_ucm_file *file; struct ib_cm_id *cm_id; __u64 uid; struct list_head events; /* list of pending events. */ struct list_head file_list; /* member in file ctx list */ }; struct ib_ucm_event { struct ib_ucm_context *ctx; struct list_head file_list; /* member in file event list */ struct list_head ctx_list; /* member in ctx event list */ struct ib_cm_id *cm_id; struct ib_ucm_event_resp resp; void *data; void *info; int data_len; int info_len; }; enum { IB_UCM_MAJOR = 231, IB_UCM_BASE_MINOR = 224, IB_UCM_MAX_DEVICES = 32 }; -/* ib_cm and ib_user_cm modules share /sys/class/infiniband_cm */ -extern struct class cm_class; - #define IB_UCM_BASE_DEV MKDEV(IB_UCM_MAJOR, IB_UCM_BASE_MINOR) static void ib_ucm_add_one(struct ib_device *device); static void ib_ucm_remove_one(struct ib_device *device); static struct ib_client ucm_client = { .name = "ucm", .add = ib_ucm_add_one, .remove = ib_ucm_remove_one }; static DEFINE_MUTEX(ctx_id_mutex); static DEFINE_IDR(ctx_id_table); static DECLARE_BITMAP(dev_map, IB_UCM_MAX_DEVICES); static struct ib_ucm_context *ib_ucm_ctx_get(struct ib_ucm_file *file, int id) { struct ib_ucm_context *ctx; mutex_lock(&ctx_id_mutex); ctx = idr_find(&ctx_id_table, id); if (!ctx) ctx = ERR_PTR(-ENOENT); else if (ctx->file != file) ctx = ERR_PTR(-EINVAL); else atomic_inc(&ctx->ref); mutex_unlock(&ctx_id_mutex); return ctx; } static void ib_ucm_ctx_put(struct ib_ucm_context *ctx) { if (atomic_dec_and_test(&ctx->ref)) complete(&ctx->comp); } static inline int ib_ucm_new_cm_id(int event) { return event == IB_CM_REQ_RECEIVED || event == IB_CM_SIDR_REQ_RECEIVED; } static void ib_ucm_cleanup_events(struct ib_ucm_context *ctx) { struct ib_ucm_event *uevent; mutex_lock(&ctx->file->file_mutex); list_del(&ctx->file_list); while (!list_empty(&ctx->events)) { uevent = list_entry(ctx->events.next, struct ib_ucm_event, ctx_list); list_del(&uevent->file_list); list_del(&uevent->ctx_list); mutex_unlock(&ctx->file->file_mutex); /* clear incoming connections. */ if (ib_ucm_new_cm_id(uevent->resp.event)) ib_destroy_cm_id(uevent->cm_id); kfree(uevent); mutex_lock(&ctx->file->file_mutex); } mutex_unlock(&ctx->file->file_mutex); } static struct ib_ucm_context *ib_ucm_ctx_alloc(struct ib_ucm_file *file) { struct ib_ucm_context *ctx; int result; ctx = kzalloc(sizeof *ctx, GFP_KERNEL); if (!ctx) return NULL; atomic_set(&ctx->ref, 1); init_completion(&ctx->comp); ctx->file = file; INIT_LIST_HEAD(&ctx->events); do { result = idr_pre_get(&ctx_id_table, GFP_KERNEL); if (!result) goto error; mutex_lock(&ctx_id_mutex); result = idr_get_new(&ctx_id_table, ctx, &ctx->id); mutex_unlock(&ctx_id_mutex); } while (result == -EAGAIN); if (result) goto error; list_add_tail(&ctx->file_list, &file->ctxs); return ctx; error: kfree(ctx); return NULL; } static void ib_ucm_event_req_get(struct ib_ucm_req_event_resp *ureq, struct ib_cm_req_event_param *kreq) { ureq->remote_ca_guid = kreq->remote_ca_guid; ureq->remote_qkey = kreq->remote_qkey; ureq->remote_qpn = kreq->remote_qpn; ureq->qp_type = kreq->qp_type; ureq->starting_psn = kreq->starting_psn; ureq->responder_resources = kreq->responder_resources; ureq->initiator_depth = kreq->initiator_depth; ureq->local_cm_response_timeout = kreq->local_cm_response_timeout; ureq->flow_control = kreq->flow_control; ureq->remote_cm_response_timeout = kreq->remote_cm_response_timeout; ureq->retry_count = kreq->retry_count; ureq->rnr_retry_count = kreq->rnr_retry_count; ureq->srq = kreq->srq; ureq->port = kreq->port; ib_copy_path_rec_to_user(&ureq->primary_path, kreq->primary_path); if (kreq->alternate_path) ib_copy_path_rec_to_user(&ureq->alternate_path, kreq->alternate_path); } static void ib_ucm_event_rep_get(struct ib_ucm_rep_event_resp *urep, struct ib_cm_rep_event_param *krep) { urep->remote_ca_guid = krep->remote_ca_guid; urep->remote_qkey = krep->remote_qkey; urep->remote_qpn = krep->remote_qpn; urep->starting_psn = krep->starting_psn; urep->responder_resources = krep->responder_resources; urep->initiator_depth = krep->initiator_depth; urep->target_ack_delay = krep->target_ack_delay; urep->failover_accepted = krep->failover_accepted; urep->flow_control = krep->flow_control; urep->rnr_retry_count = krep->rnr_retry_count; urep->srq = krep->srq; } static void ib_ucm_event_sidr_rep_get(struct ib_ucm_sidr_rep_event_resp *urep, struct ib_cm_sidr_rep_event_param *krep) { urep->status = krep->status; urep->qkey = krep->qkey; urep->qpn = krep->qpn; }; static int ib_ucm_event_process(struct ib_cm_event *evt, struct ib_ucm_event *uvt) { void *info = NULL; switch (evt->event) { case IB_CM_REQ_RECEIVED: ib_ucm_event_req_get(&uvt->resp.u.req_resp, &evt->param.req_rcvd); uvt->data_len = IB_CM_REQ_PRIVATE_DATA_SIZE; uvt->resp.present = IB_UCM_PRES_PRIMARY; uvt->resp.present |= (evt->param.req_rcvd.alternate_path ? IB_UCM_PRES_ALTERNATE : 0); break; case IB_CM_REP_RECEIVED: ib_ucm_event_rep_get(&uvt->resp.u.rep_resp, &evt->param.rep_rcvd); uvt->data_len = IB_CM_REP_PRIVATE_DATA_SIZE; break; case IB_CM_RTU_RECEIVED: uvt->data_len = IB_CM_RTU_PRIVATE_DATA_SIZE; uvt->resp.u.send_status = evt->param.send_status; break; case IB_CM_DREQ_RECEIVED: uvt->data_len = IB_CM_DREQ_PRIVATE_DATA_SIZE; uvt->resp.u.send_status = evt->param.send_status; break; case IB_CM_DREP_RECEIVED: uvt->data_len = IB_CM_DREP_PRIVATE_DATA_SIZE; uvt->resp.u.send_status = evt->param.send_status; break; case IB_CM_MRA_RECEIVED: uvt->resp.u.mra_resp.timeout = evt->param.mra_rcvd.service_timeout; uvt->data_len = IB_CM_MRA_PRIVATE_DATA_SIZE; break; case IB_CM_REJ_RECEIVED: uvt->resp.u.rej_resp.reason = evt->param.rej_rcvd.reason; uvt->data_len = IB_CM_REJ_PRIVATE_DATA_SIZE; uvt->info_len = evt->param.rej_rcvd.ari_length; info = evt->param.rej_rcvd.ari; break; case IB_CM_LAP_RECEIVED: ib_copy_path_rec_to_user(&uvt->resp.u.lap_resp.path, evt->param.lap_rcvd.alternate_path); uvt->data_len = IB_CM_LAP_PRIVATE_DATA_SIZE; uvt->resp.present = IB_UCM_PRES_ALTERNATE; break; case IB_CM_APR_RECEIVED: uvt->resp.u.apr_resp.status = evt->param.apr_rcvd.ap_status; uvt->data_len = IB_CM_APR_PRIVATE_DATA_SIZE; uvt->info_len = evt->param.apr_rcvd.info_len; info = evt->param.apr_rcvd.apr_info; break; case IB_CM_SIDR_REQ_RECEIVED: uvt->resp.u.sidr_req_resp.pkey = evt->param.sidr_req_rcvd.pkey; uvt->resp.u.sidr_req_resp.port = evt->param.sidr_req_rcvd.port; uvt->data_len = IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE; break; case IB_CM_SIDR_REP_RECEIVED: ib_ucm_event_sidr_rep_get(&uvt->resp.u.sidr_rep_resp, &evt->param.sidr_rep_rcvd); uvt->data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE; uvt->info_len = evt->param.sidr_rep_rcvd.info_len; info = evt->param.sidr_rep_rcvd.info; break; default: uvt->resp.u.send_status = evt->param.send_status; break; } if (uvt->data_len) { uvt->data = kmemdup(evt->private_data, uvt->data_len, GFP_KERNEL); if (!uvt->data) goto err1; uvt->resp.present |= IB_UCM_PRES_DATA; } if (uvt->info_len) { uvt->info = kmemdup(info, uvt->info_len, GFP_KERNEL); if (!uvt->info) goto err2; uvt->resp.present |= IB_UCM_PRES_INFO; } return 0; err2: kfree(uvt->data); err1: return -ENOMEM; } static int ib_ucm_event_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event) { struct ib_ucm_event *uevent; struct ib_ucm_context *ctx; int result = 0; ctx = cm_id->context; uevent = kzalloc(sizeof *uevent, GFP_KERNEL); if (!uevent) goto err1; uevent->ctx = ctx; uevent->cm_id = cm_id; uevent->resp.uid = ctx->uid; uevent->resp.id = ctx->id; uevent->resp.event = event->event; result = ib_ucm_event_process(event, uevent); if (result) goto err2; mutex_lock(&ctx->file->file_mutex); list_add_tail(&uevent->file_list, &ctx->file->events); list_add_tail(&uevent->ctx_list, &ctx->events); wake_up_interruptible(&ctx->file->poll_wait); if (ctx->file->filp) selwakeup(&ctx->file->filp->f_selinfo); mutex_unlock(&ctx->file->file_mutex); return 0; err2: kfree(uevent); err1: /* Destroy new cm_id's */ return ib_ucm_new_cm_id(event->event); } static ssize_t ib_ucm_event(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { struct ib_ucm_context *ctx; struct ib_ucm_event_get cmd; struct ib_ucm_event *uevent; int result = 0; DEFINE_WAIT(wait); if (out_len < sizeof(struct ib_ucm_event_resp)) return -ENOSPC; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; mutex_lock(&file->file_mutex); while (list_empty(&file->events)) { mutex_unlock(&file->file_mutex); if (file->filp->f_flags & O_NONBLOCK) return -EAGAIN; if (wait_event_interruptible(file->poll_wait, !list_empty(&file->events))) return -ERESTARTSYS; mutex_lock(&file->file_mutex); } uevent = list_entry(file->events.next, struct ib_ucm_event, file_list); if (ib_ucm_new_cm_id(uevent->resp.event)) { ctx = ib_ucm_ctx_alloc(file); if (!ctx) { result = -ENOMEM; goto done; } ctx->cm_id = uevent->cm_id; ctx->cm_id->context = ctx; uevent->resp.id = ctx->id; } if (copy_to_user((void __user *)(unsigned long)cmd.response, &uevent->resp, sizeof(uevent->resp))) { result = -EFAULT; goto done; } if (uevent->data) { if (cmd.data_len < uevent->data_len) { result = -ENOMEM; goto done; } if (copy_to_user((void __user *)(unsigned long)cmd.data, uevent->data, uevent->data_len)) { result = -EFAULT; goto done; } } if (uevent->info) { if (cmd.info_len < uevent->info_len) { result = -ENOMEM; goto done; } if (copy_to_user((void __user *)(unsigned long)cmd.info, uevent->info, uevent->info_len)) { result = -EFAULT; goto done; } } list_del(&uevent->file_list); list_del(&uevent->ctx_list); uevent->ctx->events_reported++; kfree(uevent->data); kfree(uevent->info); kfree(uevent); done: mutex_unlock(&file->file_mutex); return result; } static ssize_t ib_ucm_create_id(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { struct ib_ucm_create_id cmd; struct ib_ucm_create_id_resp resp; struct ib_ucm_context *ctx; int result; if (out_len < sizeof(resp)) return -ENOSPC; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; mutex_lock(&file->file_mutex); ctx = ib_ucm_ctx_alloc(file); mutex_unlock(&file->file_mutex); if (!ctx) return -ENOMEM; ctx->uid = cmd.uid; ctx->cm_id = ib_create_cm_id(file->device->ib_dev, ib_ucm_event_handler, ctx); if (IS_ERR(ctx->cm_id)) { result = PTR_ERR(ctx->cm_id); goto err1; } resp.id = ctx->id; if (copy_to_user((void __user *)(unsigned long)cmd.response, &resp, sizeof(resp))) { result = -EFAULT; goto err2; } return 0; err2: ib_destroy_cm_id(ctx->cm_id); err1: mutex_lock(&ctx_id_mutex); idr_remove(&ctx_id_table, ctx->id); mutex_unlock(&ctx_id_mutex); kfree(ctx); return result; } static ssize_t ib_ucm_destroy_id(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { struct ib_ucm_destroy_id cmd; struct ib_ucm_destroy_id_resp resp; struct ib_ucm_context *ctx; int result = 0; if (out_len < sizeof(resp)) return -ENOSPC; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; mutex_lock(&ctx_id_mutex); ctx = idr_find(&ctx_id_table, cmd.id); if (!ctx) ctx = ERR_PTR(-ENOENT); else if (ctx->file != file) ctx = ERR_PTR(-EINVAL); else idr_remove(&ctx_id_table, ctx->id); mutex_unlock(&ctx_id_mutex); if (IS_ERR(ctx)) return PTR_ERR(ctx); ib_ucm_ctx_put(ctx); wait_for_completion(&ctx->comp); /* No new events will be generated after destroying the cm_id. */ ib_destroy_cm_id(ctx->cm_id); /* Cleanup events not yet reported to the user. */ ib_ucm_cleanup_events(ctx); resp.events_reported = ctx->events_reported; if (copy_to_user((void __user *)(unsigned long)cmd.response, &resp, sizeof(resp))) result = -EFAULT; kfree(ctx); return result; } static ssize_t ib_ucm_attr_id(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { struct ib_ucm_attr_id_resp resp; struct ib_ucm_attr_id cmd; struct ib_ucm_context *ctx; int result = 0; if (out_len < sizeof(resp)) return -ENOSPC; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; ctx = ib_ucm_ctx_get(file, cmd.id); if (IS_ERR(ctx)) return PTR_ERR(ctx); resp.service_id = ctx->cm_id->service_id; resp.service_mask = ctx->cm_id->service_mask; resp.local_id = ctx->cm_id->local_id; resp.remote_id = ctx->cm_id->remote_id; if (copy_to_user((void __user *)(unsigned long)cmd.response, &resp, sizeof(resp))) result = -EFAULT; ib_ucm_ctx_put(ctx); return result; } static ssize_t ib_ucm_init_qp_attr(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { struct ib_uverbs_qp_attr resp; struct ib_ucm_init_qp_attr cmd; struct ib_ucm_context *ctx; struct ib_qp_attr qp_attr; int result = 0; if (out_len < sizeof(resp)) return -ENOSPC; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; ctx = ib_ucm_ctx_get(file, cmd.id); if (IS_ERR(ctx)) return PTR_ERR(ctx); resp.qp_attr_mask = 0; memset(&qp_attr, 0, sizeof qp_attr); qp_attr.qp_state = cmd.qp_state; result = ib_cm_init_qp_attr(ctx->cm_id, &qp_attr, &resp.qp_attr_mask); if (result) goto out; ib_copy_qp_attr_to_user(&resp, &qp_attr); if (copy_to_user((void __user *)(unsigned long)cmd.response, &resp, sizeof(resp))) result = -EFAULT; out: ib_ucm_ctx_put(ctx); return result; } static int ucm_validate_listen(__be64 service_id, __be64 service_mask) { service_id &= service_mask; if (((service_id & IB_CMA_SERVICE_ID_MASK) == IB_CMA_SERVICE_ID) || ((service_id & IB_SDP_SERVICE_ID_MASK) == IB_SDP_SERVICE_ID)) return -EINVAL; return 0; } static ssize_t ib_ucm_listen(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { struct ib_ucm_listen cmd; struct ib_ucm_context *ctx; int result; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; ctx = ib_ucm_ctx_get(file, cmd.id); if (IS_ERR(ctx)) return PTR_ERR(ctx); result = ucm_validate_listen(cmd.service_id, cmd.service_mask); if (result) goto out; result = ib_cm_listen(ctx->cm_id, cmd.service_id, cmd.service_mask, NULL); out: ib_ucm_ctx_put(ctx); return result; } static ssize_t ib_ucm_notify(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { struct ib_ucm_notify cmd; struct ib_ucm_context *ctx; int result; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; ctx = ib_ucm_ctx_get(file, cmd.id); if (IS_ERR(ctx)) return PTR_ERR(ctx); result = ib_cm_notify(ctx->cm_id, (enum ib_event_type) cmd.event); ib_ucm_ctx_put(ctx); return result; } static int ib_ucm_alloc_data(const void **dest, u64 src, u32 len) { void *data; *dest = NULL; if (!len) return 0; data = kmalloc(len, GFP_KERNEL); if (!data) return -ENOMEM; if (copy_from_user(data, (void __user *)(unsigned long)src, len)) { kfree(data); return -EFAULT; } *dest = data; return 0; } static int ib_ucm_path_get(struct ib_sa_path_rec **path, u64 src) { struct ib_user_path_rec upath; struct ib_sa_path_rec *sa_path; *path = NULL; if (!src) return 0; sa_path = kmalloc(sizeof(*sa_path), GFP_KERNEL); if (!sa_path) return -ENOMEM; if (copy_from_user(&upath, (void __user *)(unsigned long)src, sizeof(upath))) { kfree(sa_path); return -EFAULT; } ib_copy_path_rec_from_user(sa_path, &upath); *path = sa_path; return 0; } static ssize_t ib_ucm_send_req(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { struct ib_cm_req_param param; struct ib_ucm_context *ctx; struct ib_ucm_req cmd; int result; param.private_data = NULL; param.primary_path = NULL; param.alternate_path = NULL; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; result = ib_ucm_alloc_data(¶m.private_data, cmd.data, cmd.len); if (result) goto done; result = ib_ucm_path_get(¶m.primary_path, cmd.primary_path); if (result) goto done; result = ib_ucm_path_get(¶m.alternate_path, cmd.alternate_path); if (result) goto done; param.private_data_len = cmd.len; param.service_id = cmd.sid; param.qp_num = cmd.qpn; param.qp_type = cmd.qp_type; param.starting_psn = cmd.psn; param.peer_to_peer = cmd.peer_to_peer; param.responder_resources = cmd.responder_resources; param.initiator_depth = cmd.initiator_depth; param.remote_cm_response_timeout = cmd.remote_cm_response_timeout; param.flow_control = cmd.flow_control; param.local_cm_response_timeout = cmd.local_cm_response_timeout; param.retry_count = cmd.retry_count; param.rnr_retry_count = cmd.rnr_retry_count; param.max_cm_retries = cmd.max_cm_retries; param.srq = cmd.srq; ctx = ib_ucm_ctx_get(file, cmd.id); if (!IS_ERR(ctx)) { result = ib_send_cm_req(ctx->cm_id, ¶m); ib_ucm_ctx_put(ctx); } else result = PTR_ERR(ctx); done: kfree(param.private_data); kfree(param.primary_path); kfree(param.alternate_path); return result; } static ssize_t ib_ucm_send_rep(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { struct ib_cm_rep_param param; struct ib_ucm_context *ctx; struct ib_ucm_rep cmd; int result; param.private_data = NULL; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; result = ib_ucm_alloc_data(¶m.private_data, cmd.data, cmd.len); if (result) return result; param.qp_num = cmd.qpn; param.starting_psn = cmd.psn; param.private_data_len = cmd.len; param.responder_resources = cmd.responder_resources; param.initiator_depth = cmd.initiator_depth; param.failover_accepted = cmd.failover_accepted; param.flow_control = cmd.flow_control; param.rnr_retry_count = cmd.rnr_retry_count; param.srq = cmd.srq; ctx = ib_ucm_ctx_get(file, cmd.id); if (!IS_ERR(ctx)) { ctx->uid = cmd.uid; result = ib_send_cm_rep(ctx->cm_id, ¶m); ib_ucm_ctx_put(ctx); } else result = PTR_ERR(ctx); kfree(param.private_data); return result; } static ssize_t ib_ucm_send_private_data(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int (*func)(struct ib_cm_id *cm_id, const void *private_data, u8 private_data_len)) { struct ib_ucm_private_data cmd; struct ib_ucm_context *ctx; const void *private_data = NULL; int result; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; result = ib_ucm_alloc_data(&private_data, cmd.data, cmd.len); if (result) return result; ctx = ib_ucm_ctx_get(file, cmd.id); if (!IS_ERR(ctx)) { result = func(ctx->cm_id, private_data, cmd.len); ib_ucm_ctx_put(ctx); } else result = PTR_ERR(ctx); kfree(private_data); return result; } static ssize_t ib_ucm_send_rtu(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { return ib_ucm_send_private_data(file, inbuf, in_len, ib_send_cm_rtu); } static ssize_t ib_ucm_send_dreq(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { return ib_ucm_send_private_data(file, inbuf, in_len, ib_send_cm_dreq); } static ssize_t ib_ucm_send_drep(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { return ib_ucm_send_private_data(file, inbuf, in_len, ib_send_cm_drep); } static ssize_t ib_ucm_send_info(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int (*func)(struct ib_cm_id *cm_id, int status, const void *info, u8 info_len, const void *data, u8 data_len)) { struct ib_ucm_context *ctx; struct ib_ucm_info cmd; const void *data = NULL; const void *info = NULL; int result; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; result = ib_ucm_alloc_data(&data, cmd.data, cmd.data_len); if (result) goto done; result = ib_ucm_alloc_data(&info, cmd.info, cmd.info_len); if (result) goto done; ctx = ib_ucm_ctx_get(file, cmd.id); if (!IS_ERR(ctx)) { result = func(ctx->cm_id, cmd.status, info, cmd.info_len, data, cmd.data_len); ib_ucm_ctx_put(ctx); } else result = PTR_ERR(ctx); done: kfree(data); kfree(info); return result; } static ssize_t ib_ucm_send_rej(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { return ib_ucm_send_info(file, inbuf, in_len, (void *)ib_send_cm_rej); } static ssize_t ib_ucm_send_apr(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { return ib_ucm_send_info(file, inbuf, in_len, (void *)ib_send_cm_apr); } static ssize_t ib_ucm_send_mra(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { struct ib_ucm_context *ctx; struct ib_ucm_mra cmd; const void *data = NULL; int result; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; result = ib_ucm_alloc_data(&data, cmd.data, cmd.len); if (result) return result; ctx = ib_ucm_ctx_get(file, cmd.id); if (!IS_ERR(ctx)) { result = ib_send_cm_mra(ctx->cm_id, cmd.timeout, data, cmd.len); ib_ucm_ctx_put(ctx); } else result = PTR_ERR(ctx); kfree(data); return result; } static ssize_t ib_ucm_send_lap(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { struct ib_ucm_context *ctx; struct ib_sa_path_rec *path = NULL; struct ib_ucm_lap cmd; const void *data = NULL; int result; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; result = ib_ucm_alloc_data(&data, cmd.data, cmd.len); if (result) goto done; result = ib_ucm_path_get(&path, cmd.path); if (result) goto done; ctx = ib_ucm_ctx_get(file, cmd.id); if (!IS_ERR(ctx)) { result = ib_send_cm_lap(ctx->cm_id, path, data, cmd.len); ib_ucm_ctx_put(ctx); } else result = PTR_ERR(ctx); done: kfree(data); kfree(path); return result; } static ssize_t ib_ucm_send_sidr_req(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { struct ib_cm_sidr_req_param param; struct ib_ucm_context *ctx; struct ib_ucm_sidr_req cmd; int result; param.private_data = NULL; param.path = NULL; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; result = ib_ucm_alloc_data(¶m.private_data, cmd.data, cmd.len); if (result) goto done; result = ib_ucm_path_get(¶m.path, cmd.path); if (result) goto done; param.private_data_len = cmd.len; param.service_id = cmd.sid; param.timeout_ms = cmd.timeout; param.max_cm_retries = cmd.max_cm_retries; ctx = ib_ucm_ctx_get(file, cmd.id); if (!IS_ERR(ctx)) { result = ib_send_cm_sidr_req(ctx->cm_id, ¶m); ib_ucm_ctx_put(ctx); } else result = PTR_ERR(ctx); done: kfree(param.private_data); kfree(param.path); return result; } static ssize_t ib_ucm_send_sidr_rep(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) { struct ib_cm_sidr_rep_param param; struct ib_ucm_sidr_rep cmd; struct ib_ucm_context *ctx; int result; param.info = NULL; if (copy_from_user(&cmd, inbuf, sizeof(cmd))) return -EFAULT; result = ib_ucm_alloc_data(¶m.private_data, cmd.data, cmd.data_len); if (result) goto done; result = ib_ucm_alloc_data(¶m.info, cmd.info, cmd.info_len); if (result) goto done; param.qp_num = cmd.qpn; param.qkey = cmd.qkey; param.status = cmd.status; param.info_length = cmd.info_len; param.private_data_len = cmd.data_len; ctx = ib_ucm_ctx_get(file, cmd.id); if (!IS_ERR(ctx)) { result = ib_send_cm_sidr_rep(ctx->cm_id, ¶m); ib_ucm_ctx_put(ctx); } else result = PTR_ERR(ctx); done: kfree(param.private_data); kfree(param.info); return result; } static ssize_t (*ucm_cmd_table[])(struct ib_ucm_file *file, const char __user *inbuf, int in_len, int out_len) = { [IB_USER_CM_CMD_CREATE_ID] = ib_ucm_create_id, [IB_USER_CM_CMD_DESTROY_ID] = ib_ucm_destroy_id, [IB_USER_CM_CMD_ATTR_ID] = ib_ucm_attr_id, [IB_USER_CM_CMD_LISTEN] = ib_ucm_listen, [IB_USER_CM_CMD_NOTIFY] = ib_ucm_notify, [IB_USER_CM_CMD_SEND_REQ] = ib_ucm_send_req, [IB_USER_CM_CMD_SEND_REP] = ib_ucm_send_rep, [IB_USER_CM_CMD_SEND_RTU] = ib_ucm_send_rtu, [IB_USER_CM_CMD_SEND_DREQ] = ib_ucm_send_dreq, [IB_USER_CM_CMD_SEND_DREP] = ib_ucm_send_drep, [IB_USER_CM_CMD_SEND_REJ] = ib_ucm_send_rej, [IB_USER_CM_CMD_SEND_MRA] = ib_ucm_send_mra, [IB_USER_CM_CMD_SEND_LAP] = ib_ucm_send_lap, [IB_USER_CM_CMD_SEND_APR] = ib_ucm_send_apr, [IB_USER_CM_CMD_SEND_SIDR_REQ] = ib_ucm_send_sidr_req, [IB_USER_CM_CMD_SEND_SIDR_REP] = ib_ucm_send_sidr_rep, [IB_USER_CM_CMD_EVENT] = ib_ucm_event, [IB_USER_CM_CMD_INIT_QP_ATTR] = ib_ucm_init_qp_attr, }; static ssize_t ib_ucm_write(struct file *filp, const char __user *buf, size_t len, loff_t *pos) { struct ib_ucm_file *file = filp->private_data; struct ib_ucm_cmd_hdr hdr; ssize_t result; if (len < sizeof(hdr)) return -EINVAL; if (copy_from_user(&hdr, buf, sizeof(hdr))) return -EFAULT; if (hdr.cmd < 0 || hdr.cmd >= ARRAY_SIZE(ucm_cmd_table)) return -EINVAL; if (hdr.in + sizeof(hdr) > len) return -EINVAL; result = ucm_cmd_table[hdr.cmd](file, buf + sizeof(hdr), hdr.in, hdr.out); if (!result) result = len; return result; } static unsigned int ib_ucm_poll(struct file *filp, struct poll_table_struct *wait) { struct ib_ucm_file *file = filp->private_data; unsigned int mask = 0; poll_wait(filp, &file->poll_wait, wait); if (!list_empty(&file->events)) mask = POLLIN | POLLRDNORM; return mask; } /* * ib_ucm_open() does not need the BKL: * * - no global state is referred to; * - there is no ioctl method to race against; * - no further module initialization is required for open to work * after the device is registered. */ static int ib_ucm_open(struct inode *inode, struct file *filp) { struct ib_ucm_file *file; file = kzalloc(sizeof(*file), GFP_KERNEL); if (!file) return -ENOMEM; INIT_LIST_HEAD(&file->events); INIT_LIST_HEAD(&file->ctxs); init_waitqueue_head(&file->poll_wait); mutex_init(&file->file_mutex); filp->private_data = file; file->filp = filp; file->device = container_of(inode->i_cdev->si_drv1, struct ib_ucm_device, cdev); return 0; } static int ib_ucm_close(struct inode *inode, struct file *filp) { struct ib_ucm_file *file = filp->private_data; struct ib_ucm_context *ctx; mutex_lock(&file->file_mutex); while (!list_empty(&file->ctxs)) { ctx = list_entry(file->ctxs.next, struct ib_ucm_context, file_list); mutex_unlock(&file->file_mutex); mutex_lock(&ctx_id_mutex); idr_remove(&ctx_id_table, ctx->id); mutex_unlock(&ctx_id_mutex); ib_destroy_cm_id(ctx->cm_id); ib_ucm_cleanup_events(ctx); kfree(ctx); mutex_lock(&file->file_mutex); } mutex_unlock(&file->file_mutex); kfree(file); return 0; } static void ib_ucm_release_dev(struct device *dev) { struct ib_ucm_device *ucm_dev; ucm_dev = container_of(dev, struct ib_ucm_device, dev); cdev_del(&ucm_dev->cdev); clear_bit(ucm_dev->devnum, dev_map); kfree(ucm_dev); } static const struct file_operations ucm_fops = { .owner = THIS_MODULE, .open = ib_ucm_open, .release = ib_ucm_close, .write = ib_ucm_write, .poll = ib_ucm_poll, }; static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr, char *buf) { struct ib_ucm_device *ucm_dev; ucm_dev = container_of(dev, struct ib_ucm_device, dev); return sprintf(buf, "%s\n", ucm_dev->ib_dev->name); } static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL); static void ib_ucm_add_one(struct ib_device *device) { struct ib_ucm_device *ucm_dev; if (!device->alloc_ucontext || rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB) return; ucm_dev = kzalloc(sizeof *ucm_dev, GFP_KERNEL); if (!ucm_dev) return; ucm_dev->ib_dev = device; ucm_dev->devnum = find_first_zero_bit(dev_map, IB_UCM_MAX_DEVICES); if (ucm_dev->devnum >= IB_UCM_MAX_DEVICES) goto err; set_bit(ucm_dev->devnum, dev_map); cdev_init(&ucm_dev->cdev, &ucm_fops); ucm_dev->cdev.owner = THIS_MODULE; kobject_set_name(&ucm_dev->cdev.kobj, "ucm%d", ucm_dev->devnum); if (cdev_add(&ucm_dev->cdev, IB_UCM_BASE_DEV + ucm_dev->devnum, 1)) goto err; ucm_dev->dev.class = &cm_class; ucm_dev->dev.parent = device->dma_device; ucm_dev->dev.devt = ucm_dev->cdev.dev; ucm_dev->dev.release = ib_ucm_release_dev; dev_set_name(&ucm_dev->dev, "ucm%d", ucm_dev->devnum); if (device_register(&ucm_dev->dev)) goto err_cdev; if (device_create_file(&ucm_dev->dev, &dev_attr_ibdev)) goto err_dev; ib_set_client_data(device, &ucm_client, ucm_dev); return; err_dev: device_unregister(&ucm_dev->dev); err_cdev: cdev_del(&ucm_dev->cdev); clear_bit(ucm_dev->devnum, dev_map); err: kfree(ucm_dev); return; } static void ib_ucm_remove_one(struct ib_device *device) { struct ib_ucm_device *ucm_dev = ib_get_client_data(device, &ucm_client); if (!ucm_dev) return; device_unregister(&ucm_dev->dev); } -static ssize_t show_abi_version(struct class *class, char *buf) +static ssize_t show_abi_version(struct class *class, struct class_attribute *attr, char *buf) { return sprintf(buf, "%d\n", IB_USER_CM_ABI_VERSION); } static CLASS_ATTR(abi_version, S_IRUGO, show_abi_version, NULL); static int __init ib_ucm_init(void) { int ret; ret = register_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_MAX_DEVICES, "infiniband_cm"); if (ret) { printk(KERN_ERR "ucm: couldn't register device number\n"); goto error1; } ret = class_create_file(&cm_class, &class_attr_abi_version); if (ret) { printk(KERN_ERR "ucm: couldn't create abi_version attribute\n"); goto error2; } ret = ib_register_client(&ucm_client); if (ret) { printk(KERN_ERR "ucm: couldn't register client\n"); goto error3; } return 0; error3: class_remove_file(&cm_class, &class_attr_abi_version); error2: unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_MAX_DEVICES); error1: return ret; } static void __exit ib_ucm_cleanup(void) { ib_unregister_client(&ucm_client); class_remove_file(&cm_class, &class_attr_abi_version); unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_MAX_DEVICES); idr_destroy(&ctx_id_table); } module_init_order(ib_ucm_init, SI_ORDER_THIRD); module_exit(ib_ucm_cleanup); Index: stable/9/sys/ofed/drivers/infiniband/core/umem.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/umem.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/umem.c (revision 273864) @@ -1,532 +1,575 @@ /* * Copyright (c) 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Cisco Systems. All rights reserved. * Copyright (c) 2005 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #ifdef __linux__ #include #endif #include #include #include #include #include #include #include #include #include "uverbs.h" static int allow_weak_ordering; module_param(allow_weak_ordering, bool, 0444); MODULE_PARM_DESC(allow_weak_ordering, "Allow weak ordering for data registered memory"); #define IB_UMEM_MAX_PAGE_CHUNK \ ((PAGE_SIZE - offsetof(struct ib_umem_chunk, page_list)) / \ ((void *) &((struct ib_umem_chunk *) 0)->page_list[1] - \ (void *) &((struct ib_umem_chunk *) 0)->page_list[0])) #ifdef __ia64__ extern int dma_map_sg_hp_wa; static int dma_map_sg_ia64(struct ib_device *ibdev, struct scatterlist *sg, int nents, enum dma_data_direction dir) { int i, rc, j, lents = 0; struct device *dev; if (!dma_map_sg_hp_wa) return ib_dma_map_sg(ibdev, sg, nents, dir); dev = ibdev->dma_device; for (i = 0; i < nents; ++i) { rc = dma_map_sg(dev, sg + i, 1, dir); if (rc <= 0) { for (j = 0; j < i; ++j) dma_unmap_sg(dev, sg + j, 1, dir); return 0; } lents += rc; } return lents; } static void dma_unmap_sg_ia64(struct ib_device *ibdev, struct scatterlist *sg, int nents, enum dma_data_direction dir) { int i; struct device *dev; if (!dma_map_sg_hp_wa) return ib_dma_unmap_sg(ibdev, sg, nents, dir); dev = ibdev->dma_device; for (i = 0; i < nents; ++i) dma_unmap_sg(dev, sg + i, 1, dir); } #define ib_dma_map_sg(dev, sg, nents, dir) dma_map_sg_ia64(dev, sg, nents, dir) #define ib_dma_unmap_sg(dev, sg, nents, dir) dma_unmap_sg_ia64(dev, sg, nents, dir) #endif static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty) { #ifdef __linux__ struct ib_umem_chunk *chunk, *tmp; int i; list_for_each_entry_safe(chunk, tmp, &umem->chunk_list, list) { ib_dma_unmap_sg_attrs(dev, chunk->page_list, chunk->nents, DMA_BIDIRECTIONAL, &chunk->attrs); for (i = 0; i < chunk->nents; ++i) { struct page *page = sg_page(&chunk->page_list[i]); if (umem->writable && dirty) set_page_dirty_lock(page); put_page(page); } kfree(chunk); } #else struct ib_umem_chunk *chunk, *tmp; vm_object_t object; int i; object = NULL; list_for_each_entry_safe(chunk, tmp, &umem->chunk_list, list) { ib_dma_unmap_sg_attrs(dev, chunk->page_list, chunk->nents, DMA_BIDIRECTIONAL, &chunk->attrs); for (i = 0; i < chunk->nents; ++i) { struct page *page = sg_page(&chunk->page_list[i]); if (umem->writable && dirty) { if (object && object != page->object) VM_OBJECT_UNLOCK(object); if (object != page->object) { object = page->object; VM_OBJECT_LOCK(object); } vm_page_dirty(page); } } kfree(chunk); } if (object) VM_OBJECT_UNLOCK(object); #endif } /** * ib_umem_get - Pin and DMA map userspace memory. * @context: userspace context to pin memory for * @addr: userspace virtual address to start at * @size: length of region to pin * @access: IB_ACCESS_xxx flags for memory being pinned * @dmasync: flush in-flight DMA when the memory region is written */ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, size_t size, int access, int dmasync) { #ifdef __linux__ struct ib_umem *umem; struct page **page_list; struct vm_area_struct **vma_list; struct ib_umem_chunk *chunk; unsigned long locked; unsigned long lock_limit; unsigned long cur_base; unsigned long npages; int ret; int off; int i; DEFINE_DMA_ATTRS(attrs); if (dmasync) dma_set_attr(DMA_ATTR_WRITE_BARRIER, &attrs); else if (allow_weak_ordering) dma_set_attr(DMA_ATTR_WEAK_ORDERING, &attrs); if (!can_do_mlock()) return ERR_PTR(-EPERM); umem = kmalloc(sizeof *umem, GFP_KERNEL); if (!umem) return ERR_PTR(-ENOMEM); umem->context = context; umem->length = size; umem->offset = addr & ~PAGE_MASK; umem->page_size = PAGE_SIZE; /* * We ask for writable memory if any access flags other than * "remote read" are set. "Local write" and "remote write" * obviously require write access. "Remote atomic" can do * things like fetch and add, which will modify memory, and * "MW bind" can change permissions by binding a window. */ umem->writable = !!(access & ~IB_ACCESS_REMOTE_READ); /* We assume the memory is from hugetlb until proved otherwise */ umem->hugetlb = 1; INIT_LIST_HEAD(&umem->chunk_list); page_list = (struct page **) __get_free_page(GFP_KERNEL); if (!page_list) { kfree(umem); return ERR_PTR(-ENOMEM); } /* * if we can't alloc the vma_list, it's not so bad; * just assume the memory is not hugetlb memory */ vma_list = (struct vm_area_struct **) __get_free_page(GFP_KERNEL); if (!vma_list) umem->hugetlb = 0; npages = PAGE_ALIGN(size + umem->offset) >> PAGE_SHIFT; down_write(¤t->mm->mmap_sem); locked = npages + current->mm->locked_vm; lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT; if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) { ret = -ENOMEM; goto out; } cur_base = addr & PAGE_MASK; ret = 0; while (npages) { ret = get_user_pages(current, current->mm, cur_base, min_t(unsigned long, npages, PAGE_SIZE / sizeof (struct page *)), 1, !umem->writable, page_list, vma_list); if (ret < 0) goto out; cur_base += ret * PAGE_SIZE; npages -= ret; off = 0; while (ret) { chunk = kmalloc(sizeof *chunk + sizeof (struct scatterlist) * min_t(int, ret, IB_UMEM_MAX_PAGE_CHUNK), GFP_KERNEL); if (!chunk) { ret = -ENOMEM; goto out; } chunk->attrs = attrs; chunk->nents = min_t(int, ret, IB_UMEM_MAX_PAGE_CHUNK); sg_init_table(chunk->page_list, chunk->nents); for (i = 0; i < chunk->nents; ++i) { if (vma_list && !is_vm_hugetlb_page(vma_list[i + off])) umem->hugetlb = 0; sg_set_page(&chunk->page_list[i], page_list[i + off], PAGE_SIZE, 0); } chunk->nmap = ib_dma_map_sg_attrs(context->device, &chunk->page_list[0], chunk->nents, DMA_BIDIRECTIONAL, &attrs); if (chunk->nmap <= 0) { for (i = 0; i < chunk->nents; ++i) put_page(sg_page(&chunk->page_list[i])); kfree(chunk); ret = -ENOMEM; goto out; } ret -= chunk->nents; off += chunk->nents; list_add_tail(&chunk->list, &umem->chunk_list); } ret = 0; } out: if (ret < 0) { __ib_umem_release(context->device, umem, 0); kfree(umem); } else current->mm->locked_vm = locked; up_write(¤t->mm->mmap_sem); if (vma_list) free_page((unsigned long) vma_list); free_page((unsigned long) page_list); return ret < 0 ? ERR_PTR(ret) : umem; #else struct ib_umem *umem; struct ib_umem_chunk *chunk; struct proc *proc; pmap_t pmap; vm_offset_t end, last, start; vm_size_t npages; int error; int ents; int ret; int i; DEFINE_DMA_ATTRS(attrs); error = priv_check(curthread, PRIV_VM_MLOCK); if (error) return ERR_PTR(-error); last = addr + size; start = addr & PAGE_MASK; /* Use the linux PAGE_MASK definition. */ end = roundup2(last, PAGE_SIZE); /* Use PAGE_MASK safe operation. */ if (last < addr || end < addr) return ERR_PTR(-EINVAL); npages = atop(end - start); if (npages > vm_page_max_wired) return ERR_PTR(-ENOMEM); umem = kzalloc(sizeof *umem, GFP_KERNEL); if (!umem) return ERR_PTR(-ENOMEM); proc = curthread->td_proc; PROC_LOCK(proc); if (ptoa(npages + pmap_wired_count(vm_map_pmap(&proc->p_vmspace->vm_map))) > lim_cur(proc, RLIMIT_MEMLOCK)) { PROC_UNLOCK(proc); kfree(umem); return ERR_PTR(-ENOMEM); } PROC_UNLOCK(proc); if (npages + cnt.v_wire_count > vm_page_max_wired) { kfree(umem); return ERR_PTR(-EAGAIN); } error = vm_map_wire(&proc->p_vmspace->vm_map, start, end, VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES | (umem->writable ? VM_MAP_WIRE_WRITE : 0)); if (error != KERN_SUCCESS) { kfree(umem); return ERR_PTR(-ENOMEM); } umem->context = context; umem->length = size; umem->offset = addr & ~PAGE_MASK; umem->page_size = PAGE_SIZE; umem->start = addr; /* * We ask for writable memory if any access flags other than * "remote read" are set. "Local write" and "remote write" * obviously require write access. "Remote atomic" can do * things like fetch and add, which will modify memory, and * "MW bind" can change permissions by binding a window. */ umem->writable = !!(access & ~IB_ACCESS_REMOTE_READ); umem->hugetlb = 0; INIT_LIST_HEAD(&umem->chunk_list); pmap = vm_map_pmap(&proc->p_vmspace->vm_map); ret = 0; while (npages) { ents = min_t(int, npages, IB_UMEM_MAX_PAGE_CHUNK); chunk = kmalloc(sizeof(*chunk) + (sizeof(struct scatterlist) * ents), GFP_KERNEL); if (!chunk) { ret = -ENOMEM; goto out; } chunk->attrs = attrs; chunk->nents = ents; sg_init_table(&chunk->page_list[0], ents); for (i = 0; i < chunk->nents; ++i) { vm_paddr_t pa; pa = pmap_extract(pmap, start); if (pa == 0) { ret = -ENOMEM; kfree(chunk); goto out; } sg_set_page(&chunk->page_list[i], PHYS_TO_VM_PAGE(pa), PAGE_SIZE, 0); npages--; start += PAGE_SIZE; } chunk->nmap = ib_dma_map_sg_attrs(context->device, &chunk->page_list[0], chunk->nents, DMA_BIDIRECTIONAL, &attrs); if (chunk->nmap != chunk->nents) { kfree(chunk); ret = -ENOMEM; goto out; } list_add_tail(&chunk->list, &umem->chunk_list); } out: if (ret < 0) { __ib_umem_release(context->device, umem, 0); kfree(umem); } return ret < 0 ? ERR_PTR(ret) : umem; #endif } EXPORT_SYMBOL(ib_umem_get); #ifdef __linux__ static void ib_umem_account(struct work_struct *work) { struct ib_umem *umem = container_of(work, struct ib_umem, work); down_write(&umem->mm->mmap_sem); umem->mm->locked_vm -= umem->diff; up_write(&umem->mm->mmap_sem); mmput(umem->mm); kfree(umem); } #endif /** * ib_umem_release - release memory pinned with ib_umem_get * @umem: umem struct to release */ void ib_umem_release(struct ib_umem *umem) { #ifdef __linux__ struct ib_ucontext *context = umem->context; struct mm_struct *mm; unsigned long diff; __ib_umem_release(umem->context->device, umem, 1); mm = get_task_mm(current); if (!mm) { kfree(umem); return; } diff = PAGE_ALIGN(umem->length + umem->offset) >> PAGE_SHIFT; /* * We may be called with the mm's mmap_sem already held. This * can happen when a userspace munmap() is the call that drops * the last reference to our file and calls our release * method. If there are memory regions to destroy, we'll end * up here and not be able to take the mmap_sem. In that case * we defer the vm_locked accounting to the system workqueue. */ if (context->closing) { if (!down_write_trylock(&mm->mmap_sem)) { INIT_WORK(&umem->work, ib_umem_account); umem->mm = mm; umem->diff = diff; schedule_work(&umem->work); return; } } else down_write(&mm->mmap_sem); current->mm->locked_vm -= diff; up_write(&mm->mmap_sem); mmput(mm); #else vm_offset_t addr, end, last, start; vm_size_t size; int error; __ib_umem_release(umem->context->device, umem, 1); if (umem->context->closing) { kfree(umem); return; } error = priv_check(curthread, PRIV_VM_MUNLOCK); if (error) return; addr = umem->start; size = umem->length; last = addr + size; start = addr & PAGE_MASK; /* Use the linux PAGE_MASK definition. */ end = roundup2(last, PAGE_SIZE); /* Use PAGE_MASK safe operation. */ vm_map_unwire(&curthread->td_proc->p_vmspace->vm_map, start, end, VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); #endif kfree(umem); } EXPORT_SYMBOL(ib_umem_release); int ib_umem_page_count(struct ib_umem *umem) { struct ib_umem_chunk *chunk; int shift; int i; int n; shift = ilog2(umem->page_size); n = 0; list_for_each_entry(chunk, &umem->chunk_list, list) for (i = 0; i < chunk->nmap; ++i) n += sg_dma_len(&chunk->page_list[i]) >> shift; return n; } EXPORT_SYMBOL(ib_umem_page_count); + +/**********************************************/ +/* + * Stub functions for contiguous pages - + * We currently do not support this feature + */ +/**********************************************/ + +/** + * ib_cmem_release_contiguous_pages - release memory allocated by + * ib_cmem_alloc_contiguous_pages. + * @cmem: cmem struct to release + */ +void ib_cmem_release_contiguous_pages(struct ib_cmem *cmem) +{ +} +EXPORT_SYMBOL(ib_cmem_release_contiguous_pages); + +/** + * * ib_cmem_alloc_contiguous_pages - allocate contiguous pages + * * @context: userspace context to allocate memory for + * * @total_size: total required size for that allocation. + * * @page_size_order: order of one contiguous page. + * */ +struct ib_cmem *ib_cmem_alloc_contiguous_pages(struct ib_ucontext *context, + unsigned long total_size, + unsigned long page_size_order) +{ + return NULL; +} +EXPORT_SYMBOL(ib_cmem_alloc_contiguous_pages); + +/** + * * ib_cmem_map_contiguous_pages_to_vma - map contiguous pages into VMA + * * @ib_cmem: cmem structure returned by ib_cmem_alloc_contiguous_pages + * * @vma: VMA to inject pages into. + * */ +int ib_cmem_map_contiguous_pages_to_vma(struct ib_cmem *ib_cmem, + struct vm_area_struct *vma) +{ + return 0; +} +EXPORT_SYMBOL(ib_cmem_map_contiguous_pages_to_vma); Index: stable/9/sys/ofed/drivers/infiniband/core/user_mad.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/user_mad.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/user_mad.c (revision 273864) @@ -1,1225 +1,1224 @@ /* * Copyright (c) 2004 Topspin Communications. All rights reserved. * Copyright (c) 2005 Voltaire, Inc. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2008 Cisco. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include -#include #include #include #include #include #include #include #include #include #include #include #include #include #include MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("InfiniBand userspace MAD packet access"); MODULE_LICENSE("Dual BSD/GPL"); enum { IB_UMAD_MAX_PORTS = 64, IB_UMAD_MAX_AGENTS = 32, IB_UMAD_MAJOR = 231, IB_UMAD_MINOR_BASE = 0 }; /* * Our lifetime rules for these structs are the following: each time a * device special file is opened, we look up the corresponding struct * ib_umad_port by minor in the umad_port[] table while holding the * port_lock. If this lookup succeeds, we take a reference on the * ib_umad_port's struct ib_umad_device while still holding the * port_lock; if the lookup fails, we fail the open(). We drop these * references in the corresponding close(). * * In addition to references coming from open character devices, there * is one more reference to each ib_umad_device representing the * module's reference taken when allocating the ib_umad_device in * ib_umad_add_one(). * * When destroying an ib_umad_device, we clear all of its * ib_umad_ports from umad_port[] while holding port_lock before * dropping the module's reference to the ib_umad_device. This is * always safe because any open() calls will either succeed and obtain * a reference before we clear the umad_port[] entries, or fail after * we clear the umad_port[] entries. */ struct ib_umad_port { struct cdev *cdev; struct device *dev; struct cdev *sm_cdev; struct device *sm_dev; struct semaphore sm_sem; struct mutex file_mutex; struct list_head file_list; struct ib_device *ib_dev; struct ib_umad_device *umad_dev; int dev_num; u8 port_num; }; struct ib_umad_device { int start_port, end_port; struct kref ref; struct ib_umad_port port[0]; }; struct ib_umad_file { struct mutex mutex; struct ib_umad_port *port; struct file *filp; struct list_head recv_list; struct list_head send_list; struct list_head port_list; spinlock_t send_lock; wait_queue_head_t recv_wait; struct ib_mad_agent *agent[IB_UMAD_MAX_AGENTS]; int agents_dead; u8 use_pkey_index; u8 already_used; }; struct ib_umad_packet { struct ib_mad_send_buf *msg; struct ib_mad_recv_wc *recv_wc; struct list_head list; int length; struct ib_user_mad mad; }; static struct class *umad_class; static const dev_t base_dev = MKDEV(IB_UMAD_MAJOR, IB_UMAD_MINOR_BASE); static DEFINE_SPINLOCK(port_lock); static struct ib_umad_port *umad_port[IB_UMAD_MAX_PORTS]; static DECLARE_BITMAP(dev_map, IB_UMAD_MAX_PORTS); static void ib_umad_add_one(struct ib_device *device); static void ib_umad_remove_one(struct ib_device *device); static void ib_umad_release_dev(struct kref *ref) { struct ib_umad_device *dev = container_of(ref, struct ib_umad_device, ref); kfree(dev); } static int hdr_size(struct ib_umad_file *file) { return file->use_pkey_index ? sizeof (struct ib_user_mad_hdr) : sizeof (struct ib_user_mad_hdr_old); } /* caller must hold file->mutex */ static struct ib_mad_agent *__get_agent(struct ib_umad_file *file, int id) { return file->agents_dead ? NULL : file->agent[id]; } static int queue_packet(struct ib_umad_file *file, struct ib_mad_agent *agent, struct ib_umad_packet *packet) { int ret = 1; mutex_lock(&file->mutex); for (packet->mad.hdr.id = 0; packet->mad.hdr.id < IB_UMAD_MAX_AGENTS; packet->mad.hdr.id++) if (agent == __get_agent(file, packet->mad.hdr.id)) { list_add_tail(&packet->list, &file->recv_list); selwakeup(&file->filp->f_selinfo); wake_up_interruptible(&file->recv_wait); ret = 0; break; } mutex_unlock(&file->mutex); return ret; } static void dequeue_send(struct ib_umad_file *file, struct ib_umad_packet *packet) { spin_lock_irq(&file->send_lock); list_del(&packet->list); spin_unlock_irq(&file->send_lock); } static void send_handler(struct ib_mad_agent *agent, struct ib_mad_send_wc *send_wc) { struct ib_umad_file *file = agent->context; struct ib_umad_packet *packet = send_wc->send_buf->context[0]; dequeue_send(file, packet); ib_destroy_ah(packet->msg->ah); ib_free_send_mad(packet->msg); if (send_wc->status == IB_WC_RESP_TIMEOUT_ERR) { packet->length = IB_MGMT_MAD_HDR; packet->mad.hdr.status = ETIMEDOUT; if (!queue_packet(file, agent, packet)) return; } kfree(packet); } static void recv_handler(struct ib_mad_agent *agent, struct ib_mad_recv_wc *mad_recv_wc) { struct ib_umad_file *file = agent->context; struct ib_umad_packet *packet; if (mad_recv_wc->wc->status != IB_WC_SUCCESS) goto err1; packet = kzalloc(sizeof *packet, GFP_KERNEL); if (!packet) goto err1; packet->length = mad_recv_wc->mad_len; packet->recv_wc = mad_recv_wc; packet->mad.hdr.status = 0; packet->mad.hdr.length = hdr_size(file) + mad_recv_wc->mad_len; packet->mad.hdr.qpn = cpu_to_be32(mad_recv_wc->wc->src_qp); packet->mad.hdr.lid = cpu_to_be16(mad_recv_wc->wc->slid); packet->mad.hdr.sl = mad_recv_wc->wc->sl; packet->mad.hdr.path_bits = mad_recv_wc->wc->dlid_path_bits; packet->mad.hdr.pkey_index = mad_recv_wc->wc->pkey_index; packet->mad.hdr.grh_present = !!(mad_recv_wc->wc->wc_flags & IB_WC_GRH); if (packet->mad.hdr.grh_present) { struct ib_ah_attr ah_attr; ib_init_ah_from_wc(agent->device, agent->port_num, mad_recv_wc->wc, mad_recv_wc->recv_buf.grh, &ah_attr); packet->mad.hdr.gid_index = ah_attr.grh.sgid_index; packet->mad.hdr.hop_limit = ah_attr.grh.hop_limit; packet->mad.hdr.traffic_class = ah_attr.grh.traffic_class; memcpy(packet->mad.hdr.gid, &ah_attr.grh.dgid, 16); packet->mad.hdr.flow_label = cpu_to_be32(ah_attr.grh.flow_label); } if (queue_packet(file, agent, packet)) goto err2; return; err2: kfree(packet); err1: ib_free_recv_mad(mad_recv_wc); } static ssize_t copy_recv_mad(struct ib_umad_file *file, char __user *buf, struct ib_umad_packet *packet, size_t count) { struct ib_mad_recv_buf *recv_buf; int left, seg_payload, offset, max_seg_payload; /* We need enough room to copy the first (or only) MAD segment. */ recv_buf = &packet->recv_wc->recv_buf; if ((packet->length <= sizeof (*recv_buf->mad) && count < hdr_size(file) + packet->length) || (packet->length > sizeof (*recv_buf->mad) && count < hdr_size(file) + sizeof (*recv_buf->mad))) return -EINVAL; if (copy_to_user(buf, &packet->mad, hdr_size(file))) return -EFAULT; buf += hdr_size(file); seg_payload = min_t(int, packet->length, sizeof (*recv_buf->mad)); if (copy_to_user(buf, recv_buf->mad, seg_payload)) return -EFAULT; if (seg_payload < packet->length) { /* * Multipacket RMPP MAD message. Copy remainder of message. * Note that last segment may have a shorter payload. */ if (count < hdr_size(file) + packet->length) { /* * The buffer is too small, return the first RMPP segment, * which includes the RMPP message length. */ return -ENOSPC; } offset = ib_get_mad_data_offset(recv_buf->mad->mad_hdr.mgmt_class); max_seg_payload = sizeof (struct ib_mad) - offset; for (left = packet->length - seg_payload, buf += seg_payload; left; left -= seg_payload, buf += seg_payload) { recv_buf = container_of(recv_buf->list.next, struct ib_mad_recv_buf, list); seg_payload = min(left, max_seg_payload); if (copy_to_user(buf, ((void *) recv_buf->mad) + offset, seg_payload)) return -EFAULT; } } return hdr_size(file) + packet->length; } static ssize_t copy_send_mad(struct ib_umad_file *file, char __user *buf, struct ib_umad_packet *packet, size_t count) { ssize_t size = hdr_size(file) + packet->length; if (count < size) return -EINVAL; if (copy_to_user(buf, &packet->mad, hdr_size(file))) return -EFAULT; buf += hdr_size(file); if (copy_to_user(buf, packet->mad.data, packet->length)) return -EFAULT; return size; } static ssize_t ib_umad_read(struct file *filp, char __user *buf, size_t count, loff_t *pos) { struct ib_umad_file *file = filp->private_data; struct ib_umad_packet *packet; ssize_t ret; if (count < hdr_size(file)) return -EINVAL; mutex_lock(&file->mutex); while (list_empty(&file->recv_list)) { mutex_unlock(&file->mutex); if (filp->f_flags & O_NONBLOCK) return -EAGAIN; if (wait_event_interruptible(file->recv_wait, !list_empty(&file->recv_list))) return -ERESTARTSYS; mutex_lock(&file->mutex); } packet = list_entry(file->recv_list.next, struct ib_umad_packet, list); list_del(&packet->list); mutex_unlock(&file->mutex); if (packet->recv_wc) ret = copy_recv_mad(file, buf, packet, count); else ret = copy_send_mad(file, buf, packet, count); if (ret < 0) { /* Requeue packet */ mutex_lock(&file->mutex); list_add(&packet->list, &file->recv_list); mutex_unlock(&file->mutex); } else { if (packet->recv_wc) ib_free_recv_mad(packet->recv_wc); kfree(packet); } return ret; } static int copy_rmpp_mad(struct ib_mad_send_buf *msg, const char __user *buf) { int left, seg; /* Copy class specific header */ if ((msg->hdr_len > IB_MGMT_RMPP_HDR) && copy_from_user(msg->mad + IB_MGMT_RMPP_HDR, buf + IB_MGMT_RMPP_HDR, msg->hdr_len - IB_MGMT_RMPP_HDR)) return -EFAULT; /* All headers are in place. Copy data segments. */ for (seg = 1, left = msg->data_len, buf += msg->hdr_len; left > 0; seg++, left -= msg->seg_size, buf += msg->seg_size) { if (copy_from_user(ib_get_rmpp_segment(msg, seg), buf, min(left, msg->seg_size))) return -EFAULT; } return 0; } static int same_destination(struct ib_user_mad_hdr *hdr1, struct ib_user_mad_hdr *hdr2) { if (!hdr1->grh_present && !hdr2->grh_present) return (hdr1->lid == hdr2->lid); if (hdr1->grh_present && hdr2->grh_present) return !memcmp(hdr1->gid, hdr2->gid, 16); return 0; } static int is_duplicate(struct ib_umad_file *file, struct ib_umad_packet *packet) { struct ib_umad_packet *sent_packet; struct ib_mad_hdr *sent_hdr, *hdr; hdr = (struct ib_mad_hdr *) packet->mad.data; list_for_each_entry(sent_packet, &file->send_list, list) { sent_hdr = (struct ib_mad_hdr *) sent_packet->mad.data; if ((hdr->tid != sent_hdr->tid) || (hdr->mgmt_class != sent_hdr->mgmt_class)) continue; /* * No need to be overly clever here. If two new operations have * the same TID, reject the second as a duplicate. This is more * restrictive than required by the spec. */ if (!ib_response_mad((struct ib_mad *) hdr)) { if (!ib_response_mad((struct ib_mad *) sent_hdr)) return 1; continue; } else if (!ib_response_mad((struct ib_mad *) sent_hdr)) continue; if (same_destination(&packet->mad.hdr, &sent_packet->mad.hdr)) return 1; } return 0; } static ssize_t ib_umad_write(struct file *filp, const char __user *buf, size_t count, loff_t *pos) { struct ib_umad_file *file = filp->private_data; struct ib_umad_packet *packet; struct ib_mad_agent *agent; struct ib_ah_attr ah_attr; struct ib_ah *ah; struct ib_rmpp_mad *rmpp_mad; __be64 *tid; int ret, data_len, hdr_len, copy_offset, rmpp_active; if (count < hdr_size(file) + IB_MGMT_RMPP_HDR) return -EINVAL; packet = kzalloc(sizeof *packet + IB_MGMT_RMPP_HDR, GFP_KERNEL); if (!packet) return -ENOMEM; if (copy_from_user(&packet->mad, buf, hdr_size(file))) { ret = -EFAULT; goto err; } if (packet->mad.hdr.id < 0 || packet->mad.hdr.id >= IB_UMAD_MAX_AGENTS) { ret = -EINVAL; goto err; } buf += hdr_size(file); if (copy_from_user(packet->mad.data, buf, IB_MGMT_RMPP_HDR)) { ret = -EFAULT; goto err; } mutex_lock(&file->mutex); agent = __get_agent(file, packet->mad.hdr.id); if (!agent) { ret = -EINVAL; goto err_up; } memset(&ah_attr, 0, sizeof ah_attr); ah_attr.dlid = be16_to_cpu(packet->mad.hdr.lid); ah_attr.sl = packet->mad.hdr.sl; ah_attr.src_path_bits = packet->mad.hdr.path_bits; ah_attr.port_num = file->port->port_num; if (packet->mad.hdr.grh_present) { ah_attr.ah_flags = IB_AH_GRH; memcpy(ah_attr.grh.dgid.raw, packet->mad.hdr.gid, 16); ah_attr.grh.sgid_index = packet->mad.hdr.gid_index; ah_attr.grh.flow_label = be32_to_cpu(packet->mad.hdr.flow_label); ah_attr.grh.hop_limit = packet->mad.hdr.hop_limit; ah_attr.grh.traffic_class = packet->mad.hdr.traffic_class; } ah = ib_create_ah(agent->qp->pd, &ah_attr); if (IS_ERR(ah)) { ret = PTR_ERR(ah); goto err_up; } rmpp_mad = (struct ib_rmpp_mad *) packet->mad.data; hdr_len = ib_get_mad_data_offset(rmpp_mad->mad_hdr.mgmt_class); if (!ib_is_mad_class_rmpp(rmpp_mad->mad_hdr.mgmt_class)) { copy_offset = IB_MGMT_MAD_HDR; rmpp_active = 0; } else { copy_offset = IB_MGMT_RMPP_HDR; rmpp_active = ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) & IB_MGMT_RMPP_FLAG_ACTIVE; } data_len = count - hdr_size(file) - hdr_len; packet->msg = ib_create_send_mad(agent, be32_to_cpu(packet->mad.hdr.qpn), packet->mad.hdr.pkey_index, rmpp_active, hdr_len, data_len, GFP_KERNEL); if (IS_ERR(packet->msg)) { ret = PTR_ERR(packet->msg); goto err_ah; } packet->msg->ah = ah; packet->msg->timeout_ms = packet->mad.hdr.timeout_ms; packet->msg->retries = packet->mad.hdr.retries; packet->msg->context[0] = packet; /* Copy MAD header. Any RMPP header is already in place. */ memcpy(packet->msg->mad, packet->mad.data, IB_MGMT_MAD_HDR); if (!rmpp_active) { if (copy_from_user(packet->msg->mad + copy_offset, buf + copy_offset, hdr_len + data_len - copy_offset)) { ret = -EFAULT; goto err_msg; } } else { ret = copy_rmpp_mad(packet->msg, buf); if (ret) goto err_msg; } /* * Set the high-order part of the transaction ID to make MADs from * different agents unique, and allow routing responses back to the * original requestor. */ if (!ib_response_mad(packet->msg->mad)) { tid = &((struct ib_mad_hdr *) packet->msg->mad)->tid; *tid = cpu_to_be64(((u64) agent->hi_tid) << 32 | (be64_to_cpup(tid) & 0xffffffff)); rmpp_mad->mad_hdr.tid = *tid; } spin_lock_irq(&file->send_lock); ret = is_duplicate(file, packet); if (!ret) list_add_tail(&packet->list, &file->send_list); spin_unlock_irq(&file->send_lock); if (ret) { ret = -EINVAL; goto err_msg; } ret = ib_post_send_mad(packet->msg, NULL); if (ret) goto err_send; mutex_unlock(&file->mutex); return count; err_send: dequeue_send(file, packet); err_msg: ib_free_send_mad(packet->msg); err_ah: ib_destroy_ah(ah); err_up: mutex_unlock(&file->mutex); err: kfree(packet); return ret; } static unsigned int ib_umad_poll(struct file *filp, struct poll_table_struct *wait) { struct ib_umad_file *file = filp->private_data; /* we will always be able to post a MAD send */ unsigned int mask = POLLOUT | POLLWRNORM; poll_wait(filp, &file->recv_wait, wait); if (!list_empty(&file->recv_list)) mask |= POLLIN | POLLRDNORM; return mask; } static int ib_umad_reg_agent(struct ib_umad_file *file, void __user *arg, int compat_method_mask) { struct ib_user_mad_reg_req ureq; struct ib_mad_reg_req req; struct ib_mad_agent *agent = NULL; int agent_id; int ret; mutex_lock(&file->port->file_mutex); mutex_lock(&file->mutex); if (!file->port->ib_dev) { ret = -EPIPE; goto out; } if (copy_from_user(&ureq, arg, sizeof ureq)) { ret = -EFAULT; goto out; } if (ureq.qpn != 0 && ureq.qpn != 1) { ret = -EINVAL; goto out; } for (agent_id = 0; agent_id < IB_UMAD_MAX_AGENTS; ++agent_id) if (!__get_agent(file, agent_id)) goto found; ret = -ENOMEM; goto out; found: if (ureq.mgmt_class) { req.mgmt_class = ureq.mgmt_class; req.mgmt_class_version = ureq.mgmt_class_version; memcpy(req.oui, ureq.oui, sizeof req.oui); if (compat_method_mask) { u32 *umm = (u32 *) ureq.method_mask; int i; for (i = 0; i < BITS_TO_LONGS(IB_MGMT_MAX_METHODS); ++i) req.method_mask[i] = umm[i * 2] | ((u64) umm[i * 2 + 1] << 32); } else memcpy(req.method_mask, ureq.method_mask, sizeof req.method_mask); } agent = ib_register_mad_agent(file->port->ib_dev, file->port->port_num, ureq.qpn ? IB_QPT_GSI : IB_QPT_SMI, ureq.mgmt_class ? &req : NULL, ureq.rmpp_version, send_handler, recv_handler, file); if (IS_ERR(agent)) { ret = PTR_ERR(agent); agent = NULL; goto out; } if (put_user(agent_id, (u32 __user *) (arg + offsetof(struct ib_user_mad_reg_req, id)))) { ret = -EFAULT; goto out; } if (!file->already_used) { file->already_used = 1; if (!file->use_pkey_index) { printk(KERN_WARNING "user_mad: process %s did not enable " "P_Key index support.\n", curproc->p_comm); printk(KERN_WARNING "user_mad: Documentation/infiniband/user_mad.txt " "has info on the new ABI.\n"); } } file->agent[agent_id] = agent; ret = 0; out: mutex_unlock(&file->mutex); if (ret && agent) ib_unregister_mad_agent(agent); mutex_unlock(&file->port->file_mutex); return ret; } static int ib_umad_unreg_agent(struct ib_umad_file *file, u32 __user *arg) { struct ib_mad_agent *agent = NULL; u32 id; int ret = 0; if (get_user(id, arg)) return -EFAULT; mutex_lock(&file->port->file_mutex); mutex_lock(&file->mutex); if (id < 0 || id >= IB_UMAD_MAX_AGENTS || !__get_agent(file, id)) { ret = -EINVAL; goto out; } agent = file->agent[id]; file->agent[id] = NULL; out: mutex_unlock(&file->mutex); if (agent) ib_unregister_mad_agent(agent); mutex_unlock(&file->port->file_mutex); return ret; } static long ib_umad_enable_pkey(struct ib_umad_file *file) { int ret = 0; mutex_lock(&file->mutex); if (file->already_used) ret = -EINVAL; else file->use_pkey_index = 1; mutex_unlock(&file->mutex); return ret; } static long ib_umad_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { switch (cmd) { case IB_USER_MAD_REGISTER_AGENT: return ib_umad_reg_agent(filp->private_data, (void __user *) arg, 0); case IB_USER_MAD_UNREGISTER_AGENT: return ib_umad_unreg_agent(filp->private_data, (__u32 __user *) arg); case IB_USER_MAD_ENABLE_PKEY: return ib_umad_enable_pkey(filp->private_data); default: return -ENOIOCTLCMD; } } #ifdef CONFIG_COMPAT static long ib_umad_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { switch (cmd) { case IB_USER_MAD_REGISTER_AGENT: return ib_umad_reg_agent(filp->private_data, compat_ptr(arg), 1); case IB_USER_MAD_UNREGISTER_AGENT: return ib_umad_unreg_agent(filp->private_data, compat_ptr(arg)); case IB_USER_MAD_ENABLE_PKEY: return ib_umad_enable_pkey(filp->private_data); default: return -ENOIOCTLCMD; } } #endif /* * ib_umad_open() does not need the BKL: * * - umad_port[] accesses are protected by port_lock, the * ib_umad_port structures are properly reference counted, and * everything else is purely local to the file being created, so * races against other open calls are not a problem; * - the ioctl method does not affect any global state outside of the * file structure being operated on; * - the port is added to umad_port[] as the last part of module * initialization so the open method will either immediately run * -ENXIO, or all required initialization will be done. */ static int ib_umad_open(struct inode *inode, struct file *filp) { struct ib_umad_port *port; struct ib_umad_file *file; int ret = 0; spin_lock(&port_lock); port = umad_port[iminor(inode) - IB_UMAD_MINOR_BASE]; if (port) kref_get(&port->umad_dev->ref); spin_unlock(&port_lock); if (!port) return -ENXIO; mutex_lock(&port->file_mutex); if (!port->ib_dev) { ret = -ENXIO; goto out; } file = kzalloc(sizeof *file, GFP_KERNEL); if (!file) { kref_put(&port->umad_dev->ref, ib_umad_release_dev); ret = -ENOMEM; goto out; } mutex_init(&file->mutex); spin_lock_init(&file->send_lock); INIT_LIST_HEAD(&file->recv_list); INIT_LIST_HEAD(&file->send_list); init_waitqueue_head(&file->recv_wait); file->port = port; file->filp = filp; filp->private_data = file; list_add_tail(&file->port_list, &port->file_list); out: mutex_unlock(&port->file_mutex); return ret; } static int ib_umad_close(struct inode *inode, struct file *filp) { struct ib_umad_file *file = filp->private_data; struct ib_umad_device *dev = file->port->umad_dev; struct ib_umad_packet *packet, *tmp; int already_dead; int i; mutex_lock(&file->port->file_mutex); mutex_lock(&file->mutex); already_dead = file->agents_dead; file->agents_dead = 1; list_for_each_entry_safe(packet, tmp, &file->recv_list, list) { if (packet->recv_wc) ib_free_recv_mad(packet->recv_wc); kfree(packet); } list_del(&file->port_list); mutex_unlock(&file->mutex); if (!already_dead) for (i = 0; i < IB_UMAD_MAX_AGENTS; ++i) if (file->agent[i]) ib_unregister_mad_agent(file->agent[i]); mutex_unlock(&file->port->file_mutex); kfree(file); kref_put(&dev->ref, ib_umad_release_dev); return 0; } static const struct file_operations umad_fops = { .owner = THIS_MODULE, .read = ib_umad_read, .write = ib_umad_write, .poll = ib_umad_poll, .unlocked_ioctl = ib_umad_ioctl, #ifdef CONFIG_COMPAT .compat_ioctl = ib_umad_compat_ioctl, #endif .open = ib_umad_open, .release = ib_umad_close }; static int ib_umad_sm_open(struct inode *inode, struct file *filp) { struct ib_umad_port *port; struct ib_port_modify props = { .set_port_cap_mask = IB_PORT_SM }; int ret; spin_lock(&port_lock); port = umad_port[iminor(inode) - IB_UMAD_MINOR_BASE - IB_UMAD_MAX_PORTS]; if (port) kref_get(&port->umad_dev->ref); spin_unlock(&port_lock); if (!port) return -ENXIO; if (filp->f_flags & O_NONBLOCK) { if (down_trylock(&port->sm_sem)) { ret = -EAGAIN; goto fail; } } else { if (down_interruptible(&port->sm_sem)) { ret = -ERESTARTSYS; goto fail; } } ret = ib_modify_port(port->ib_dev, port->port_num, 0, &props); if (ret) { up(&port->sm_sem); goto fail; } filp->private_data = port; return 0; fail: kref_put(&port->umad_dev->ref, ib_umad_release_dev); return ret; } static int ib_umad_sm_close(struct inode *inode, struct file *filp) { struct ib_umad_port *port = filp->private_data; struct ib_port_modify props = { .clr_port_cap_mask = IB_PORT_SM }; int ret = 0; mutex_lock(&port->file_mutex); if (port->ib_dev) ret = ib_modify_port(port->ib_dev, port->port_num, 0, &props); mutex_unlock(&port->file_mutex); up(&port->sm_sem); kref_put(&port->umad_dev->ref, ib_umad_release_dev); return ret; } static const struct file_operations umad_sm_fops = { .owner = THIS_MODULE, .open = ib_umad_sm_open, .release = ib_umad_sm_close }; static struct ib_client umad_client = { .name = "umad", .add = ib_umad_add_one, .remove = ib_umad_remove_one }; static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr, char *buf) { struct ib_umad_port *port = dev_get_drvdata(dev); if (!port) return -ENODEV; return sprintf(buf, "%s\n", port->ib_dev->name); } static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL); static ssize_t show_port(struct device *dev, struct device_attribute *attr, char *buf) { struct ib_umad_port *port = dev_get_drvdata(dev); if (!port) return -ENODEV; return sprintf(buf, "%d\n", port->port_num); } static DEVICE_ATTR(port, S_IRUGO, show_port, NULL); -static ssize_t show_abi_version(struct class *class, char *buf) +static ssize_t show_abi_version(struct class *class, struct class_attribute *attr, char *buf) { return sprintf(buf, "%d\n", IB_USER_MAD_ABI_VERSION); } static CLASS_ATTR(abi_version, S_IRUGO, show_abi_version, NULL); static int ib_umad_init_port(struct ib_device *device, int port_num, struct ib_umad_port *port) { spin_lock(&port_lock); port->dev_num = find_first_zero_bit(dev_map, IB_UMAD_MAX_PORTS); if (port->dev_num >= IB_UMAD_MAX_PORTS) { spin_unlock(&port_lock); return -1; } set_bit(port->dev_num, dev_map); spin_unlock(&port_lock); port->ib_dev = device; port->port_num = port_num; init_MUTEX(&port->sm_sem); mutex_init(&port->file_mutex); INIT_LIST_HEAD(&port->file_list); port->cdev = cdev_alloc(); if (!port->cdev) return -1; port->cdev->owner = THIS_MODULE; port->cdev->ops = &umad_fops; kobject_set_name(&port->cdev->kobj, "umad%d", port->dev_num); if (cdev_add(port->cdev, base_dev + port->dev_num, 1)) goto err_cdev; port->dev = device_create(umad_class, device->dma_device, port->cdev->dev, port, "umad%d", port->dev_num); if (IS_ERR(port->dev)) goto err_cdev; if (device_create_file(port->dev, &dev_attr_ibdev)) goto err_dev; if (device_create_file(port->dev, &dev_attr_port)) goto err_dev; port->sm_cdev = cdev_alloc(); if (!port->sm_cdev) goto err_dev; port->sm_cdev->owner = THIS_MODULE; port->sm_cdev->ops = &umad_sm_fops; kobject_set_name(&port->sm_cdev->kobj, "issm%d", port->dev_num); if (cdev_add(port->sm_cdev, base_dev + port->dev_num + IB_UMAD_MAX_PORTS, 1)) goto err_sm_cdev; port->sm_dev = device_create(umad_class, device->dma_device, port->sm_cdev->dev, port, "issm%d", port->dev_num); if (IS_ERR(port->sm_dev)) goto err_sm_cdev; if (device_create_file(port->sm_dev, &dev_attr_ibdev)) goto err_sm_dev; if (device_create_file(port->sm_dev, &dev_attr_port)) goto err_sm_dev; spin_lock(&port_lock); umad_port[port->dev_num] = port; spin_unlock(&port_lock); return 0; err_sm_dev: device_destroy(umad_class, port->sm_cdev->dev); err_sm_cdev: cdev_del(port->sm_cdev); err_dev: device_destroy(umad_class, port->cdev->dev); err_cdev: cdev_del(port->cdev); clear_bit(port->dev_num, dev_map); return -1; } static void ib_umad_kill_port(struct ib_umad_port *port) { struct ib_umad_file *file; int already_dead; int id; dev_set_drvdata(port->dev, NULL); dev_set_drvdata(port->sm_dev, NULL); device_destroy(umad_class, port->cdev->dev); device_destroy(umad_class, port->sm_cdev->dev); cdev_del(port->cdev); cdev_del(port->sm_cdev); spin_lock(&port_lock); umad_port[port->dev_num] = NULL; spin_unlock(&port_lock); mutex_lock(&port->file_mutex); port->ib_dev = NULL; list_for_each_entry(file, &port->file_list, port_list) { mutex_lock(&file->mutex); already_dead = file->agents_dead; file->agents_dead = 1; mutex_unlock(&file->mutex); for (id = 0; id < IB_UMAD_MAX_AGENTS; ++id) if (file->agent[id]) ib_unregister_mad_agent(file->agent[id]); } mutex_unlock(&port->file_mutex); clear_bit(port->dev_num, dev_map); } static void ib_umad_add_one(struct ib_device *device) { struct ib_umad_device *umad_dev; int s, e, i; if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB) return; if (device->node_type == RDMA_NODE_IB_SWITCH) s = e = 0; else { s = 1; e = device->phys_port_cnt; } umad_dev = kzalloc(sizeof *umad_dev + (e - s + 1) * sizeof (struct ib_umad_port), GFP_KERNEL); if (!umad_dev) return; kref_init(&umad_dev->ref); umad_dev->start_port = s; umad_dev->end_port = e; for (i = s; i <= e; ++i) { umad_dev->port[i - s].umad_dev = umad_dev; if (rdma_port_get_link_layer(device, i) == IB_LINK_LAYER_INFINIBAND) if (ib_umad_init_port(device, i, &umad_dev->port[i - s])) goto err; } ib_set_client_data(device, &umad_client, umad_dev); return; err: while (--i >= s) if (rdma_port_get_link_layer(device, i) == IB_LINK_LAYER_INFINIBAND) ib_umad_kill_port(&umad_dev->port[i - s]); kref_put(&umad_dev->ref, ib_umad_release_dev); } static void ib_umad_remove_one(struct ib_device *device) { struct ib_umad_device *umad_dev = ib_get_client_data(device, &umad_client); int i; if (!umad_dev) return; for (i = 0; i <= umad_dev->end_port - umad_dev->start_port; ++i) if (rdma_port_get_link_layer(device, i + 1) == IB_LINK_LAYER_INFINIBAND) ib_umad_kill_port(&umad_dev->port[i]); kref_put(&umad_dev->ref, ib_umad_release_dev); } static int __init ib_umad_init(void) { int ret; ret = register_chrdev_region(base_dev, IB_UMAD_MAX_PORTS * 2, "infiniband_mad"); if (ret) { printk(KERN_ERR "user_mad: couldn't register device number\n"); goto out; } umad_class = class_create(THIS_MODULE, "infiniband_mad"); if (IS_ERR(umad_class)) { ret = PTR_ERR(umad_class); printk(KERN_ERR "user_mad: couldn't create class infiniband_mad\n"); goto out_chrdev; } ret = class_create_file(umad_class, &class_attr_abi_version); if (ret) { printk(KERN_ERR "user_mad: couldn't create abi_version attribute\n"); goto out_class; } ret = ib_register_client(&umad_client); if (ret) { printk(KERN_ERR "user_mad: couldn't register ib_umad client\n"); goto out_class; } return 0; out_class: class_destroy(umad_class); out_chrdev: unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS * 2); out: return ret; } static void __exit ib_umad_cleanup(void) { ib_unregister_client(&umad_client); class_destroy(umad_class); unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS * 2); } module_init(ib_umad_init); module_exit(ib_umad_cleanup); Index: stable/9/sys/ofed/drivers/infiniband/core/uverbs_cmd.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/uverbs_cmd.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/uverbs_cmd.c (revision 273864) @@ -1,3022 +1,3023 @@ /* * Copyright (c) 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. * Copyright (c) 2005 PathScale, Inc. All rights reserved. * Copyright (c) 2006 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include +#include #include #include #include "uverbs.h" static struct lock_class_key pd_lock_key; static struct lock_class_key mr_lock_key; static struct lock_class_key cq_lock_key; static struct lock_class_key qp_lock_key; static struct lock_class_key ah_lock_key; static struct lock_class_key srq_lock_key; #define INIT_UDATA(udata, ibuf, obuf, ilen, olen) \ do { \ (udata)->inbuf = (void __user *) (ibuf); \ (udata)->outbuf = (void __user *) (obuf); \ (udata)->inlen = (ilen); \ (udata)->outlen = (olen); \ } while (0) /* * The ib_uobject locking scheme is as follows: * * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it * needs to be held during all idr operations. When an object is * looked up, a reference must be taken on the object's kref before * dropping this lock. * * - Each object also has an rwsem. This rwsem must be held for * reading while an operation that uses the object is performed. * For example, while registering an MR, the associated PD's * uobject.mutex must be held for reading. The rwsem must be held * for writing while initializing or destroying an object. * * - In addition, each object has a "live" flag. If this flag is not * set, then lookups of the object will fail even if it is found in * the idr. This handles a reader that blocks and does not acquire * the rwsem until after the object is destroyed. The destroy * operation will set the live flag to 0 and then drop the rwsem; * this will allow the reader to acquire the rwsem, see that the * live flag is 0, and then drop the rwsem and its reference to * object. The underlying storage will not be freed until the last * reference to the object is dropped. */ static void init_uobj(struct ib_uobject *uobj, u64 user_handle, struct ib_ucontext *context, struct lock_class_key *key) { uobj->user_handle = user_handle; uobj->context = context; kref_init(&uobj->ref); init_rwsem(&uobj->mutex); lockdep_set_class(&uobj->mutex, key); uobj->live = 0; } static void release_uobj(struct kref *kref) { kfree(container_of(kref, struct ib_uobject, ref)); } static void put_uobj(struct ib_uobject *uobj) { kref_put(&uobj->ref, release_uobj); } static void put_uobj_read(struct ib_uobject *uobj) { up_read(&uobj->mutex); put_uobj(uobj); } static void put_uobj_write(struct ib_uobject *uobj) { up_write(&uobj->mutex); put_uobj(uobj); } static int idr_add_uobj(struct idr *idr, struct ib_uobject *uobj) { int ret; retry: if (!idr_pre_get(idr, GFP_KERNEL)) return -ENOMEM; spin_lock(&ib_uverbs_idr_lock); ret = idr_get_new(idr, uobj, &uobj->id); spin_unlock(&ib_uverbs_idr_lock); if (ret == -EAGAIN) goto retry; return ret; } void idr_remove_uobj(struct idr *idr, struct ib_uobject *uobj) { spin_lock(&ib_uverbs_idr_lock); idr_remove(idr, uobj->id); spin_unlock(&ib_uverbs_idr_lock); } static struct ib_uobject *__idr_get_uobj(struct idr *idr, int id, struct ib_ucontext *context) { struct ib_uobject *uobj; spin_lock(&ib_uverbs_idr_lock); uobj = idr_find(idr, id); if (uobj) { if (uobj->context == context) kref_get(&uobj->ref); else uobj = NULL; } spin_unlock(&ib_uverbs_idr_lock); return uobj; } static struct ib_uobject *idr_read_uobj(struct idr *idr, int id, struct ib_ucontext *context, int nested) { struct ib_uobject *uobj; uobj = __idr_get_uobj(idr, id, context); if (!uobj) return NULL; if (nested) down_read_nested(&uobj->mutex, SINGLE_DEPTH_NESTING); else down_read(&uobj->mutex); if (!uobj->live) { put_uobj_read(uobj); return NULL; } return uobj; } static struct ib_uobject *idr_write_uobj(struct idr *idr, int id, struct ib_ucontext *context) { struct ib_uobject *uobj; uobj = __idr_get_uobj(idr, id, context); if (!uobj) return NULL; down_write(&uobj->mutex); if (!uobj->live) { put_uobj_write(uobj); return NULL; } return uobj; } static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context, int nested) { struct ib_uobject *uobj; uobj = idr_read_uobj(idr, id, context, nested); return uobj ? uobj->object : NULL; } static struct ib_pd *idr_read_pd(int pd_handle, struct ib_ucontext *context) { return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context, 0); } static void put_pd_read(struct ib_pd *pd) { put_uobj_read(pd->uobject); } static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context, int nested) { return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context, nested); } static void put_cq_read(struct ib_cq *cq) { put_uobj_read(cq->uobject); } static struct ib_ah *idr_read_ah(int ah_handle, struct ib_ucontext *context) { return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context, 0); } static void put_ah_read(struct ib_ah *ah) { put_uobj_read(ah->uobject); } static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context) { return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context, 0); } static void put_qp_read(struct ib_qp *qp) { put_uobj_read(qp->uobject); } static struct ib_srq *idr_read_srq(int srq_handle, struct ib_ucontext *context) { return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context, 0); } static void put_srq_read(struct ib_srq *srq) { put_uobj_read(srq->uobject); } static struct ib_xrcd *idr_read_xrcd(int xrcd_handle, struct ib_ucontext *context, struct ib_uobject **uobj) { *uobj = idr_read_uobj(&ib_uverbs_xrc_domain_idr, xrcd_handle, context, 0); return *uobj ? (*uobj)->object : NULL; } static void put_xrcd_read(struct ib_uobject *uobj) { put_uobj_read(uobj); } ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_get_context cmd; struct ib_uverbs_get_context_resp resp; struct ib_udata udata; struct ib_device *ibdev = file->device->ib_dev; struct ib_ucontext *ucontext; struct file *filp; int ret; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; mutex_lock(&file->mutex); if (file->ucontext) { ret = -EINVAL; goto err; } INIT_UDATA(&udata, buf + sizeof cmd, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); ucontext = ibdev->alloc_ucontext(ibdev, &udata); if (IS_ERR(ucontext)) { ret = PTR_ERR(file->ucontext); goto err; } ucontext->device = ibdev; INIT_LIST_HEAD(&ucontext->pd_list); INIT_LIST_HEAD(&ucontext->mr_list); INIT_LIST_HEAD(&ucontext->mw_list); INIT_LIST_HEAD(&ucontext->cq_list); INIT_LIST_HEAD(&ucontext->qp_list); INIT_LIST_HEAD(&ucontext->srq_list); INIT_LIST_HEAD(&ucontext->ah_list); - INIT_LIST_HEAD(&ucontext->xrc_domain_list); + INIT_LIST_HEAD(&ucontext->xrcd_list); ucontext->closing = 0; resp.num_comp_vectors = file->device->num_comp_vectors; filp = ib_uverbs_alloc_event_file(file, 1, &resp.async_fd); if (IS_ERR(filp)) { ret = PTR_ERR(filp); goto err_free; } if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; goto err_file; } file->async_file = filp->private_data; INIT_IB_EVENT_HANDLER(&file->event_handler, file->device->ib_dev, ib_uverbs_event_handler); ret = ib_register_event_handler(&file->event_handler); if (ret) goto err_file; kref_get(&file->async_file->ref); kref_get(&file->ref); file->ucontext = ucontext; fd_install(resp.async_fd, filp); mutex_unlock(&file->mutex); return in_len; err_file: put_unused_fd(resp.async_fd); fput(filp); err_free: ibdev->dealloc_ucontext(ucontext); err: mutex_unlock(&file->mutex); return ret; } ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_query_device cmd; struct ib_uverbs_query_device_resp resp; struct ib_device_attr attr; int ret; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; ret = ib_query_device(file->device->ib_dev, &attr); if (ret) return ret; memset(&resp, 0, sizeof resp); resp.fw_ver = attr.fw_ver; resp.node_guid = file->device->ib_dev->node_guid; resp.sys_image_guid = attr.sys_image_guid; resp.max_mr_size = attr.max_mr_size; resp.page_size_cap = attr.page_size_cap; resp.vendor_id = attr.vendor_id; resp.vendor_part_id = attr.vendor_part_id; resp.hw_ver = attr.hw_ver; resp.max_qp = attr.max_qp; resp.max_qp_wr = attr.max_qp_wr; resp.device_cap_flags = attr.device_cap_flags; resp.max_sge = attr.max_sge; resp.max_sge_rd = attr.max_sge_rd; resp.max_cq = attr.max_cq; resp.max_cqe = attr.max_cqe; resp.max_mr = attr.max_mr; resp.max_pd = attr.max_pd; resp.max_qp_rd_atom = attr.max_qp_rd_atom; resp.max_ee_rd_atom = attr.max_ee_rd_atom; resp.max_res_rd_atom = attr.max_res_rd_atom; resp.max_qp_init_rd_atom = attr.max_qp_init_rd_atom; resp.max_ee_init_rd_atom = attr.max_ee_init_rd_atom; resp.atomic_cap = attr.atomic_cap; resp.max_ee = attr.max_ee; resp.max_rdd = attr.max_rdd; resp.max_mw = attr.max_mw; resp.max_raw_ipv6_qp = attr.max_raw_ipv6_qp; resp.max_raw_ethy_qp = attr.max_raw_ethy_qp; resp.max_mcast_grp = attr.max_mcast_grp; resp.max_mcast_qp_attach = attr.max_mcast_qp_attach; resp.max_total_mcast_qp_attach = attr.max_total_mcast_qp_attach; resp.max_ah = attr.max_ah; resp.max_fmr = attr.max_fmr; resp.max_map_per_fmr = attr.max_map_per_fmr; resp.max_srq = attr.max_srq; resp.max_srq_wr = attr.max_srq_wr; resp.max_srq_sge = attr.max_srq_sge; resp.max_pkeys = attr.max_pkeys; resp.local_ca_ack_delay = attr.local_ca_ack_delay; resp.phys_port_cnt = file->device->ib_dev->phys_port_cnt; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) return -EFAULT; return in_len; } ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_query_port cmd; struct ib_uverbs_query_port_resp resp; struct ib_port_attr attr; int ret; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; ret = ib_query_port(file->device->ib_dev, cmd.port_num, &attr); if (ret) return ret; memset(&resp, 0, sizeof resp); resp.state = attr.state; resp.max_mtu = attr.max_mtu; resp.active_mtu = attr.active_mtu; resp.gid_tbl_len = attr.gid_tbl_len; resp.port_cap_flags = attr.port_cap_flags; resp.max_msg_sz = attr.max_msg_sz; resp.bad_pkey_cntr = attr.bad_pkey_cntr; resp.qkey_viol_cntr = attr.qkey_viol_cntr; resp.pkey_tbl_len = attr.pkey_tbl_len; resp.lid = attr.lid; resp.sm_lid = attr.sm_lid; resp.lmc = attr.lmc; resp.max_vl_num = attr.max_vl_num; resp.sm_sl = attr.sm_sl; resp.subnet_timeout = attr.subnet_timeout; resp.init_type_reply = attr.init_type_reply; resp.active_width = attr.active_width; resp.active_speed = attr.active_speed; resp.phys_state = attr.phys_state; resp.link_layer = attr.link_layer; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) return -EFAULT; return in_len; } ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_alloc_pd cmd; struct ib_uverbs_alloc_pd_resp resp; struct ib_udata udata; struct ib_uobject *uobj; struct ib_pd *pd; int ret; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; INIT_UDATA(&udata, buf + sizeof cmd, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); uobj = kmalloc(sizeof *uobj, GFP_KERNEL); if (!uobj) return -ENOMEM; init_uobj(uobj, 0, file->ucontext, &pd_lock_key); down_write(&uobj->mutex); pd = file->device->ib_dev->alloc_pd(file->device->ib_dev, file->ucontext, &udata); if (IS_ERR(pd)) { ret = PTR_ERR(pd); goto err; } pd->device = file->device->ib_dev; pd->uobject = uobj; atomic_set(&pd->usecnt, 0); uobj->object = pd; ret = idr_add_uobj(&ib_uverbs_pd_idr, uobj); if (ret) goto err_idr; memset(&resp, 0, sizeof resp); resp.pd_handle = uobj->id; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; goto err_copy; } mutex_lock(&file->mutex); list_add_tail(&uobj->list, &file->ucontext->pd_list); mutex_unlock(&file->mutex); uobj->live = 1; up_write(&uobj->mutex); return in_len; err_copy: idr_remove_uobj(&ib_uverbs_pd_idr, uobj); err_idr: ib_dealloc_pd(pd); err: put_uobj_write(uobj); return ret; } ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_dealloc_pd cmd; struct ib_uobject *uobj; int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; uobj = idr_write_uobj(&ib_uverbs_pd_idr, cmd.pd_handle, file->ucontext); if (!uobj) return -EINVAL; ret = ib_dealloc_pd(uobj->object); if (!ret) uobj->live = 0; put_uobj_write(uobj); if (ret) return ret; idr_remove_uobj(&ib_uverbs_pd_idr, uobj); mutex_lock(&file->mutex); list_del(&uobj->list); mutex_unlock(&file->mutex); put_uobj(uobj); return in_len; } ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_reg_mr cmd; struct ib_uverbs_reg_mr_resp resp; struct ib_udata udata; struct ib_uobject *uobj; struct ib_pd *pd; struct ib_mr *mr; int ret; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; INIT_UDATA(&udata, buf + sizeof cmd, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)) return -EINVAL; /* * Local write permission is required if remote write or * remote atomic permission is also requested. */ if (cmd.access_flags & (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE) && !(cmd.access_flags & IB_ACCESS_LOCAL_WRITE)) return -EINVAL; uobj = kmalloc(sizeof *uobj, GFP_KERNEL); if (!uobj) return -ENOMEM; init_uobj(uobj, 0, file->ucontext, &mr_lock_key); down_write(&uobj->mutex); pd = idr_read_pd(cmd.pd_handle, file->ucontext); if (!pd) { ret = -EINVAL; goto err_free; } mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va, - cmd.access_flags, &udata); + cmd.access_flags, &udata, 0); if (IS_ERR(mr)) { ret = PTR_ERR(mr); goto err_put; } mr->device = pd->device; mr->pd = pd; mr->uobject = uobj; atomic_inc(&pd->usecnt); atomic_set(&mr->usecnt, 0); uobj->object = mr; ret = idr_add_uobj(&ib_uverbs_mr_idr, uobj); if (ret) goto err_unreg; memset(&resp, 0, sizeof resp); resp.lkey = mr->lkey; resp.rkey = mr->rkey; resp.mr_handle = uobj->id; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; goto err_copy; } put_pd_read(pd); mutex_lock(&file->mutex); list_add_tail(&uobj->list, &file->ucontext->mr_list); mutex_unlock(&file->mutex); uobj->live = 1; up_write(&uobj->mutex); return in_len; err_copy: idr_remove_uobj(&ib_uverbs_mr_idr, uobj); err_unreg: ib_dereg_mr(mr); err_put: put_pd_read(pd); err_free: put_uobj_write(uobj); return ret; } ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_dereg_mr cmd; struct ib_mr *mr; struct ib_uobject *uobj; int ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, file->ucontext); if (!uobj) return -EINVAL; mr = uobj->object; ret = ib_dereg_mr(mr); if (!ret) uobj->live = 0; put_uobj_write(uobj); if (ret) return ret; idr_remove_uobj(&ib_uverbs_mr_idr, uobj); mutex_lock(&file->mutex); list_del(&uobj->list); mutex_unlock(&file->mutex); put_uobj(uobj); return in_len; } ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_create_comp_channel cmd; struct ib_uverbs_create_comp_channel_resp resp; struct file *filp; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; filp = ib_uverbs_alloc_event_file(file, 0, &resp.fd); if (IS_ERR(filp)) return PTR_ERR(filp); if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { put_unused_fd(resp.fd); fput(filp); return -EFAULT; } fd_install(resp.fd, filp); return in_len; } ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_create_cq cmd; struct ib_uverbs_create_cq_resp resp; struct ib_udata udata; struct ib_ucq_object *obj; struct ib_uverbs_event_file *ev_file = NULL; struct ib_cq *cq; int ret; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; INIT_UDATA(&udata, buf + sizeof cmd, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); if (cmd.comp_vector >= file->device->num_comp_vectors) return -EINVAL; obj = kmalloc(sizeof *obj, GFP_KERNEL); if (!obj) return -ENOMEM; init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, &cq_lock_key); down_write(&obj->uobject.mutex); if (cmd.comp_channel >= 0) { ev_file = ib_uverbs_lookup_comp_file(cmd.comp_channel); if (!ev_file) { ret = -EINVAL; goto err; } } obj->uverbs_file = file; obj->comp_events_reported = 0; obj->async_events_reported = 0; INIT_LIST_HEAD(&obj->comp_list); INIT_LIST_HEAD(&obj->async_list); cq = file->device->ib_dev->create_cq(file->device->ib_dev, cmd.cqe, cmd.comp_vector, file->ucontext, &udata); if (IS_ERR(cq)) { ret = PTR_ERR(cq); goto err_file; } cq->device = file->device->ib_dev; cq->uobject = &obj->uobject; cq->comp_handler = ib_uverbs_comp_handler; cq->event_handler = ib_uverbs_cq_event_handler; cq->cq_context = ev_file; atomic_set(&cq->usecnt, 0); obj->uobject.object = cq; ret = idr_add_uobj(&ib_uverbs_cq_idr, &obj->uobject); if (ret) goto err_free; memset(&resp, 0, sizeof resp); resp.cq_handle = obj->uobject.id; resp.cqe = cq->cqe; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; goto err_copy; } mutex_lock(&file->mutex); list_add_tail(&obj->uobject.list, &file->ucontext->cq_list); mutex_unlock(&file->mutex); obj->uobject.live = 1; up_write(&obj->uobject.mutex); return in_len; err_copy: idr_remove_uobj(&ib_uverbs_cq_idr, &obj->uobject); err_free: ib_destroy_cq(cq); err_file: if (ev_file) ib_uverbs_release_ucq(file, ev_file, obj); err: put_uobj_write(&obj->uobject); return ret; } ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_resize_cq cmd; struct ib_uverbs_resize_cq_resp resp; struct ib_udata udata; struct ib_cq *cq; int ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; INIT_UDATA(&udata, buf + sizeof cmd, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); if (!cq) return -EINVAL; ret = cq->device->resize_cq(cq, cmd.cqe, &udata); if (ret) goto out; resp.cqe = cq->cqe; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp.cqe)) ret = -EFAULT; out: put_cq_read(cq); return ret ? ret : in_len; } ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_poll_cq cmd; struct ib_uverbs_poll_cq_resp *resp; struct ib_cq *cq; struct ib_wc *wc; int ret = 0; int i; int rsize; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; wc = kmalloc(cmd.ne * sizeof *wc, GFP_KERNEL); if (!wc) return -ENOMEM; rsize = sizeof *resp + cmd.ne * sizeof(struct ib_uverbs_wc); resp = kmalloc(rsize, GFP_KERNEL); if (!resp) { ret = -ENOMEM; goto out_wc; } cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); if (!cq) { ret = -EINVAL; goto out; } resp->count = ib_poll_cq(cq, cmd.ne, wc); put_cq_read(cq); for (i = 0; i < resp->count; i++) { resp->wc[i].wr_id = wc[i].wr_id; resp->wc[i].status = wc[i].status; resp->wc[i].opcode = wc[i].opcode; resp->wc[i].vendor_err = wc[i].vendor_err; resp->wc[i].byte_len = wc[i].byte_len; resp->wc[i].ex.imm_data = (__u32 __force) wc[i].ex.imm_data; resp->wc[i].qp_num = wc[i].qp->qp_num; resp->wc[i].src_qp = wc[i].src_qp; resp->wc[i].wc_flags = wc[i].wc_flags; resp->wc[i].pkey_index = wc[i].pkey_index; resp->wc[i].slid = wc[i].slid; resp->wc[i].sl = wc[i].sl; resp->wc[i].dlid_path_bits = wc[i].dlid_path_bits; resp->wc[i].port_num = wc[i].port_num; } if (copy_to_user((void __user *) (unsigned long) cmd.response, resp, rsize)) ret = -EFAULT; out: kfree(resp); out_wc: kfree(wc); return ret ? ret : in_len; } ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_req_notify_cq cmd; struct ib_cq *cq; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); if (!cq) return -EINVAL; ib_req_notify_cq(cq, cmd.solicited_only ? IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); put_cq_read(cq); return in_len; } ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_destroy_cq cmd; struct ib_uverbs_destroy_cq_resp resp; struct ib_uobject *uobj; struct ib_cq *cq; struct ib_ucq_object *obj; struct ib_uverbs_event_file *ev_file; int ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; uobj = idr_write_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext); if (!uobj) return -EINVAL; cq = uobj->object; ev_file = cq->cq_context; obj = container_of(cq->uobject, struct ib_ucq_object, uobject); ret = ib_destroy_cq(cq); if (!ret) uobj->live = 0; put_uobj_write(uobj); if (ret) return ret; idr_remove_uobj(&ib_uverbs_cq_idr, uobj); mutex_lock(&file->mutex); list_del(&uobj->list); mutex_unlock(&file->mutex); ib_uverbs_release_ucq(file, ev_file, obj); memset(&resp, 0, sizeof resp); resp.comp_events_reported = obj->comp_events_reported; resp.async_events_reported = obj->async_events_reported; put_uobj(uobj); if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) return -EFAULT; return in_len; } ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_create_qp cmd; struct ib_uverbs_create_qp_resp resp; struct ib_udata udata; struct ib_uqp_object *obj; struct ib_pd *pd; struct ib_cq *scq, *rcq; struct ib_srq *srq; struct ib_qp *qp; struct ib_qp_init_attr attr; struct ib_xrcd *xrcd; struct ib_uobject *xrcd_uobj; int ret; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; INIT_UDATA(&udata, buf + sizeof cmd, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); obj = kmalloc(sizeof *obj, GFP_KERNEL); if (!obj) return -ENOMEM; init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_key); down_write(&obj->uevent.uobject.mutex); srq = (cmd.is_srq && cmd.qp_type != IB_QPT_XRC) ? idr_read_srq(cmd.srq_handle, file->ucontext) : NULL; xrcd = cmd.qp_type == IB_QPT_XRC ? idr_read_xrcd(cmd.srq_handle, file->ucontext, &xrcd_uobj) : NULL; pd = idr_read_pd(cmd.pd_handle, file->ucontext); scq = idr_read_cq(cmd.send_cq_handle, file->ucontext, 0); rcq = cmd.recv_cq_handle == cmd.send_cq_handle ? scq : idr_read_cq(cmd.recv_cq_handle, file->ucontext, 1); if (!pd || !scq || !rcq || (cmd.is_srq && !srq) || (cmd.qp_type == IB_QPT_XRC && !xrcd)) { ret = -EINVAL; goto err_put; } attr.create_flags = 0; attr.event_handler = ib_uverbs_qp_event_handler; attr.qp_context = file; attr.send_cq = scq; attr.recv_cq = rcq; attr.srq = srq; attr.sq_sig_type = cmd.sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; attr.qp_type = cmd.qp_type; - attr.xrc_domain = xrcd; + attr.xrcd = xrcd; attr.create_flags = 0; attr.cap.max_send_wr = cmd.max_send_wr; attr.cap.max_recv_wr = cmd.max_recv_wr; attr.cap.max_send_sge = cmd.max_send_sge; attr.cap.max_recv_sge = cmd.max_recv_sge; attr.cap.max_inline_data = cmd.max_inline_data; obj->uevent.events_reported = 0; INIT_LIST_HEAD(&obj->uevent.event_list); INIT_LIST_HEAD(&obj->mcast_list); qp = pd->device->create_qp(pd, &attr, &udata); if (IS_ERR(qp)) { ret = PTR_ERR(qp); goto err_put; } qp->device = pd->device; qp->pd = pd; qp->send_cq = attr.send_cq; qp->recv_cq = attr.recv_cq; qp->srq = attr.srq; qp->uobject = &obj->uevent.uobject; qp->event_handler = attr.event_handler; qp->qp_context = attr.qp_context; qp->qp_type = attr.qp_type; - qp->xrcd = attr.xrc_domain; + qp->xrcd = attr.xrcd; atomic_inc(&pd->usecnt); atomic_inc(&attr.send_cq->usecnt); atomic_inc(&attr.recv_cq->usecnt); if (attr.srq) atomic_inc(&attr.srq->usecnt); - else if (attr.xrc_domain) - atomic_inc(&attr.xrc_domain->usecnt); + else if (attr.xrcd) + atomic_inc(&attr.xrcd->usecnt); obj->uevent.uobject.object = qp; ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); if (ret) goto err_destroy; memset(&resp, 0, sizeof resp); resp.qpn = qp->qp_num; resp.qp_handle = obj->uevent.uobject.id; resp.max_recv_sge = attr.cap.max_recv_sge; resp.max_send_sge = attr.cap.max_send_sge; resp.max_recv_wr = attr.cap.max_recv_wr; resp.max_send_wr = attr.cap.max_send_wr; resp.max_inline_data = attr.cap.max_inline_data; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; goto err_copy; } put_pd_read(pd); put_cq_read(scq); if (rcq != scq) put_cq_read(rcq); if (srq) put_srq_read(srq); if (xrcd) put_xrcd_read(xrcd_uobj); mutex_lock(&file->mutex); list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); mutex_unlock(&file->mutex); obj->uevent.uobject.live = 1; up_write(&obj->uevent.uobject.mutex); return in_len; err_copy: idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); err_destroy: ib_destroy_qp(qp); err_put: if (pd) put_pd_read(pd); if (scq) put_cq_read(scq); if (rcq && rcq != scq) put_cq_read(rcq); if (srq) put_srq_read(srq); if (xrcd) put_xrcd_read(xrcd_uobj); put_uobj_write(&obj->uevent.uobject); return ret; } ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_query_qp cmd; struct ib_uverbs_query_qp_resp resp; struct ib_qp *qp; struct ib_qp_attr *attr; struct ib_qp_init_attr *init_attr; int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; attr = kmalloc(sizeof *attr, GFP_KERNEL); init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); if (!attr || !init_attr) { ret = -ENOMEM; goto out; } qp = idr_read_qp(cmd.qp_handle, file->ucontext); if (!qp) { ret = -EINVAL; goto out; } ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); put_qp_read(qp); if (ret) goto out; memset(&resp, 0, sizeof resp); resp.qp_state = attr->qp_state; resp.cur_qp_state = attr->cur_qp_state; resp.path_mtu = attr->path_mtu; resp.path_mig_state = attr->path_mig_state; resp.qkey = attr->qkey; resp.rq_psn = attr->rq_psn; resp.sq_psn = attr->sq_psn; resp.dest_qp_num = attr->dest_qp_num; resp.qp_access_flags = attr->qp_access_flags; resp.pkey_index = attr->pkey_index; resp.alt_pkey_index = attr->alt_pkey_index; resp.sq_draining = attr->sq_draining; resp.max_rd_atomic = attr->max_rd_atomic; resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; resp.min_rnr_timer = attr->min_rnr_timer; resp.port_num = attr->port_num; resp.timeout = attr->timeout; resp.retry_cnt = attr->retry_cnt; resp.rnr_retry = attr->rnr_retry; resp.alt_port_num = attr->alt_port_num; resp.alt_timeout = attr->alt_timeout; memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); resp.dest.flow_label = attr->ah_attr.grh.flow_label; resp.dest.sgid_index = attr->ah_attr.grh.sgid_index; resp.dest.hop_limit = attr->ah_attr.grh.hop_limit; resp.dest.traffic_class = attr->ah_attr.grh.traffic_class; resp.dest.dlid = attr->ah_attr.dlid; resp.dest.sl = attr->ah_attr.sl; resp.dest.src_path_bits = attr->ah_attr.src_path_bits; resp.dest.static_rate = attr->ah_attr.static_rate; resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH); resp.dest.port_num = attr->ah_attr.port_num; memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; resp.alt_dest.dlid = attr->alt_ah_attr.dlid; resp.alt_dest.sl = attr->alt_ah_attr.sl; resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate; resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH); resp.alt_dest.port_num = attr->alt_ah_attr.port_num; resp.max_send_wr = init_attr->cap.max_send_wr; resp.max_recv_wr = init_attr->cap.max_recv_wr; resp.max_send_sge = init_attr->cap.max_send_sge; resp.max_recv_sge = init_attr->cap.max_recv_sge; resp.max_inline_data = init_attr->cap.max_inline_data; resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) ret = -EFAULT; out: kfree(attr); kfree(init_attr); return ret ? ret : in_len; } ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_modify_qp cmd; struct ib_udata udata; struct ib_qp *qp; struct ib_qp_attr *attr; int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, out_len); attr = kmalloc(sizeof *attr, GFP_KERNEL); if (!attr) return -ENOMEM; qp = idr_read_qp(cmd.qp_handle, file->ucontext); if (!qp) { ret = -EINVAL; goto out; } attr->qp_state = cmd.qp_state; attr->cur_qp_state = cmd.cur_qp_state; attr->path_mtu = cmd.path_mtu; attr->path_mig_state = cmd.path_mig_state; attr->qkey = cmd.qkey; attr->rq_psn = cmd.rq_psn; attr->sq_psn = cmd.sq_psn; attr->dest_qp_num = cmd.dest_qp_num; attr->qp_access_flags = cmd.qp_access_flags; attr->pkey_index = cmd.pkey_index; attr->alt_pkey_index = cmd.alt_pkey_index; attr->en_sqd_async_notify = cmd.en_sqd_async_notify; attr->max_rd_atomic = cmd.max_rd_atomic; attr->max_dest_rd_atomic = cmd.max_dest_rd_atomic; attr->min_rnr_timer = cmd.min_rnr_timer; attr->port_num = cmd.port_num; attr->timeout = cmd.timeout; attr->retry_cnt = cmd.retry_cnt; attr->rnr_retry = cmd.rnr_retry; attr->alt_port_num = cmd.alt_port_num; attr->alt_timeout = cmd.alt_timeout; memcpy(attr->ah_attr.grh.dgid.raw, cmd.dest.dgid, 16); attr->ah_attr.grh.flow_label = cmd.dest.flow_label; attr->ah_attr.grh.sgid_index = cmd.dest.sgid_index; attr->ah_attr.grh.hop_limit = cmd.dest.hop_limit; attr->ah_attr.grh.traffic_class = cmd.dest.traffic_class; attr->ah_attr.dlid = cmd.dest.dlid; attr->ah_attr.sl = cmd.dest.sl; attr->ah_attr.src_path_bits = cmd.dest.src_path_bits; attr->ah_attr.static_rate = cmd.dest.static_rate; attr->ah_attr.ah_flags = cmd.dest.is_global ? IB_AH_GRH : 0; attr->ah_attr.port_num = cmd.dest.port_num; memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd.alt_dest.dgid, 16); attr->alt_ah_attr.grh.flow_label = cmd.alt_dest.flow_label; attr->alt_ah_attr.grh.sgid_index = cmd.alt_dest.sgid_index; attr->alt_ah_attr.grh.hop_limit = cmd.alt_dest.hop_limit; attr->alt_ah_attr.grh.traffic_class = cmd.alt_dest.traffic_class; attr->alt_ah_attr.dlid = cmd.alt_dest.dlid; attr->alt_ah_attr.sl = cmd.alt_dest.sl; attr->alt_ah_attr.src_path_bits = cmd.alt_dest.src_path_bits; attr->alt_ah_attr.static_rate = cmd.alt_dest.static_rate; attr->alt_ah_attr.ah_flags = cmd.alt_dest.is_global ? IB_AH_GRH : 0; attr->alt_ah_attr.port_num = cmd.alt_dest.port_num; ret = qp->device->modify_qp(qp, attr, cmd.attr_mask, &udata); put_qp_read(qp); if (ret) goto out; ret = in_len; out: kfree(attr); return ret; } ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_destroy_qp cmd; struct ib_uverbs_destroy_qp_resp resp; struct ib_uobject *uobj; struct ib_qp *qp; struct ib_uqp_object *obj; int ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; memset(&resp, 0, sizeof resp); uobj = idr_write_uobj(&ib_uverbs_qp_idr, cmd.qp_handle, file->ucontext); if (!uobj) return -EINVAL; qp = uobj->object; obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); if (!list_empty(&obj->mcast_list)) { put_uobj_write(uobj); return -EBUSY; } ret = ib_destroy_qp(qp); if (!ret) uobj->live = 0; put_uobj_write(uobj); if (ret) return ret; idr_remove_uobj(&ib_uverbs_qp_idr, uobj); mutex_lock(&file->mutex); list_del(&uobj->list); mutex_unlock(&file->mutex); ib_uverbs_release_uevent(file, &obj->uevent); resp.events_reported = obj->uevent.events_reported; put_uobj(uobj); if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) return -EFAULT; return in_len; } ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_post_send cmd; struct ib_uverbs_post_send_resp resp; struct ib_uverbs_send_wr *user_wr; struct ib_send_wr *wr = NULL, *last, *next, *bad_wr; struct ib_qp *qp; int i, sg_ind; int is_ud; ssize_t ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count + cmd.sge_count * sizeof (struct ib_uverbs_sge)) return -EINVAL; if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr)) return -EINVAL; user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); if (!user_wr) return -ENOMEM; qp = idr_read_qp(cmd.qp_handle, file->ucontext); if (!qp) goto out; is_ud = qp->qp_type == IB_QPT_UD; sg_ind = 0; last = NULL; for (i = 0; i < cmd.wr_count; ++i) { if (copy_from_user(user_wr, buf + sizeof cmd + i * cmd.wqe_size, cmd.wqe_size)) { ret = -EFAULT; goto out_put; } if (user_wr->num_sge + sg_ind > cmd.sge_count) { ret = -EINVAL; goto out_put; } next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + user_wr->num_sge * sizeof (struct ib_sge), GFP_KERNEL); if (!next) { ret = -ENOMEM; goto out_put; } if (!last) wr = next; else last->next = next; last = next; next->next = NULL; next->wr_id = user_wr->wr_id; next->num_sge = user_wr->num_sge; next->opcode = user_wr->opcode; next->send_flags = user_wr->send_flags; if (is_ud) { next->wr.ud.ah = idr_read_ah(user_wr->wr.ud.ah, file->ucontext); if (!next->wr.ud.ah) { ret = -EINVAL; goto out_put; } next->wr.ud.remote_qpn = user_wr->wr.ud.remote_qpn; next->wr.ud.remote_qkey = user_wr->wr.ud.remote_qkey; } else { switch (next->opcode) { case IB_WR_RDMA_WRITE_WITH_IMM: next->ex.imm_data = (__be32 __force) user_wr->ex.imm_data; case IB_WR_RDMA_WRITE: case IB_WR_RDMA_READ: next->wr.rdma.remote_addr = user_wr->wr.rdma.remote_addr; next->wr.rdma.rkey = user_wr->wr.rdma.rkey; break; case IB_WR_SEND_WITH_IMM: next->ex.imm_data = (__be32 __force) user_wr->ex.imm_data; break; case IB_WR_SEND_WITH_INV: next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey; break; case IB_WR_ATOMIC_CMP_AND_SWP: case IB_WR_ATOMIC_FETCH_AND_ADD: next->wr.atomic.remote_addr = user_wr->wr.atomic.remote_addr; next->wr.atomic.compare_add = user_wr->wr.atomic.compare_add; next->wr.atomic.swap = user_wr->wr.atomic.swap; next->wr.atomic.rkey = user_wr->wr.atomic.rkey; break; default: break; } } if (next->num_sge) { next->sg_list = (void *) next + ALIGN(sizeof *next, sizeof (struct ib_sge)); if (copy_from_user(next->sg_list, buf + sizeof cmd + cmd.wr_count * cmd.wqe_size + sg_ind * sizeof (struct ib_sge), next->num_sge * sizeof (struct ib_sge))) { ret = -EFAULT; goto out_put; } sg_ind += next->num_sge; } else next->sg_list = NULL; } resp.bad_wr = 0; ret = qp->device->post_send(qp, wr, &bad_wr); if (ret) for (next = wr; next; next = next->next) { ++resp.bad_wr; if (next == bad_wr) break; } if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) ret = -EFAULT; out_put: put_qp_read(qp); while (wr) { if (is_ud && wr->wr.ud.ah) put_ah_read(wr->wr.ud.ah); next = wr->next; kfree(wr); wr = next; } out: kfree(user_wr); return ret ? ret : in_len; } static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf, int in_len, u32 wr_count, u32 sge_count, u32 wqe_size) { struct ib_uverbs_recv_wr *user_wr; struct ib_recv_wr *wr = NULL, *last, *next; int sg_ind; int i; int ret; if (in_len < wqe_size * wr_count + sge_count * sizeof (struct ib_uverbs_sge)) return ERR_PTR(-EINVAL); if (wqe_size < sizeof (struct ib_uverbs_recv_wr)) return ERR_PTR(-EINVAL); user_wr = kmalloc(wqe_size, GFP_KERNEL); if (!user_wr) return ERR_PTR(-ENOMEM); sg_ind = 0; last = NULL; for (i = 0; i < wr_count; ++i) { if (copy_from_user(user_wr, buf + i * wqe_size, wqe_size)) { ret = -EFAULT; goto err; } if (user_wr->num_sge + sg_ind > sge_count) { ret = -EINVAL; goto err; } next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + user_wr->num_sge * sizeof (struct ib_sge), GFP_KERNEL); if (!next) { ret = -ENOMEM; goto err; } if (!last) wr = next; else last->next = next; last = next; next->next = NULL; next->wr_id = user_wr->wr_id; next->num_sge = user_wr->num_sge; if (next->num_sge) { next->sg_list = (void *) next + ALIGN(sizeof *next, sizeof (struct ib_sge)); if (copy_from_user(next->sg_list, buf + wr_count * wqe_size + sg_ind * sizeof (struct ib_sge), next->num_sge * sizeof (struct ib_sge))) { ret = -EFAULT; goto err; } sg_ind += next->num_sge; } else next->sg_list = NULL; } kfree(user_wr); return wr; err: kfree(user_wr); while (wr) { next = wr->next; kfree(wr); wr = next; } return ERR_PTR(ret); } ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_post_recv cmd; struct ib_uverbs_post_recv_resp resp; struct ib_recv_wr *wr, *next, *bad_wr; struct ib_qp *qp; ssize_t ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, in_len - sizeof cmd, cmd.wr_count, cmd.sge_count, cmd.wqe_size); if (IS_ERR(wr)) return PTR_ERR(wr); qp = idr_read_qp(cmd.qp_handle, file->ucontext); if (!qp) goto out; resp.bad_wr = 0; ret = qp->device->post_recv(qp, wr, &bad_wr); put_qp_read(qp); if (ret) for (next = wr; next; next = next->next) { ++resp.bad_wr; if (next == bad_wr) break; } if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) ret = -EFAULT; out: while (wr) { next = wr->next; kfree(wr); wr = next; } return ret ? ret : in_len; } ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_post_srq_recv cmd; struct ib_uverbs_post_srq_recv_resp resp; struct ib_recv_wr *wr, *next, *bad_wr; struct ib_srq *srq; ssize_t ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, in_len - sizeof cmd, cmd.wr_count, cmd.sge_count, cmd.wqe_size); if (IS_ERR(wr)) return PTR_ERR(wr); srq = idr_read_srq(cmd.srq_handle, file->ucontext); if (!srq) goto out; resp.bad_wr = 0; ret = srq->device->post_srq_recv(srq, wr, &bad_wr); put_srq_read(srq); if (ret) for (next = wr; next; next = next->next) { ++resp.bad_wr; if (next == bad_wr) break; } if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) ret = -EFAULT; out: while (wr) { next = wr->next; kfree(wr); wr = next; } return ret ? ret : in_len; } ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_create_ah cmd; struct ib_uverbs_create_ah_resp resp; struct ib_uobject *uobj; struct ib_pd *pd; struct ib_ah *ah; struct ib_ah_attr attr; int ret; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; uobj = kmalloc(sizeof *uobj, GFP_KERNEL); if (!uobj) return -ENOMEM; init_uobj(uobj, cmd.user_handle, file->ucontext, &ah_lock_key); down_write(&uobj->mutex); pd = idr_read_pd(cmd.pd_handle, file->ucontext); if (!pd) { ret = -EINVAL; goto err; } attr.dlid = cmd.attr.dlid; attr.sl = cmd.attr.sl; attr.src_path_bits = cmd.attr.src_path_bits; attr.static_rate = cmd.attr.static_rate; attr.ah_flags = cmd.attr.is_global ? IB_AH_GRH : 0; attr.port_num = cmd.attr.port_num; attr.grh.flow_label = cmd.attr.grh.flow_label; attr.grh.sgid_index = cmd.attr.grh.sgid_index; attr.grh.hop_limit = cmd.attr.grh.hop_limit; attr.grh.traffic_class = cmd.attr.grh.traffic_class; memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16); ah = ib_create_ah(pd, &attr); if (IS_ERR(ah)) { ret = PTR_ERR(ah); goto err_put; } ah->uobject = uobj; uobj->object = ah; ret = idr_add_uobj(&ib_uverbs_ah_idr, uobj); if (ret) goto err_destroy; resp.ah_handle = uobj->id; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; goto err_copy; } put_pd_read(pd); mutex_lock(&file->mutex); list_add_tail(&uobj->list, &file->ucontext->ah_list); mutex_unlock(&file->mutex); uobj->live = 1; up_write(&uobj->mutex); return in_len; err_copy: idr_remove_uobj(&ib_uverbs_ah_idr, uobj); err_destroy: ib_destroy_ah(ah); err_put: put_pd_read(pd); err: put_uobj_write(uobj); return ret; } ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_destroy_ah cmd; struct ib_ah *ah; struct ib_uobject *uobj; int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; uobj = idr_write_uobj(&ib_uverbs_ah_idr, cmd.ah_handle, file->ucontext); if (!uobj) return -EINVAL; ah = uobj->object; ret = ib_destroy_ah(ah); if (!ret) uobj->live = 0; put_uobj_write(uobj); if (ret) return ret; idr_remove_uobj(&ib_uverbs_ah_idr, uobj); mutex_lock(&file->mutex); list_del(&uobj->list); mutex_unlock(&file->mutex); put_uobj(uobj); return in_len; } ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_attach_mcast cmd; struct ib_qp *qp; struct ib_uqp_object *obj; struct ib_uverbs_mcast_entry *mcast; int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; qp = idr_read_qp(cmd.qp_handle, file->ucontext); if (!qp) return -EINVAL; obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); list_for_each_entry(mcast, &obj->mcast_list, list) if (cmd.mlid == mcast->lid && !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { ret = 0; goto out_put; } mcast = kmalloc(sizeof *mcast, GFP_KERNEL); if (!mcast) { ret = -ENOMEM; goto out_put; } mcast->lid = cmd.mlid; memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw); ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); if (!ret) list_add_tail(&mcast->list, &obj->mcast_list); else kfree(mcast); out_put: put_qp_read(qp); return ret ? ret : in_len; } ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_detach_mcast cmd; struct ib_uqp_object *obj; struct ib_qp *qp; struct ib_uverbs_mcast_entry *mcast; int ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; qp = idr_read_qp(cmd.qp_handle, file->ucontext); if (!qp) return -EINVAL; ret = ib_detach_mcast(qp, (union ib_gid *) cmd.gid, cmd.mlid); if (ret) goto out_put; obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); list_for_each_entry(mcast, &obj->mcast_list, list) if (cmd.mlid == mcast->lid && !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { list_del(&mcast->list); kfree(mcast); break; } out_put: put_qp_read(qp); return ret ? ret : in_len; } ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_create_srq cmd; struct ib_uverbs_create_srq_resp resp; struct ib_udata udata; struct ib_uevent_object *obj; struct ib_pd *pd; struct ib_srq *srq; struct ib_srq_init_attr attr; int ret; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; INIT_UDATA(&udata, buf + sizeof cmd, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); obj = kmalloc(sizeof *obj, GFP_KERNEL); if (!obj) return -ENOMEM; init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, &srq_lock_key); down_write(&obj->uobject.mutex); pd = idr_read_pd(cmd.pd_handle, file->ucontext); if (!pd) { ret = -EINVAL; goto err; } attr.event_handler = ib_uverbs_srq_event_handler; attr.srq_context = file; attr.attr.max_wr = cmd.max_wr; attr.attr.max_sge = cmd.max_sge; attr.attr.srq_limit = cmd.srq_limit; obj->events_reported = 0; INIT_LIST_HEAD(&obj->event_list); srq = pd->device->create_srq(pd, &attr, &udata); if (IS_ERR(srq)) { ret = PTR_ERR(srq); goto err_put; } srq->device = pd->device; srq->pd = pd; srq->uobject = &obj->uobject; srq->event_handler = attr.event_handler; srq->srq_context = attr.srq_context; - srq->xrc_cq = NULL; - srq->xrcd = NULL; + srq->ext.xrc.cq = NULL; + srq->ext.xrc.xrcd = NULL; atomic_inc(&pd->usecnt); atomic_set(&srq->usecnt, 0); obj->uobject.object = srq; ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uobject); if (ret) goto err_destroy; memset(&resp, 0, sizeof resp); resp.srq_handle = obj->uobject.id; resp.max_wr = attr.attr.max_wr; resp.max_sge = attr.attr.max_sge; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; goto err_copy; } put_pd_read(pd); mutex_lock(&file->mutex); list_add_tail(&obj->uobject.list, &file->ucontext->srq_list); mutex_unlock(&file->mutex); obj->uobject.live = 1; up_write(&obj->uobject.mutex); return in_len; err_copy: idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uobject); err_destroy: ib_destroy_srq(srq); err_put: put_pd_read(pd); err: put_uobj_write(&obj->uobject); return ret; } ssize_t ib_uverbs_create_xrc_srq(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { - struct ib_uverbs_create_xrc_srq cmd; + struct ib_uverbs_create_xsrq cmd; struct ib_uverbs_create_srq_resp resp; struct ib_udata udata; struct ib_uevent_object *obj; struct ib_pd *pd; struct ib_srq *srq; struct ib_cq *xrc_cq; struct ib_xrcd *xrcd; struct ib_srq_init_attr attr; struct ib_uobject *xrcd_uobj; int ret; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; INIT_UDATA(&udata, buf + sizeof cmd, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); obj = kmalloc(sizeof *obj, GFP_KERNEL); if (!obj) return -ENOMEM; init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, &srq_lock_key); down_write(&obj->uobject.mutex); pd = idr_read_pd(cmd.pd_handle, file->ucontext); if (!pd) { ret = -EINVAL; goto err; } - xrc_cq = idr_read_cq(cmd.xrc_cq, file->ucontext, 0); + xrc_cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); if (!xrc_cq) { ret = -EINVAL; goto err_put_pd; } xrcd = idr_read_xrcd(cmd.xrcd_handle, file->ucontext, &xrcd_uobj); if (!xrcd) { ret = -EINVAL; goto err_put_cq; } attr.event_handler = ib_uverbs_srq_event_handler; attr.srq_context = file; attr.attr.max_wr = cmd.max_wr; attr.attr.max_sge = cmd.max_sge; attr.attr.srq_limit = cmd.srq_limit; obj->events_reported = 0; INIT_LIST_HEAD(&obj->event_list); srq = pd->device->create_xrc_srq(pd, xrc_cq, xrcd, &attr, &udata); if (IS_ERR(srq)) { ret = PTR_ERR(srq); goto err_put; } srq->device = pd->device; srq->pd = pd; srq->uobject = &obj->uobject; srq->event_handler = attr.event_handler; srq->srq_context = attr.srq_context; - srq->xrc_cq = xrc_cq; - srq->xrcd = xrcd; + srq->ext.xrc.cq = xrc_cq; + srq->ext.xrc.xrcd = xrcd; atomic_inc(&pd->usecnt); atomic_inc(&xrc_cq->usecnt); atomic_inc(&xrcd->usecnt); atomic_set(&srq->usecnt, 0); obj->uobject.object = srq; ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uobject); if (ret) goto err_destroy; memset(&resp, 0, sizeof resp); resp.srq_handle = obj->uobject.id; resp.max_wr = attr.attr.max_wr; resp.max_sge = attr.attr.max_sge; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; goto err_copy; } put_xrcd_read(xrcd_uobj); put_cq_read(xrc_cq); put_pd_read(pd); mutex_lock(&file->mutex); list_add_tail(&obj->uobject.list, &file->ucontext->srq_list); mutex_unlock(&file->mutex); obj->uobject.live = 1; up_write(&obj->uobject.mutex); return in_len; err_copy: idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uobject); err_destroy: ib_destroy_srq(srq); err_put: put_xrcd_read(xrcd_uobj); err_put_cq: put_cq_read(xrc_cq); err_put_pd: put_pd_read(pd); err: put_uobj_write(&obj->uobject); return ret; } ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_modify_srq cmd; struct ib_udata udata; struct ib_srq *srq; struct ib_srq_attr attr; int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, out_len); srq = idr_read_srq(cmd.srq_handle, file->ucontext); if (!srq) return -EINVAL; attr.max_wr = cmd.max_wr; attr.srq_limit = cmd.srq_limit; ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata); put_srq_read(srq); return ret ? ret : in_len; } ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_query_srq cmd; struct ib_uverbs_query_srq_resp resp; struct ib_srq_attr attr; struct ib_srq *srq; int ret; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; srq = idr_read_srq(cmd.srq_handle, file->ucontext); if (!srq) return -EINVAL; ret = ib_query_srq(srq, &attr); put_srq_read(srq); if (ret) return ret; memset(&resp, 0, sizeof resp); resp.max_wr = attr.max_wr; resp.max_sge = attr.max_sge; resp.srq_limit = attr.srq_limit; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) return -EFAULT; return in_len; } ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_destroy_srq cmd; struct ib_uverbs_destroy_srq_resp resp; struct ib_uobject *uobj; struct ib_srq *srq; struct ib_uevent_object *obj; int ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; uobj = idr_write_uobj(&ib_uverbs_srq_idr, cmd.srq_handle, file->ucontext); if (!uobj) return -EINVAL; srq = uobj->object; obj = container_of(uobj, struct ib_uevent_object, uobject); ret = ib_destroy_srq(srq); if (!ret) uobj->live = 0; put_uobj_write(uobj); if (ret) return ret; idr_remove_uobj(&ib_uverbs_srq_idr, uobj); mutex_lock(&file->mutex); list_del(&uobj->list); mutex_unlock(&file->mutex); ib_uverbs_release_uevent(file, obj); memset(&resp, 0, sizeof resp); resp.events_reported = obj->events_reported; put_uobj(uobj); if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) ret = -EFAULT; return ret ? ret : in_len; } static struct inode *xrc_file2inode(struct file *f) { return f->f_dentry->d_inode; } struct xrcd_table_entry { struct rb_node node; struct inode *inode; struct ib_xrcd *xrcd; }; static int xrcd_table_insert(struct ib_device *dev, struct inode *i_n, struct ib_xrcd *xrcd) { struct xrcd_table_entry *entry, *scan; struct rb_node **p = &dev->ib_uverbs_xrcd_table.rb_node; struct rb_node *parent = NULL; entry = kmalloc(sizeof(struct xrcd_table_entry), GFP_KERNEL); if (!entry) return -ENOMEM; entry->inode = i_n; entry->xrcd = xrcd; while (*p) { parent = *p; scan = rb_entry(parent, struct xrcd_table_entry, node); if (i_n < scan->inode) p = &(*p)->rb_left; else if (i_n > scan->inode) p = &(*p)->rb_right; else { kfree(entry); return -EEXIST; } } rb_link_node(&entry->node, parent, p); rb_insert_color(&entry->node, &dev->ib_uverbs_xrcd_table); igrab(i_n); return 0; } static struct xrcd_table_entry *xrcd_table_search(struct ib_device *dev, struct inode *i_n) { struct xrcd_table_entry *scan; struct rb_node **p = &dev->ib_uverbs_xrcd_table.rb_node; struct rb_node *parent = NULL; while (*p) { parent = *p; scan = rb_entry(parent, struct xrcd_table_entry, node); if (i_n < scan->inode) p = &(*p)->rb_left; else if (i_n > scan->inode) p = &(*p)->rb_right; else return scan; } return NULL; } static int find_xrcd(struct ib_device *dev, struct inode *i_n, struct ib_xrcd **xrcd) { struct xrcd_table_entry *entry; entry = xrcd_table_search(dev, i_n); if (!entry) return -EINVAL; *xrcd = entry->xrcd; return 0; } static void xrcd_table_delete(struct ib_device *dev, struct inode *i_n) { struct xrcd_table_entry *entry = xrcd_table_search(dev, i_n); if (entry) { iput(i_n); rb_erase(&entry->node, &dev->ib_uverbs_xrcd_table); kfree(entry); } } ssize_t ib_uverbs_open_xrc_domain(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_open_xrc_domain cmd; struct ib_uverbs_open_xrc_domain_resp resp; struct ib_udata udata; struct ib_uobject *uobj; struct ib_uxrcd_object *xrcd_uobj; struct ib_xrcd *xrcd = NULL; struct file *f = NULL; struct inode *inode = NULL; int ret = 0; int new_xrcd = 0; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; INIT_UDATA(&udata, buf + sizeof cmd, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); mutex_lock(&file->device->ib_dev->xrcd_table_mutex); if (cmd.fd != (u32) (-1)) { /* search for file descriptor */ f = fget(cmd.fd); if (!f) { ret = -EBADF; goto err_table_mutex_unlock; } inode = xrc_file2inode(f); if (!inode) { ret = -EBADF; goto err_table_mutex_unlock; } ret = find_xrcd(file->device->ib_dev, inode, &xrcd); if (ret && !(cmd.oflags & O_CREAT)) { /* no file descriptor. Need CREATE flag */ ret = -EAGAIN; goto err_table_mutex_unlock; } if (xrcd && cmd.oflags & O_EXCL) { ret = -EINVAL; goto err_table_mutex_unlock; } } xrcd_uobj = kmalloc(sizeof *xrcd_uobj, GFP_KERNEL); if (!xrcd_uobj) { ret = -ENOMEM; goto err_table_mutex_unlock; } uobj = &xrcd_uobj->uobject; init_uobj(uobj, 0, file->ucontext, &pd_lock_key); down_write(&uobj->mutex); if (!xrcd) { xrcd = file->device->ib_dev->alloc_xrcd(file->device->ib_dev, file->ucontext, &udata); if (IS_ERR(xrcd)) { ret = PTR_ERR(xrcd); goto err; } xrcd->uobject = (cmd.fd == -1) ? uobj : NULL; xrcd->inode = inode; xrcd->device = file->device->ib_dev; atomic_set(&xrcd->usecnt, 0); new_xrcd = 1; } uobj->object = xrcd; ret = idr_add_uobj(&ib_uverbs_xrc_domain_idr, uobj); if (ret) goto err_idr; memset(&resp, 0, sizeof resp); resp.xrcd_handle = uobj->id; if (inode) { if (new_xrcd) { /* create new inode/xrcd table entry */ ret = xrcd_table_insert(file->device->ib_dev, inode, xrcd); if (ret) goto err_insert_xrcd; } atomic_inc(&xrcd->usecnt); } if (f) fput(f); if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; goto err_copy; } INIT_LIST_HEAD(&xrcd_uobj->xrc_reg_qp_list); mutex_lock(&file->mutex); - list_add_tail(&uobj->list, &file->ucontext->xrc_domain_list); + list_add_tail(&uobj->list, &file->ucontext->xrcd_list); mutex_unlock(&file->mutex); uobj->live = 1; up_write(&uobj->mutex); mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); return in_len; err_copy: if (inode) { if (new_xrcd) xrcd_table_delete(file->device->ib_dev, inode); atomic_dec(&xrcd->usecnt); } err_insert_xrcd: idr_remove_uobj(&ib_uverbs_xrc_domain_idr, uobj); err_idr: ib_dealloc_xrcd(xrcd); err: put_uobj_write(uobj); err_table_mutex_unlock: if (f) fput(f); mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); return ret; } ssize_t ib_uverbs_close_xrc_domain(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_close_xrc_domain cmd; struct ib_uobject *uobj, *t_uobj; struct ib_uxrcd_object *xrcd_uobj; struct ib_xrcd *xrcd = NULL; struct inode *inode = NULL; int ret = 0; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; mutex_lock(&file->device->ib_dev->xrcd_table_mutex); uobj = idr_write_uobj(&ib_uverbs_xrc_domain_idr, cmd.xrcd_handle, file->ucontext); if (!uobj) { ret = -EINVAL; goto err_unlock_mutex; } mutex_lock(&file->mutex); if (!ret) { list_for_each_entry(t_uobj, &file->ucontext->qp_list, list) { struct ib_qp *qp = t_uobj->object; if (qp->xrcd && qp->xrcd == uobj->object) { ret = -EBUSY; break; } } } if (!ret) { list_for_each_entry(t_uobj, &file->ucontext->srq_list, list) { struct ib_srq *srq = t_uobj->object; - if (srq->xrcd && srq->xrcd == uobj->object) { + if (srq->ext.xrc.xrcd && srq->ext.xrc.xrcd == uobj->object) { ret = -EBUSY; break; } } } mutex_unlock(&file->mutex); if (ret) { put_uobj_write(uobj); goto err_unlock_mutex; } xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); if (!list_empty(&xrcd_uobj->xrc_reg_qp_list)) { ret = -EBUSY; put_uobj_write(uobj); goto err_unlock_mutex; } xrcd = (struct ib_xrcd *) (uobj->object); inode = xrcd->inode; if (inode) atomic_dec(&xrcd->usecnt); ret = ib_dealloc_xrcd(uobj->object); if (!ret) uobj->live = 0; put_uobj_write(uobj); if (ret && !inode) goto err_unlock_mutex; if (!ret && inode) xrcd_table_delete(file->device->ib_dev, inode); idr_remove_uobj(&ib_uverbs_xrc_domain_idr, uobj); mutex_lock(&file->mutex); list_del(&uobj->list); mutex_unlock(&file->mutex); put_uobj(uobj); mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); return in_len; err_unlock_mutex: mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); return ret; } void ib_uverbs_dealloc_xrcd(struct ib_device *ib_dev, struct ib_xrcd *xrcd) { struct inode *inode = NULL; int ret = 0; inode = xrcd->inode; if (inode) atomic_dec(&xrcd->usecnt); ret = ib_dealloc_xrcd(xrcd); if (!ret && inode) xrcd_table_delete(ib_dev, inode); } ssize_t ib_uverbs_create_xrc_rcv_qp(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_create_xrc_rcv_qp cmd; struct ib_uverbs_create_xrc_rcv_qp_resp resp; struct ib_uxrc_rcv_object *obj; struct ib_qp_init_attr init_attr; struct ib_xrcd *xrcd; struct ib_uobject *uobj; struct ib_uxrcd_object *xrcd_uobj; u32 qp_num; int err; if (out_len < sizeof resp) return -ENOSPC; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; obj = kzalloc(sizeof *obj, GFP_KERNEL); if (!obj) return -ENOMEM; xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); if (!xrcd) { err = -EINVAL; goto err_out; } init_attr.event_handler = ib_uverbs_xrc_rcv_qp_event_handler; init_attr.qp_context = file; init_attr.srq = NULL; init_attr.sq_sig_type = cmd.sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; init_attr.qp_type = IB_QPT_XRC; - init_attr.xrc_domain = xrcd; + init_attr.xrcd = xrcd; init_attr.cap.max_send_wr = 1; init_attr.cap.max_recv_wr = 0; init_attr.cap.max_send_sge = 1; init_attr.cap.max_recv_sge = 0; init_attr.cap.max_inline_data = 0; err = xrcd->device->create_xrc_rcv_qp(&init_attr, &qp_num); if (err) goto err_put; memset(&resp, 0, sizeof resp); resp.qpn = qp_num; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { err = -EFAULT; goto err_destroy; } atomic_inc(&xrcd->usecnt); put_xrcd_read(uobj); obj->qp_num = qp_num; obj->domain_handle = cmd.xrc_domain_handle; xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); mutex_lock(&file->device->ib_dev->xrcd_table_mutex); list_add_tail(&obj->list, &xrcd_uobj->xrc_reg_qp_list); mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); return in_len; err_destroy: xrcd->device->unreg_xrc_rcv_qp(xrcd, file, qp_num); err_put: put_xrcd_read(uobj); err_out: kfree(obj); return err; } ssize_t ib_uverbs_modify_xrc_rcv_qp(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_modify_xrc_rcv_qp cmd; struct ib_qp_attr *attr; struct ib_xrcd *xrcd; struct ib_uobject *uobj; int err; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; attr = kzalloc(sizeof *attr, GFP_KERNEL); if (!attr) return -ENOMEM; xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); if (!xrcd) { kfree(attr); return -EINVAL; } attr->qp_state = cmd.qp_state; attr->cur_qp_state = cmd.cur_qp_state; attr->qp_access_flags = cmd.qp_access_flags; attr->pkey_index = cmd.pkey_index; attr->port_num = cmd.port_num; attr->path_mtu = cmd.path_mtu; attr->path_mig_state = cmd.path_mig_state; attr->qkey = cmd.qkey; attr->rq_psn = cmd.rq_psn; attr->sq_psn = cmd.sq_psn; attr->dest_qp_num = cmd.dest_qp_num; attr->alt_pkey_index = cmd.alt_pkey_index; attr->en_sqd_async_notify = cmd.en_sqd_async_notify; attr->max_rd_atomic = cmd.max_rd_atomic; attr->max_dest_rd_atomic = cmd.max_dest_rd_atomic; attr->min_rnr_timer = cmd.min_rnr_timer; attr->port_num = cmd.port_num; attr->timeout = cmd.timeout; attr->retry_cnt = cmd.retry_cnt; attr->rnr_retry = cmd.rnr_retry; attr->alt_port_num = cmd.alt_port_num; attr->alt_timeout = cmd.alt_timeout; memcpy(attr->ah_attr.grh.dgid.raw, cmd.dest.dgid, 16); attr->ah_attr.grh.flow_label = cmd.dest.flow_label; attr->ah_attr.grh.sgid_index = cmd.dest.sgid_index; attr->ah_attr.grh.hop_limit = cmd.dest.hop_limit; attr->ah_attr.grh.traffic_class = cmd.dest.traffic_class; attr->ah_attr.dlid = cmd.dest.dlid; attr->ah_attr.sl = cmd.dest.sl; attr->ah_attr.src_path_bits = cmd.dest.src_path_bits; attr->ah_attr.static_rate = cmd.dest.static_rate; attr->ah_attr.ah_flags = cmd.dest.is_global ? IB_AH_GRH : 0; attr->ah_attr.port_num = cmd.dest.port_num; memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd.alt_dest.dgid, 16); attr->alt_ah_attr.grh.flow_label = cmd.alt_dest.flow_label; attr->alt_ah_attr.grh.sgid_index = cmd.alt_dest.sgid_index; attr->alt_ah_attr.grh.hop_limit = cmd.alt_dest.hop_limit; attr->alt_ah_attr.grh.traffic_class = cmd.alt_dest.traffic_class; attr->alt_ah_attr.dlid = cmd.alt_dest.dlid; attr->alt_ah_attr.sl = cmd.alt_dest.sl; attr->alt_ah_attr.src_path_bits = cmd.alt_dest.src_path_bits; attr->alt_ah_attr.static_rate = cmd.alt_dest.static_rate; attr->alt_ah_attr.ah_flags = cmd.alt_dest.is_global ? IB_AH_GRH : 0; attr->alt_ah_attr.port_num = cmd.alt_dest.port_num; err = xrcd->device->modify_xrc_rcv_qp(xrcd, cmd.qp_num, attr, cmd.attr_mask); put_xrcd_read(uobj); kfree(attr); return err ? err : in_len; } ssize_t ib_uverbs_query_xrc_rcv_qp(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_query_xrc_rcv_qp cmd; struct ib_uverbs_query_qp_resp resp; struct ib_qp_attr *attr; struct ib_qp_init_attr *init_attr; struct ib_xrcd *xrcd; struct ib_uobject *uobj; int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; attr = kmalloc(sizeof *attr, GFP_KERNEL); init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); if (!attr || !init_attr) { ret = -ENOMEM; goto out; } xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); if (!xrcd) { ret = -EINVAL; goto out; } ret = xrcd->device->query_xrc_rcv_qp(xrcd, cmd.qp_num, attr, cmd.attr_mask, init_attr); put_xrcd_read(uobj); if (ret) goto out; memset(&resp, 0, sizeof resp); resp.qp_state = attr->qp_state; resp.cur_qp_state = attr->cur_qp_state; resp.path_mtu = attr->path_mtu; resp.path_mig_state = attr->path_mig_state; resp.qkey = attr->qkey; resp.rq_psn = attr->rq_psn; resp.sq_psn = attr->sq_psn; resp.dest_qp_num = attr->dest_qp_num; resp.qp_access_flags = attr->qp_access_flags; resp.pkey_index = attr->pkey_index; resp.alt_pkey_index = attr->alt_pkey_index; resp.sq_draining = attr->sq_draining; resp.max_rd_atomic = attr->max_rd_atomic; resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; resp.min_rnr_timer = attr->min_rnr_timer; resp.port_num = attr->port_num; resp.timeout = attr->timeout; resp.retry_cnt = attr->retry_cnt; resp.rnr_retry = attr->rnr_retry; resp.alt_port_num = attr->alt_port_num; resp.alt_timeout = attr->alt_timeout; memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); resp.dest.flow_label = attr->ah_attr.grh.flow_label; resp.dest.sgid_index = attr->ah_attr.grh.sgid_index; resp.dest.hop_limit = attr->ah_attr.grh.hop_limit; resp.dest.traffic_class = attr->ah_attr.grh.traffic_class; resp.dest.dlid = attr->ah_attr.dlid; resp.dest.sl = attr->ah_attr.sl; resp.dest.src_path_bits = attr->ah_attr.src_path_bits; resp.dest.static_rate = attr->ah_attr.static_rate; resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH); resp.dest.port_num = attr->ah_attr.port_num; memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; resp.alt_dest.dlid = attr->alt_ah_attr.dlid; resp.alt_dest.sl = attr->alt_ah_attr.sl; resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate; resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH); resp.alt_dest.port_num = attr->alt_ah_attr.port_num; resp.max_send_wr = init_attr->cap.max_send_wr; resp.max_recv_wr = init_attr->cap.max_recv_wr; resp.max_send_sge = init_attr->cap.max_send_sge; resp.max_recv_sge = init_attr->cap.max_recv_sge; resp.max_inline_data = init_attr->cap.max_inline_data; resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) ret = -EFAULT; out: kfree(attr); kfree(init_attr); return ret ? ret : in_len; } ssize_t ib_uverbs_reg_xrc_rcv_qp(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_reg_xrc_rcv_qp cmd; struct ib_uxrc_rcv_object *qp_obj, *tmp; struct ib_xrcd *xrcd; struct ib_uobject *uobj; struct ib_uxrcd_object *xrcd_uobj; int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; qp_obj = kmalloc(sizeof *qp_obj, GFP_KERNEL); if (!qp_obj) return -ENOMEM; xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); if (!xrcd) { ret = -EINVAL; goto err_out; } ret = xrcd->device->reg_xrc_rcv_qp(xrcd, file, cmd.qp_num); if (ret) goto err_put; xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); mutex_lock(&file->device->ib_dev->xrcd_table_mutex); list_for_each_entry(tmp, &xrcd_uobj->xrc_reg_qp_list, list) if (cmd.qp_num == tmp->qp_num) { kfree(qp_obj); mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); put_xrcd_read(uobj); return in_len; } qp_obj->qp_num = cmd.qp_num; qp_obj->domain_handle = cmd.xrc_domain_handle; list_add_tail(&qp_obj->list, &xrcd_uobj->xrc_reg_qp_list); mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); atomic_inc(&xrcd->usecnt); put_xrcd_read(uobj); return in_len; err_put: put_xrcd_read(uobj); err_out: kfree(qp_obj); return ret; } int ib_uverbs_cleanup_xrc_rcv_qp(struct ib_uverbs_file *file, struct ib_xrcd *xrcd, u32 qp_num) { int err; err = xrcd->device->unreg_xrc_rcv_qp(xrcd, file, qp_num); if (!err) atomic_dec(&xrcd->usecnt); return err; } ssize_t ib_uverbs_unreg_xrc_rcv_qp(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_unreg_xrc_rcv_qp cmd; struct ib_uxrc_rcv_object *qp_obj, *tmp; struct ib_xrcd *xrcd; struct ib_uobject *uobj; struct ib_uxrcd_object *xrcd_uobj; int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); if (!xrcd) return -EINVAL; ret = xrcd->device->unreg_xrc_rcv_qp(xrcd, file, cmd.qp_num); if (ret) { put_xrcd_read(uobj); return -EINVAL; } atomic_dec(&xrcd->usecnt); xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); mutex_lock(&file->device->ib_dev->xrcd_table_mutex); list_for_each_entry_safe(qp_obj, tmp, &xrcd_uobj->xrc_reg_qp_list, list) if (cmd.qp_num == qp_obj->qp_num) { list_del(&qp_obj->list); kfree(qp_obj); break; } mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); put_xrcd_read(uobj); return in_len; } Index: stable/9/sys/ofed/drivers/infiniband/core/uverbs_main.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/uverbs_main.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/uverbs_main.c (revision 273864) @@ -1,1012 +1,1014 @@ /* * Copyright (c) 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. * Copyright (c) 2005 Mellanox Technologies. All rights reserved. * Copyright (c) 2005 Voltaire, Inc. All rights reserved. * Copyright (c) 2005 PathScale, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include -#include #include #include #include #include #include #include #include #include #include "uverbs.h" MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("InfiniBand userspace verbs access"); MODULE_LICENSE("Dual BSD/GPL"); #define INFINIBANDEVENTFS_MAGIC 0x49426576 /* "IBev" */ enum { IB_UVERBS_MAJOR = 231, IB_UVERBS_BASE_MINOR = 192, IB_UVERBS_MAX_DEVICES = 32 }; #define IB_UVERBS_BASE_DEV MKDEV(IB_UVERBS_MAJOR, IB_UVERBS_BASE_MINOR) static struct class *uverbs_class; DEFINE_SPINLOCK(ib_uverbs_idr_lock); DEFINE_IDR(ib_uverbs_pd_idr); DEFINE_IDR(ib_uverbs_mr_idr); DEFINE_IDR(ib_uverbs_mw_idr); DEFINE_IDR(ib_uverbs_ah_idr); DEFINE_IDR(ib_uverbs_cq_idr); DEFINE_IDR(ib_uverbs_qp_idr); DEFINE_IDR(ib_uverbs_srq_idr); DEFINE_IDR(ib_uverbs_xrc_domain_idr); static spinlock_t map_lock; static struct ib_uverbs_device *dev_table[IB_UVERBS_MAX_DEVICES]; static DECLARE_BITMAP(dev_map, IB_UVERBS_MAX_DEVICES); static ssize_t (*uverbs_cmd_table[])(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) = { [IB_USER_VERBS_CMD_GET_CONTEXT] = ib_uverbs_get_context, [IB_USER_VERBS_CMD_QUERY_DEVICE] = ib_uverbs_query_device, [IB_USER_VERBS_CMD_QUERY_PORT] = ib_uverbs_query_port, [IB_USER_VERBS_CMD_ALLOC_PD] = ib_uverbs_alloc_pd, [IB_USER_VERBS_CMD_DEALLOC_PD] = ib_uverbs_dealloc_pd, [IB_USER_VERBS_CMD_REG_MR] = ib_uverbs_reg_mr, [IB_USER_VERBS_CMD_DEREG_MR] = ib_uverbs_dereg_mr, [IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL] = ib_uverbs_create_comp_channel, [IB_USER_VERBS_CMD_CREATE_CQ] = ib_uverbs_create_cq, [IB_USER_VERBS_CMD_RESIZE_CQ] = ib_uverbs_resize_cq, [IB_USER_VERBS_CMD_POLL_CQ] = ib_uverbs_poll_cq, [IB_USER_VERBS_CMD_REQ_NOTIFY_CQ] = ib_uverbs_req_notify_cq, [IB_USER_VERBS_CMD_DESTROY_CQ] = ib_uverbs_destroy_cq, [IB_USER_VERBS_CMD_CREATE_QP] = ib_uverbs_create_qp, [IB_USER_VERBS_CMD_QUERY_QP] = ib_uverbs_query_qp, [IB_USER_VERBS_CMD_MODIFY_QP] = ib_uverbs_modify_qp, [IB_USER_VERBS_CMD_DESTROY_QP] = ib_uverbs_destroy_qp, [IB_USER_VERBS_CMD_POST_SEND] = ib_uverbs_post_send, [IB_USER_VERBS_CMD_POST_RECV] = ib_uverbs_post_recv, [IB_USER_VERBS_CMD_POST_SRQ_RECV] = ib_uverbs_post_srq_recv, [IB_USER_VERBS_CMD_CREATE_AH] = ib_uverbs_create_ah, [IB_USER_VERBS_CMD_DESTROY_AH] = ib_uverbs_destroy_ah, [IB_USER_VERBS_CMD_ATTACH_MCAST] = ib_uverbs_attach_mcast, [IB_USER_VERBS_CMD_DETACH_MCAST] = ib_uverbs_detach_mcast, [IB_USER_VERBS_CMD_CREATE_SRQ] = ib_uverbs_create_srq, [IB_USER_VERBS_CMD_MODIFY_SRQ] = ib_uverbs_modify_srq, [IB_USER_VERBS_CMD_QUERY_SRQ] = ib_uverbs_query_srq, [IB_USER_VERBS_CMD_DESTROY_SRQ] = ib_uverbs_destroy_srq, [IB_USER_VERBS_CMD_CREATE_XRC_SRQ] = ib_uverbs_create_xrc_srq, - [IB_USER_VERBS_CMD_OPEN_XRC_DOMAIN] = ib_uverbs_open_xrc_domain, - [IB_USER_VERBS_CMD_CLOSE_XRC_DOMAIN] = ib_uverbs_close_xrc_domain, + [IB_USER_VERBS_CMD_OPEN_XRCD] = ib_uverbs_open_xrc_domain, + [IB_USER_VERBS_CMD_CLOSE_XRCD] = ib_uverbs_close_xrc_domain, [IB_USER_VERBS_CMD_CREATE_XRC_RCV_QP] = ib_uverbs_create_xrc_rcv_qp, [IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP] = ib_uverbs_modify_xrc_rcv_qp, [IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP] = ib_uverbs_query_xrc_rcv_qp, [IB_USER_VERBS_CMD_REG_XRC_RCV_QP] = ib_uverbs_reg_xrc_rcv_qp, [IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP] = ib_uverbs_unreg_xrc_rcv_qp, }; #ifdef __linux__ /* BSD Does not require a fake mountpoint for all files. */ static struct vfsmount *uverbs_event_mnt; #endif static void ib_uverbs_add_one(struct ib_device *device); static void ib_uverbs_remove_one(struct ib_device *device); static void ib_uverbs_release_dev(struct kref *ref) { struct ib_uverbs_device *dev = container_of(ref, struct ib_uverbs_device, ref); complete(&dev->comp); } static void ib_uverbs_release_event_file(struct kref *ref) { struct ib_uverbs_event_file *file = container_of(ref, struct ib_uverbs_event_file, ref); kfree(file); } void ib_uverbs_release_ucq(struct ib_uverbs_file *file, struct ib_uverbs_event_file *ev_file, struct ib_ucq_object *uobj) { struct ib_uverbs_event *evt, *tmp; if (ev_file) { spin_lock_irq(&ev_file->lock); list_for_each_entry_safe(evt, tmp, &uobj->comp_list, obj_list) { list_del(&evt->list); kfree(evt); } spin_unlock_irq(&ev_file->lock); kref_put(&ev_file->ref, ib_uverbs_release_event_file); } spin_lock_irq(&file->async_file->lock); list_for_each_entry_safe(evt, tmp, &uobj->async_list, obj_list) { list_del(&evt->list); kfree(evt); } spin_unlock_irq(&file->async_file->lock); } void ib_uverbs_release_uevent(struct ib_uverbs_file *file, struct ib_uevent_object *uobj) { struct ib_uverbs_event *evt, *tmp; spin_lock_irq(&file->async_file->lock); list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) { list_del(&evt->list); kfree(evt); } spin_unlock_irq(&file->async_file->lock); } static void ib_uverbs_detach_umcast(struct ib_qp *qp, struct ib_uqp_object *uobj) { struct ib_uverbs_mcast_entry *mcast, *tmp; list_for_each_entry_safe(mcast, tmp, &uobj->mcast_list, list) { ib_detach_mcast(qp, &mcast->gid, mcast->lid); list_del(&mcast->list); kfree(mcast); } } static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file, struct ib_ucontext *context) { struct ib_uobject *uobj, *tmp; if (!context) return 0; context->closing = 1; list_for_each_entry_safe(uobj, tmp, &context->ah_list, list) { struct ib_ah *ah = uobj->object; idr_remove_uobj(&ib_uverbs_ah_idr, uobj); ib_destroy_ah(ah); kfree(uobj); } list_for_each_entry_safe(uobj, tmp, &context->qp_list, list) { struct ib_qp *qp = uobj->object; struct ib_uqp_object *uqp = container_of(uobj, struct ib_uqp_object, uevent.uobject); idr_remove_uobj(&ib_uverbs_qp_idr, uobj); ib_uverbs_detach_umcast(qp, uqp); ib_destroy_qp(qp); ib_uverbs_release_uevent(file, &uqp->uevent); kfree(uqp); } list_for_each_entry_safe(uobj, tmp, &context->srq_list, list) { struct ib_srq *srq = uobj->object; struct ib_uevent_object *uevent = container_of(uobj, struct ib_uevent_object, uobject); idr_remove_uobj(&ib_uverbs_srq_idr, uobj); ib_destroy_srq(srq); ib_uverbs_release_uevent(file, uevent); kfree(uevent); } list_for_each_entry_safe(uobj, tmp, &context->cq_list, list) { struct ib_cq *cq = uobj->object; struct ib_uverbs_event_file *ev_file = cq->cq_context; struct ib_ucq_object *ucq = container_of(uobj, struct ib_ucq_object, uobject); idr_remove_uobj(&ib_uverbs_cq_idr, uobj); ib_destroy_cq(cq); ib_uverbs_release_ucq(file, ev_file, ucq); kfree(ucq); } /* XXX Free MWs */ list_for_each_entry_safe(uobj, tmp, &context->mr_list, list) { struct ib_mr *mr = uobj->object; idr_remove_uobj(&ib_uverbs_mr_idr, uobj); ib_dereg_mr(mr); kfree(uobj); } mutex_lock(&file->device->ib_dev->xrcd_table_mutex); - list_for_each_entry_safe(uobj, tmp, &context->xrc_domain_list, list) { + list_for_each_entry_safe(uobj, tmp, &context->xrcd_list, list) { struct ib_xrcd *xrcd = uobj->object; struct ib_uxrc_rcv_object *xrc_qp_obj, *tmp1; struct ib_uxrcd_object *xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); list_for_each_entry_safe(xrc_qp_obj, tmp1, &xrcd_uobj->xrc_reg_qp_list, list) { list_del(&xrc_qp_obj->list); ib_uverbs_cleanup_xrc_rcv_qp(file, xrcd, xrc_qp_obj->qp_num); kfree(xrc_qp_obj); } idr_remove_uobj(&ib_uverbs_xrc_domain_idr, uobj); ib_uverbs_dealloc_xrcd(file->device->ib_dev, xrcd); kfree(uobj); } mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); list_for_each_entry_safe(uobj, tmp, &context->pd_list, list) { struct ib_pd *pd = uobj->object; idr_remove_uobj(&ib_uverbs_pd_idr, uobj); ib_dealloc_pd(pd); kfree(uobj); } return context->device->dealloc_ucontext(context); } static void ib_uverbs_release_file(struct kref *ref) { struct ib_uverbs_file *file = container_of(ref, struct ib_uverbs_file, ref); module_put(file->device->ib_dev->owner); kref_put(&file->device->ref, ib_uverbs_release_dev); kfree(file); } static ssize_t ib_uverbs_event_read(struct file *filp, char __user *buf, size_t count, loff_t *pos) { struct ib_uverbs_event_file *file = filp->private_data; struct ib_uverbs_event *event; int eventsz; int ret = 0; spin_lock_irq(&file->lock); while (list_empty(&file->event_list)) { spin_unlock_irq(&file->lock); if (filp->f_flags & O_NONBLOCK) return -EAGAIN; if (wait_event_interruptible(file->poll_wait, !list_empty(&file->event_list))) return -ERESTARTSYS; spin_lock_irq(&file->lock); } event = list_entry(file->event_list.next, struct ib_uverbs_event, list); if (file->is_async) eventsz = sizeof (struct ib_uverbs_async_event_desc); else eventsz = sizeof (struct ib_uverbs_comp_event_desc); if (eventsz > count) { ret = -EINVAL; event = NULL; } else { list_del(file->event_list.next); if (event->counter) { ++(*event->counter); list_del(&event->obj_list); } } spin_unlock_irq(&file->lock); if (event) { if (copy_to_user(buf, event, eventsz)) ret = -EFAULT; else ret = eventsz; } kfree(event); return ret; } static unsigned int ib_uverbs_event_poll(struct file *filp, struct poll_table_struct *wait) { unsigned int pollflags = 0; struct ib_uverbs_event_file *file = filp->private_data; file->filp = filp; poll_wait(filp, &file->poll_wait, wait); spin_lock_irq(&file->lock); if (!list_empty(&file->event_list)) pollflags = POLLIN | POLLRDNORM; spin_unlock_irq(&file->lock); return pollflags; } static int ib_uverbs_event_fasync(int fd, struct file *filp, int on) { struct ib_uverbs_event_file *file = filp->private_data; return fasync_helper(fd, filp, on, &file->async_queue); } static int ib_uverbs_event_close(struct inode *inode, struct file *filp) { struct ib_uverbs_event_file *file = filp->private_data; struct ib_uverbs_event *entry, *tmp; spin_lock_irq(&file->lock); file->is_closed = 1; list_for_each_entry_safe(entry, tmp, &file->event_list, list) { if (entry->counter) list_del(&entry->obj_list); kfree(entry); } spin_unlock_irq(&file->lock); if (file->is_async) { ib_unregister_event_handler(&file->uverbs_file->event_handler); kref_put(&file->uverbs_file->ref, ib_uverbs_release_file); } kref_put(&file->ref, ib_uverbs_release_event_file); return 0; } static const struct file_operations uverbs_event_fops = { .owner = THIS_MODULE, .read = ib_uverbs_event_read, .poll = ib_uverbs_event_poll, .release = ib_uverbs_event_close, .fasync = ib_uverbs_event_fasync }; void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context) { struct ib_uverbs_event_file *file = cq_context; struct ib_ucq_object *uobj; struct ib_uverbs_event *entry; unsigned long flags; if (!file) return; spin_lock_irqsave(&file->lock, flags); if (file->is_closed) { spin_unlock_irqrestore(&file->lock, flags); return; } entry = kmalloc(sizeof *entry, GFP_ATOMIC); if (!entry) { spin_unlock_irqrestore(&file->lock, flags); return; } uobj = container_of(cq->uobject, struct ib_ucq_object, uobject); entry->desc.comp.cq_handle = cq->uobject->user_handle; entry->counter = &uobj->comp_events_reported; list_add_tail(&entry->list, &file->event_list); list_add_tail(&entry->obj_list, &uobj->comp_list); spin_unlock_irqrestore(&file->lock, flags); wake_up_interruptible(&file->poll_wait); if (file->filp) selwakeup(&file->filp->f_selinfo); kill_fasync(&file->async_queue, SIGIO, POLL_IN); } static void ib_uverbs_async_handler(struct ib_uverbs_file *file, __u64 element, __u64 event, struct list_head *obj_list, u32 *counter) { struct ib_uverbs_event *entry; unsigned long flags; spin_lock_irqsave(&file->async_file->lock, flags); if (file->async_file->is_closed) { spin_unlock_irqrestore(&file->async_file->lock, flags); return; } entry = kmalloc(sizeof *entry, GFP_ATOMIC); if (!entry) { spin_unlock_irqrestore(&file->async_file->lock, flags); return; } entry->desc.async.element = element; entry->desc.async.event_type = event; entry->counter = counter; list_add_tail(&entry->list, &file->async_file->event_list); if (obj_list) list_add_tail(&entry->obj_list, obj_list); spin_unlock_irqrestore(&file->async_file->lock, flags); wake_up_interruptible(&file->async_file->poll_wait); if (file->async_file->filp) selwakeup(&file->async_file->filp->f_selinfo); kill_fasync(&file->async_file->async_queue, SIGIO, POLL_IN); } void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr) { struct ib_ucq_object *uobj = container_of(event->element.cq->uobject, struct ib_ucq_object, uobject); ib_uverbs_async_handler(uobj->uverbs_file, uobj->uobject.user_handle, event->event, &uobj->async_list, &uobj->async_events_reported); } void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr) { struct ib_uevent_object *uobj; uobj = container_of(event->element.qp->uobject, struct ib_uevent_object, uobject); ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, event->event, &uobj->event_list, &uobj->events_reported); } void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr) { struct ib_uevent_object *uobj; uobj = container_of(event->element.srq->uobject, struct ib_uevent_object, uobject); ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, event->event, &uobj->event_list, &uobj->events_reported); } void ib_uverbs_event_handler(struct ib_event_handler *handler, struct ib_event *event) { struct ib_uverbs_file *file = container_of(handler, struct ib_uverbs_file, event_handler); ib_uverbs_async_handler(file, event->element.port_num, event->event, NULL, NULL); } void ib_uverbs_xrc_rcv_qp_event_handler(struct ib_event *event, void *context_ptr) { ib_uverbs_async_handler(context_ptr, event->element.xrc_qp_num, event->event, NULL, NULL); } struct file *ib_uverbs_alloc_event_file(struct ib_uverbs_file *uverbs_file, int is_async, int *fd) { struct ib_uverbs_event_file *ev_file; struct file *filp; int ret; ev_file = kmalloc(sizeof *ev_file, GFP_KERNEL); if (!ev_file) return ERR_PTR(-ENOMEM); kref_init(&ev_file->ref); spin_lock_init(&ev_file->lock); INIT_LIST_HEAD(&ev_file->event_list); init_waitqueue_head(&ev_file->poll_wait); ev_file->uverbs_file = uverbs_file; ev_file->async_queue = NULL; ev_file->is_async = is_async; ev_file->is_closed = 0; ev_file->filp = NULL; *fd = get_unused_fd(); if (*fd < 0) { ret = *fd; goto err; } /* * fops_get() can't fail here, because we're coming from a * system call on a uverbs file, which will already have a * module reference. */ +#ifdef __linux__ filp = alloc_file(uverbs_event_mnt, dget(uverbs_event_mnt->mnt_root), FMODE_READ, fops_get(&uverbs_event_fops)); +#else + filp = alloc_file(FMODE_READ, fops_get(&uverbs_event_fops)); +#endif if (!filp) { ret = -ENFILE; goto err_fd; } filp->private_data = ev_file; return filp; err_fd: put_unused_fd(*fd); err: kfree(ev_file); return ERR_PTR(ret); } /* * Look up a completion event file by FD. If lookup is successful, * takes a ref to the event file struct that it returns; if * unsuccessful, returns NULL. */ struct ib_uverbs_event_file *ib_uverbs_lookup_comp_file(int fd) { struct ib_uverbs_event_file *ev_file = NULL; struct file *filp; filp = fget(fd); if (!filp) return NULL; if (filp->f_op != &uverbs_event_fops) goto out; ev_file = filp->private_data; if (ev_file->is_async) { ev_file = NULL; goto out; } kref_get(&ev_file->ref); out: fput(filp); return ev_file; } static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf, size_t count, loff_t *pos) { struct ib_uverbs_file *file = filp->private_data; struct ib_uverbs_cmd_hdr hdr; if (count < sizeof hdr) return -EINVAL; if (copy_from_user(&hdr, buf, sizeof hdr)) return -EFAULT; if (hdr.in_words * 4 != count) return -EINVAL; - if (hdr.command < 0 || - hdr.command >= ARRAY_SIZE(uverbs_cmd_table) || + if (hdr.command >= ARRAY_SIZE(uverbs_cmd_table) || !uverbs_cmd_table[hdr.command] || !(file->device->ib_dev->uverbs_cmd_mask & (1ull << hdr.command))) return -EINVAL; if (!file->ucontext && hdr.command != IB_USER_VERBS_CMD_GET_CONTEXT) return -EINVAL; return uverbs_cmd_table[hdr.command](file, buf + sizeof hdr, hdr.in_words * 4, hdr.out_words * 4); } static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma) { struct ib_uverbs_file *file = filp->private_data; if (!file->ucontext) return -ENODEV; else return file->device->ib_dev->mmap(file->ucontext, vma); } /* * ib_uverbs_open() does not need the BKL: * * - dev_table[] accesses are protected by map_lock, the * ib_uverbs_device structures are properly reference counted, and * everything else is purely local to the file being created, so * races against other open calls are not a problem; * - there is no ioctl method to race against; * - the device is added to dev_table[] as the last part of module * initialization, the open method will either immediately run * -ENXIO, or all required initialization will be done. */ static int ib_uverbs_open(struct inode *inode, struct file *filp) { struct ib_uverbs_device *dev; struct ib_uverbs_file *file; int ret; spin_lock(&map_lock); dev = dev_table[iminor(inode) - IB_UVERBS_BASE_MINOR]; if (dev) kref_get(&dev->ref); spin_unlock(&map_lock); if (!dev) return -ENXIO; if (!try_module_get(dev->ib_dev->owner)) { ret = -ENODEV; goto err; } file = kmalloc(sizeof *file, GFP_KERNEL); if (!file) { ret = -ENOMEM; goto err_module; } file->device = dev; file->ucontext = NULL; file->async_file = NULL; kref_init(&file->ref); mutex_init(&file->mutex); filp->private_data = file; return 0; err_module: module_put(dev->ib_dev->owner); err: kref_put(&dev->ref, ib_uverbs_release_dev); return ret; } static int ib_uverbs_close(struct inode *inode, struct file *filp) { struct ib_uverbs_file *file = filp->private_data; ib_uverbs_cleanup_ucontext(file, file->ucontext); if (file->async_file) kref_put(&file->async_file->ref, ib_uverbs_release_event_file); kref_put(&file->ref, ib_uverbs_release_file); return 0; } static const struct file_operations uverbs_fops = { .owner = THIS_MODULE, .write = ib_uverbs_write, .open = ib_uverbs_open, .release = ib_uverbs_close }; static const struct file_operations uverbs_mmap_fops = { .owner = THIS_MODULE, .write = ib_uverbs_write, .mmap = ib_uverbs_mmap, .open = ib_uverbs_open, .release = ib_uverbs_close }; static struct ib_client uverbs_client = { .name = "uverbs", .add = ib_uverbs_add_one, .remove = ib_uverbs_remove_one }; static ssize_t show_ibdev(struct device *device, struct device_attribute *attr, char *buf) { struct ib_uverbs_device *dev = dev_get_drvdata(device); if (!dev) return -ENODEV; return sprintf(buf, "%s\n", dev->ib_dev->name); } static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL); static ssize_t show_dev_abi_version(struct device *device, struct device_attribute *attr, char *buf) { struct ib_uverbs_device *dev = dev_get_drvdata(device); if (!dev) return -ENODEV; return sprintf(buf, "%d\n", dev->ib_dev->uverbs_abi_ver); } static DEVICE_ATTR(abi_version, S_IRUGO, show_dev_abi_version, NULL); -static ssize_t show_abi_version(struct class *class, char *buf) +static ssize_t show_abi_version(struct class *class, struct class_attribute *attr, char *buf) { return sprintf(buf, "%d\n", IB_USER_VERBS_ABI_VERSION); } static CLASS_ATTR(abi_version, S_IRUGO, show_abi_version, NULL); #include static ssize_t show_dev_device(struct device *device, struct device_attribute *attr, char *buf) { struct ib_uverbs_device *dev = dev_get_drvdata(device); if (!dev) return -ENODEV; return sprintf(buf, "0x%04x\n", ((struct pci_dev *)dev->ib_dev->dma_device)->device); } static DEVICE_ATTR(device, S_IRUGO, show_dev_device, NULL); static ssize_t show_dev_vendor(struct device *device, struct device_attribute *attr, char *buf) { struct ib_uverbs_device *dev = dev_get_drvdata(device); if (!dev) return -ENODEV; return sprintf(buf, "0x%04x\n", ((struct pci_dev *)dev->ib_dev->dma_device)->vendor); } static DEVICE_ATTR(vendor, S_IRUGO, show_dev_vendor, NULL); struct attribute *device_attrs[] = { &dev_attr_device.attr, &dev_attr_vendor.attr, NULL }; static struct attribute_group device_group = { .name = "device", .attrs = device_attrs }; static void ib_uverbs_add_one(struct ib_device *device) { struct ib_uverbs_device *uverbs_dev; if (!device->alloc_ucontext) return; uverbs_dev = kzalloc(sizeof *uverbs_dev, GFP_KERNEL); if (!uverbs_dev) return; kref_init(&uverbs_dev->ref); init_completion(&uverbs_dev->comp); spin_lock(&map_lock); uverbs_dev->devnum = find_first_zero_bit(dev_map, IB_UVERBS_MAX_DEVICES); if (uverbs_dev->devnum >= IB_UVERBS_MAX_DEVICES) { spin_unlock(&map_lock); goto err; } set_bit(uverbs_dev->devnum, dev_map); spin_unlock(&map_lock); uverbs_dev->ib_dev = device; uverbs_dev->num_comp_vectors = device->num_comp_vectors; uverbs_dev->cdev = cdev_alloc(); if (!uverbs_dev->cdev) goto err; uverbs_dev->cdev->owner = THIS_MODULE; uverbs_dev->cdev->ops = device->mmap ? &uverbs_mmap_fops : &uverbs_fops; kobject_set_name(&uverbs_dev->cdev->kobj, "uverbs%d", uverbs_dev->devnum); if (cdev_add(uverbs_dev->cdev, IB_UVERBS_BASE_DEV + uverbs_dev->devnum, 1)) goto err_cdev; uverbs_dev->dev = device_create(uverbs_class, device->dma_device, uverbs_dev->cdev->dev, uverbs_dev, "uverbs%d", uverbs_dev->devnum); if (IS_ERR(uverbs_dev->dev)) goto err_cdev; if (device_create_file(uverbs_dev->dev, &dev_attr_ibdev)) goto err_class; if (device_create_file(uverbs_dev->dev, &dev_attr_abi_version)) goto err_class; if (sysfs_create_group(&uverbs_dev->dev->kobj, &device_group)) goto err_class; spin_lock(&map_lock); dev_table[uverbs_dev->devnum] = uverbs_dev; spin_unlock(&map_lock); ib_set_client_data(device, &uverbs_client, uverbs_dev); return; err_class: device_destroy(uverbs_class, uverbs_dev->cdev->dev); err_cdev: cdev_del(uverbs_dev->cdev); clear_bit(uverbs_dev->devnum, dev_map); err: kref_put(&uverbs_dev->ref, ib_uverbs_release_dev); wait_for_completion(&uverbs_dev->comp); kfree(uverbs_dev); return; } static void ib_uverbs_remove_one(struct ib_device *device) { struct ib_uverbs_device *uverbs_dev = ib_get_client_data(device, &uverbs_client); if (!uverbs_dev) return; sysfs_remove_group(&uverbs_dev->dev->kobj, &device_group); dev_set_drvdata(uverbs_dev->dev, NULL); device_destroy(uverbs_class, uverbs_dev->cdev->dev); cdev_del(uverbs_dev->cdev); spin_lock(&map_lock); dev_table[uverbs_dev->devnum] = NULL; spin_unlock(&map_lock); clear_bit(uverbs_dev->devnum, dev_map); kref_put(&uverbs_dev->ref, ib_uverbs_release_dev); wait_for_completion(&uverbs_dev->comp); kfree(uverbs_dev); } #ifdef __linux__ static int uverbs_event_get_sb(struct file_system_type *fs_type, int flags, const char *dev_name, void *data, struct vfsmount *mnt) { return get_sb_pseudo(fs_type, "infinibandevent:", NULL, INFINIBANDEVENTFS_MAGIC, mnt); } static struct file_system_type uverbs_event_fs = { /* No owner field so module can be unloaded */ .name = "infinibandeventfs", .get_sb = uverbs_event_get_sb, .kill_sb = kill_litter_super }; #endif static int __init ib_uverbs_init(void) { int ret; spin_lock_init(&map_lock); ret = register_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES, "infiniband_verbs"); if (ret) { printk(KERN_ERR "user_verbs: couldn't register device number\n"); goto out; } uverbs_class = class_create(THIS_MODULE, "infiniband_verbs"); if (IS_ERR(uverbs_class)) { ret = PTR_ERR(uverbs_class); printk(KERN_ERR "user_verbs: couldn't create class infiniband_verbs\n"); goto out_chrdev; } ret = class_create_file(uverbs_class, &class_attr_abi_version); if (ret) { printk(KERN_ERR "user_verbs: couldn't create abi_version attribute\n"); goto out_class; } #ifdef __linux__ ret = register_filesystem(&uverbs_event_fs); if (ret) { printk(KERN_ERR "user_verbs: couldn't register infinibandeventfs\n"); goto out_class; } uverbs_event_mnt = kern_mount(&uverbs_event_fs); if (IS_ERR(uverbs_event_mnt)) { ret = PTR_ERR(uverbs_event_mnt); printk(KERN_ERR "user_verbs: couldn't mount infinibandeventfs\n"); goto out_fs; } #endif ret = ib_register_client(&uverbs_client); if (ret) { printk(KERN_ERR "user_verbs: couldn't register client\n"); goto out_mnt; } return 0; out_mnt: #ifdef __linux__ mntput(uverbs_event_mnt); out_fs: unregister_filesystem(&uverbs_event_fs); #endif out_class: class_destroy(uverbs_class); out_chrdev: unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES); out: return ret; } static void __exit ib_uverbs_cleanup(void) { ib_unregister_client(&uverbs_client); #ifdef __linux__ mntput(uverbs_event_mnt); unregister_filesystem(&uverbs_event_fs); #endif class_destroy(uverbs_class); unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES); idr_destroy(&ib_uverbs_pd_idr); idr_destroy(&ib_uverbs_mr_idr); idr_destroy(&ib_uverbs_mw_idr); idr_destroy(&ib_uverbs_ah_idr); idr_destroy(&ib_uverbs_cq_idr); idr_destroy(&ib_uverbs_qp_idr); idr_destroy(&ib_uverbs_srq_idr); } module_init(ib_uverbs_init); module_exit(ib_uverbs_cleanup); Index: stable/9/sys/ofed/drivers/infiniband/core/verbs.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/core/verbs.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/core/verbs.c (revision 273864) @@ -1,1073 +1,1073 @@ /* * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2004 Infinicon Corporation. All rights reserved. * Copyright (c) 2004 Intel Corporation. All rights reserved. * Copyright (c) 2004 Topspin Corporation. All rights reserved. * Copyright (c) 2004 Voltaire Corporation. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #include int ib_rate_to_mult(enum ib_rate rate) { switch (rate) { case IB_RATE_2_5_GBPS: return 1; case IB_RATE_5_GBPS: return 2; case IB_RATE_10_GBPS: return 4; case IB_RATE_20_GBPS: return 8; case IB_RATE_30_GBPS: return 12; case IB_RATE_40_GBPS: return 16; case IB_RATE_60_GBPS: return 24; case IB_RATE_80_GBPS: return 32; case IB_RATE_120_GBPS: return 48; default: return -1; } } EXPORT_SYMBOL(ib_rate_to_mult); enum ib_rate mult_to_ib_rate(int mult) { switch (mult) { case 1: return IB_RATE_2_5_GBPS; case 2: return IB_RATE_5_GBPS; case 4: return IB_RATE_10_GBPS; case 8: return IB_RATE_20_GBPS; case 12: return IB_RATE_30_GBPS; case 16: return IB_RATE_40_GBPS; case 24: return IB_RATE_60_GBPS; case 32: return IB_RATE_80_GBPS; case 48: return IB_RATE_120_GBPS; default: return IB_RATE_PORT_CURRENT; } } EXPORT_SYMBOL(mult_to_ib_rate); enum rdma_transport_type rdma_node_get_transport(enum rdma_node_type node_type) { switch (node_type) { case RDMA_NODE_IB_CA: case RDMA_NODE_IB_SWITCH: case RDMA_NODE_IB_ROUTER: return RDMA_TRANSPORT_IB; case RDMA_NODE_RNIC: return RDMA_TRANSPORT_IWARP; default: BUG(); return 0; } } EXPORT_SYMBOL(rdma_node_get_transport); enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num) { if (device->get_link_layer) return device->get_link_layer(device, port_num); switch (rdma_node_get_transport(device->node_type)) { case RDMA_TRANSPORT_IB: return IB_LINK_LAYER_INFINIBAND; case RDMA_TRANSPORT_IWARP: return IB_LINK_LAYER_ETHERNET; default: return IB_LINK_LAYER_UNSPECIFIED; } } EXPORT_SYMBOL(rdma_port_get_link_layer); /* Protection domains */ struct ib_pd *ib_alloc_pd(struct ib_device *device) { struct ib_pd *pd; pd = device->alloc_pd(device, NULL, NULL); if (!IS_ERR(pd)) { pd->device = device; pd->uobject = NULL; atomic_set(&pd->usecnt, 0); } return pd; } EXPORT_SYMBOL(ib_alloc_pd); int ib_dealloc_pd(struct ib_pd *pd) { if (atomic_read(&pd->usecnt)) return -EBUSY; return pd->device->dealloc_pd(pd); } EXPORT_SYMBOL(ib_dealloc_pd); /* Address handles */ struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) { struct ib_ah *ah; ah = pd->device->create_ah(pd, ah_attr); if (!IS_ERR(ah)) { ah->device = pd->device; ah->pd = pd; ah->uobject = NULL; atomic_inc(&pd->usecnt); } return ah; } EXPORT_SYMBOL(ib_create_ah); int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc, struct ib_grh *grh, struct ib_ah_attr *ah_attr) { u32 flow_class; u16 gid_index; int ret; memset(ah_attr, 0, sizeof *ah_attr); ah_attr->dlid = wc->slid; ah_attr->sl = wc->sl; ah_attr->src_path_bits = wc->dlid_path_bits; ah_attr->port_num = port_num; if (wc->wc_flags & IB_WC_GRH) { ah_attr->ah_flags = IB_AH_GRH; ah_attr->grh.dgid = grh->sgid; ret = ib_find_cached_gid(device, &grh->dgid, &port_num, &gid_index); if (ret) return ret; ah_attr->grh.sgid_index = (u8) gid_index; flow_class = be32_to_cpu(grh->version_tclass_flow); ah_attr->grh.flow_label = flow_class & 0xFFFFF; ah_attr->grh.hop_limit = 0xFF; ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF; } return 0; } EXPORT_SYMBOL(ib_init_ah_from_wc); struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc, struct ib_grh *grh, u8 port_num) { struct ib_ah_attr ah_attr; int ret; ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr); if (ret) return ERR_PTR(ret); return ib_create_ah(pd, &ah_attr); } EXPORT_SYMBOL(ib_create_ah_from_wc); int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr) { return ah->device->modify_ah ? ah->device->modify_ah(ah, ah_attr) : -ENOSYS; } EXPORT_SYMBOL(ib_modify_ah); int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr) { return ah->device->query_ah ? ah->device->query_ah(ah, ah_attr) : -ENOSYS; } EXPORT_SYMBOL(ib_query_ah); int ib_destroy_ah(struct ib_ah *ah) { struct ib_pd *pd; int ret; pd = ah->pd; ret = ah->device->destroy_ah(ah); if (!ret) atomic_dec(&pd->usecnt); return ret; } EXPORT_SYMBOL(ib_destroy_ah); /* Shared receive queues */ struct ib_srq *ib_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *srq_init_attr) { struct ib_srq *srq; if (!pd->device->create_srq) return ERR_PTR(-ENOSYS); srq = pd->device->create_srq(pd, srq_init_attr, NULL); if (!IS_ERR(srq)) { srq->device = pd->device; srq->pd = pd; srq->uobject = NULL; srq->event_handler = srq_init_attr->event_handler; srq->srq_context = srq_init_attr->srq_context; - srq->xrc_cq = NULL; - srq->xrcd = NULL; + srq->ext.xrc.cq = NULL; + srq->ext.xrc.xrcd = NULL; atomic_inc(&pd->usecnt); atomic_set(&srq->usecnt, 0); } return srq; } EXPORT_SYMBOL(ib_create_srq); struct ib_srq *ib_create_xrc_srq(struct ib_pd *pd, struct ib_cq *xrc_cq, struct ib_xrcd *xrcd, struct ib_srq_init_attr *srq_init_attr) { struct ib_srq *srq; if (!pd->device->create_xrc_srq) return ERR_PTR(-ENOSYS); srq = pd->device->create_xrc_srq(pd, xrc_cq, xrcd, srq_init_attr, NULL); if (!IS_ERR(srq)) { srq->device = pd->device; srq->pd = pd; srq->uobject = NULL; srq->event_handler = srq_init_attr->event_handler; srq->srq_context = srq_init_attr->srq_context; - srq->xrc_cq = xrc_cq; - srq->xrcd = xrcd; + srq->ext.xrc.cq = xrc_cq; + srq->ext.xrc.xrcd = xrcd; atomic_inc(&pd->usecnt); atomic_inc(&xrcd->usecnt); atomic_inc(&xrc_cq->usecnt); atomic_set(&srq->usecnt, 0); } return srq; } EXPORT_SYMBOL(ib_create_xrc_srq); int ib_modify_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr, enum ib_srq_attr_mask srq_attr_mask) { return srq->device->modify_srq ? srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) : -ENOSYS; } EXPORT_SYMBOL(ib_modify_srq); int ib_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr) { return srq->device->query_srq ? srq->device->query_srq(srq, srq_attr) : -ENOSYS; } EXPORT_SYMBOL(ib_query_srq); int ib_destroy_srq(struct ib_srq *srq) { struct ib_pd *pd; struct ib_cq *xrc_cq; struct ib_xrcd *xrcd; int ret; if (atomic_read(&srq->usecnt)) return -EBUSY; pd = srq->pd; - xrc_cq = srq->xrc_cq; - xrcd = srq->xrcd; + xrc_cq = srq->ext.xrc.cq; + xrcd = srq->ext.xrc.xrcd; ret = srq->device->destroy_srq(srq); if (!ret) { atomic_dec(&pd->usecnt); if (xrc_cq) atomic_dec(&xrc_cq->usecnt); if (xrcd) atomic_dec(&xrcd->usecnt); } return ret; } EXPORT_SYMBOL(ib_destroy_srq); /* Queue pairs */ struct ib_qp *ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *qp_init_attr) { struct ib_qp *qp; qp = pd->device->create_qp(pd, qp_init_attr, NULL); if (!IS_ERR(qp)) { qp->device = pd->device; qp->pd = pd; qp->send_cq = qp_init_attr->send_cq; qp->recv_cq = qp_init_attr->recv_cq; qp->srq = qp_init_attr->srq; qp->uobject = NULL; qp->event_handler = qp_init_attr->event_handler; qp->qp_context = qp_init_attr->qp_context; qp->qp_type = qp_init_attr->qp_type; qp->xrcd = qp->qp_type == IB_QPT_XRC ? - qp_init_attr->xrc_domain : NULL; + qp_init_attr->xrcd : NULL; atomic_inc(&pd->usecnt); atomic_inc(&qp_init_attr->send_cq->usecnt); atomic_inc(&qp_init_attr->recv_cq->usecnt); if (qp_init_attr->srq) atomic_inc(&qp_init_attr->srq->usecnt); if (qp->qp_type == IB_QPT_XRC) atomic_inc(&qp->xrcd->usecnt); } return qp; } EXPORT_SYMBOL(ib_create_qp); static const struct { int valid; - enum ib_qp_attr_mask req_param[IB_QPT_RAW_ETH + 1]; - enum ib_qp_attr_mask opt_param[IB_QPT_RAW_ETH + 1]; + enum ib_qp_attr_mask req_param[IB_QPT_RAW_PACKET + 1]; + enum ib_qp_attr_mask opt_param[IB_QPT_RAW_PACKET + 1]; } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = { [IB_QPS_RESET] = { [IB_QPS_RESET] = { .valid = 1 }, [IB_QPS_INIT] = { .valid = 1, .req_param = { [IB_QPT_UD] = (IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_QKEY), - [IB_QPT_RAW_ETH] = IB_QP_PORT, + [IB_QPT_RAW_PACKET] = IB_QP_PORT, [IB_QPT_UC] = (IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS), [IB_QPT_RC] = (IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS), [IB_QPT_XRC] = (IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS), [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY), [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY), } }, }, [IB_QPS_INIT] = { [IB_QPS_RESET] = { .valid = 1 }, [IB_QPS_ERR] = { .valid = 1 }, [IB_QPS_INIT] = { .valid = 1, .opt_param = { [IB_QPT_UD] = (IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_QKEY), [IB_QPT_UC] = (IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS), [IB_QPT_RC] = (IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS), [IB_QPT_XRC] = (IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS), [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY), [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY), } }, [IB_QPS_RTR] = { .valid = 1, .req_param = { [IB_QPT_UC] = (IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN | IB_QP_RQ_PSN), [IB_QPT_RC] = (IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN | IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER), [IB_QPT_XRC] = (IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN | IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER), }, .opt_param = { [IB_QPT_UD] = (IB_QP_PKEY_INDEX | IB_QP_QKEY), [IB_QPT_UC] = (IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX), [IB_QPT_RC] = (IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX), [IB_QPT_XRC] = (IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX), [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY), [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY), } } }, [IB_QPS_RTR] = { [IB_QPS_RESET] = { .valid = 1 }, [IB_QPS_ERR] = { .valid = 1 }, [IB_QPS_RTS] = { .valid = 1, .req_param = { [IB_QPT_UD] = IB_QP_SQ_PSN, [IB_QPT_UC] = IB_QP_SQ_PSN, [IB_QPT_RC] = (IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY | IB_QP_SQ_PSN | IB_QP_MAX_QP_RD_ATOMIC), [IB_QPT_XRC] = (IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY | IB_QP_SQ_PSN | IB_QP_MAX_QP_RD_ATOMIC), [IB_QPT_SMI] = IB_QP_SQ_PSN, [IB_QPT_GSI] = IB_QP_SQ_PSN, }, .opt_param = { [IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY), [IB_QPT_UC] = (IB_QP_CUR_STATE | IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_PATH_MIG_STATE), [IB_QPT_RC] = (IB_QP_CUR_STATE | IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_MIN_RNR_TIMER | IB_QP_PATH_MIG_STATE), [IB_QPT_XRC] = (IB_QP_CUR_STATE | IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_MIN_RNR_TIMER | IB_QP_PATH_MIG_STATE), [IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY), [IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY), } } }, [IB_QPS_RTS] = { [IB_QPS_RESET] = { .valid = 1 }, [IB_QPS_ERR] = { .valid = 1 }, [IB_QPS_RTS] = { .valid = 1, .opt_param = { [IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY), [IB_QPT_UC] = (IB_QP_CUR_STATE | IB_QP_ACCESS_FLAGS | IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE), [IB_QPT_RC] = (IB_QP_CUR_STATE | IB_QP_ACCESS_FLAGS | IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE | IB_QP_MIN_RNR_TIMER), [IB_QPT_XRC] = (IB_QP_CUR_STATE | IB_QP_ACCESS_FLAGS | IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE | IB_QP_MIN_RNR_TIMER), [IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY), [IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY), } }, [IB_QPS_SQD] = { .valid = 1, .opt_param = { [IB_QPT_UD] = IB_QP_EN_SQD_ASYNC_NOTIFY, [IB_QPT_UC] = IB_QP_EN_SQD_ASYNC_NOTIFY, [IB_QPT_RC] = IB_QP_EN_SQD_ASYNC_NOTIFY, [IB_QPT_XRC] = IB_QP_EN_SQD_ASYNC_NOTIFY, [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY, [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY } }, }, [IB_QPS_SQD] = { [IB_QPS_RESET] = { .valid = 1 }, [IB_QPS_ERR] = { .valid = 1 }, [IB_QPS_RTS] = { .valid = 1, .opt_param = { [IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY), [IB_QPT_UC] = (IB_QP_CUR_STATE | IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_PATH_MIG_STATE), [IB_QPT_RC] = (IB_QP_CUR_STATE | IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_MIN_RNR_TIMER | IB_QP_PATH_MIG_STATE), [IB_QPT_XRC] = (IB_QP_CUR_STATE | IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_MIN_RNR_TIMER | IB_QP_PATH_MIG_STATE), [IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY), [IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY), } }, [IB_QPS_SQD] = { .valid = 1, .opt_param = { [IB_QPT_UD] = (IB_QP_PKEY_INDEX | IB_QP_QKEY), [IB_QPT_UC] = (IB_QP_AV | IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX | IB_QP_PATH_MIG_STATE), [IB_QPT_RC] = (IB_QP_PORT | IB_QP_AV | IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY | IB_QP_MAX_QP_RD_ATOMIC | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX | IB_QP_MIN_RNR_TIMER | IB_QP_PATH_MIG_STATE), [IB_QPT_XRC] = (IB_QP_PORT | IB_QP_AV | IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY | IB_QP_MAX_QP_RD_ATOMIC | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX | IB_QP_MIN_RNR_TIMER | IB_QP_PATH_MIG_STATE), [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY), [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY), } } }, [IB_QPS_SQE] = { [IB_QPS_RESET] = { .valid = 1 }, [IB_QPS_ERR] = { .valid = 1 }, [IB_QPS_RTS] = { .valid = 1, .opt_param = { [IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY), [IB_QPT_UC] = (IB_QP_CUR_STATE | IB_QP_ACCESS_FLAGS), [IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY), [IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY), } } }, [IB_QPS_ERR] = { [IB_QPS_RESET] = { .valid = 1 }, [IB_QPS_ERR] = { .valid = 1 } } }; int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, enum ib_qp_type type, enum ib_qp_attr_mask mask) { enum ib_qp_attr_mask req_param, opt_param; if (cur_state < 0 || cur_state > IB_QPS_ERR || next_state < 0 || next_state > IB_QPS_ERR) return 0; if (mask & IB_QP_CUR_STATE && cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS && cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE) return 0; if (!qp_state_table[cur_state][next_state].valid) return 0; req_param = qp_state_table[cur_state][next_state].req_param[type]; opt_param = qp_state_table[cur_state][next_state].opt_param[type]; if ((mask & req_param) != req_param) return 0; if (mask & ~(req_param | opt_param | IB_QP_STATE)) return 0; return 1; } EXPORT_SYMBOL(ib_modify_qp_is_ok); int ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask) { return qp->device->modify_qp(qp, qp_attr, qp_attr_mask, NULL); } EXPORT_SYMBOL(ib_modify_qp); int ib_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) { return qp->device->query_qp ? qp->device->query_qp(qp, qp_attr, qp_attr_mask, qp_init_attr) : -ENOSYS; } EXPORT_SYMBOL(ib_query_qp); int ib_destroy_qp(struct ib_qp *qp) { struct ib_pd *pd; struct ib_cq *scq, *rcq; struct ib_srq *srq; struct ib_xrcd *xrcd; enum ib_qp_type qp_type = qp->qp_type; int ret; pd = qp->pd; scq = qp->send_cq; rcq = qp->recv_cq; srq = qp->srq; xrcd = qp->xrcd; ret = qp->device->destroy_qp(qp); if (!ret) { atomic_dec(&pd->usecnt); atomic_dec(&scq->usecnt); atomic_dec(&rcq->usecnt); if (srq) atomic_dec(&srq->usecnt); if (qp_type == IB_QPT_XRC) atomic_dec(&xrcd->usecnt); } return ret; } EXPORT_SYMBOL(ib_destroy_qp); /* Completion queues */ struct ib_cq *ib_create_cq(struct ib_device *device, ib_comp_handler comp_handler, void (*event_handler)(struct ib_event *, void *), void *cq_context, int cqe, int comp_vector) { struct ib_cq *cq; cq = device->create_cq(device, cqe, comp_vector, NULL, NULL); if (!IS_ERR(cq)) { cq->device = device; cq->uobject = NULL; cq->comp_handler = comp_handler; cq->event_handler = event_handler; cq->cq_context = cq_context; atomic_set(&cq->usecnt, 0); } return cq; } EXPORT_SYMBOL(ib_create_cq); int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) { return cq->device->modify_cq ? cq->device->modify_cq(cq, cq_count, cq_period) : -ENOSYS; } EXPORT_SYMBOL(ib_modify_cq); int ib_destroy_cq(struct ib_cq *cq) { if (atomic_read(&cq->usecnt)) return -EBUSY; return cq->device->destroy_cq(cq); } EXPORT_SYMBOL(ib_destroy_cq); int ib_resize_cq(struct ib_cq *cq, int cqe) { return cq->device->resize_cq ? cq->device->resize_cq(cq, cqe, NULL) : -ENOSYS; } EXPORT_SYMBOL(ib_resize_cq); /* Memory regions */ struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags) { struct ib_mr *mr; mr = pd->device->get_dma_mr(pd, mr_access_flags); if (!IS_ERR(mr)) { mr->device = pd->device; mr->pd = pd; mr->uobject = NULL; atomic_inc(&pd->usecnt); atomic_set(&mr->usecnt, 0); } return mr; } EXPORT_SYMBOL(ib_get_dma_mr); struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd, struct ib_phys_buf *phys_buf_array, int num_phys_buf, int mr_access_flags, u64 *iova_start) { struct ib_mr *mr; if (!pd->device->reg_phys_mr) return ERR_PTR(-ENOSYS); mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf, mr_access_flags, iova_start); if (!IS_ERR(mr)) { mr->device = pd->device; mr->pd = pd; mr->uobject = NULL; atomic_inc(&pd->usecnt); atomic_set(&mr->usecnt, 0); } return mr; } EXPORT_SYMBOL(ib_reg_phys_mr); int ib_rereg_phys_mr(struct ib_mr *mr, int mr_rereg_mask, struct ib_pd *pd, struct ib_phys_buf *phys_buf_array, int num_phys_buf, int mr_access_flags, u64 *iova_start) { struct ib_pd *old_pd; int ret; if (!mr->device->rereg_phys_mr) return -ENOSYS; if (atomic_read(&mr->usecnt)) return -EBUSY; old_pd = mr->pd; ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd, phys_buf_array, num_phys_buf, mr_access_flags, iova_start); if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) { atomic_dec(&old_pd->usecnt); atomic_inc(&pd->usecnt); } return ret; } EXPORT_SYMBOL(ib_rereg_phys_mr); int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr) { return mr->device->query_mr ? mr->device->query_mr(mr, mr_attr) : -ENOSYS; } EXPORT_SYMBOL(ib_query_mr); int ib_dereg_mr(struct ib_mr *mr) { struct ib_pd *pd; int ret; if (atomic_read(&mr->usecnt)) return -EBUSY; pd = mr->pd; ret = mr->device->dereg_mr(mr); if (!ret) atomic_dec(&pd->usecnt); return ret; } EXPORT_SYMBOL(ib_dereg_mr); struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len) { struct ib_mr *mr; if (!pd->device->alloc_fast_reg_mr) return ERR_PTR(-ENOSYS); mr = pd->device->alloc_fast_reg_mr(pd, max_page_list_len); if (!IS_ERR(mr)) { mr->device = pd->device; mr->pd = pd; mr->uobject = NULL; atomic_inc(&pd->usecnt); atomic_set(&mr->usecnt, 0); } return mr; } EXPORT_SYMBOL(ib_alloc_fast_reg_mr); struct ib_fast_reg_page_list *ib_alloc_fast_reg_page_list(struct ib_device *device, int max_page_list_len) { struct ib_fast_reg_page_list *page_list; if (!device->alloc_fast_reg_page_list) return ERR_PTR(-ENOSYS); page_list = device->alloc_fast_reg_page_list(device, max_page_list_len); if (!IS_ERR(page_list)) { page_list->device = device; page_list->max_page_list_len = max_page_list_len; } return page_list; } EXPORT_SYMBOL(ib_alloc_fast_reg_page_list); void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list) { page_list->device->free_fast_reg_page_list(page_list); } EXPORT_SYMBOL(ib_free_fast_reg_page_list); /* Memory windows */ struct ib_mw *ib_alloc_mw(struct ib_pd *pd) { struct ib_mw *mw; if (!pd->device->alloc_mw) return ERR_PTR(-ENOSYS); mw = pd->device->alloc_mw(pd); if (!IS_ERR(mw)) { mw->device = pd->device; mw->pd = pd; mw->uobject = NULL; atomic_inc(&pd->usecnt); } return mw; } EXPORT_SYMBOL(ib_alloc_mw); int ib_dealloc_mw(struct ib_mw *mw) { struct ib_pd *pd; int ret; pd = mw->pd; ret = mw->device->dealloc_mw(mw); if (!ret) atomic_dec(&pd->usecnt); return ret; } EXPORT_SYMBOL(ib_dealloc_mw); /* "Fast" memory regions */ struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd, int mr_access_flags, struct ib_fmr_attr *fmr_attr) { struct ib_fmr *fmr; if (!pd->device->alloc_fmr) return ERR_PTR(-ENOSYS); fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr); if (!IS_ERR(fmr)) { fmr->device = pd->device; fmr->pd = pd; atomic_inc(&pd->usecnt); } return fmr; } EXPORT_SYMBOL(ib_alloc_fmr); int ib_unmap_fmr(struct list_head *fmr_list) { struct ib_fmr *fmr; if (list_empty(fmr_list)) return 0; fmr = list_entry(fmr_list->next, struct ib_fmr, list); return fmr->device->unmap_fmr(fmr_list); } EXPORT_SYMBOL(ib_unmap_fmr); int ib_dealloc_fmr(struct ib_fmr *fmr) { struct ib_pd *pd; int ret; pd = fmr->pd; ret = fmr->device->dealloc_fmr(fmr); if (!ret) atomic_dec(&pd->usecnt); return ret; } EXPORT_SYMBOL(ib_dealloc_fmr); /* Multicast groups */ int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) { if (!qp->device->attach_mcast) return -ENOSYS; switch (rdma_node_get_transport(qp->device->node_type)) { case RDMA_TRANSPORT_IB: - if (qp->qp_type == IB_QPT_RAW_ETH) { + if (qp->qp_type == IB_QPT_RAW_PACKET) { /* In raw Etherent mgids the 63 msb's should be 0 */ if (gid->global.subnet_prefix & cpu_to_be64(~1ULL)) return -EINVAL; } else if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) return -EINVAL; break; case RDMA_TRANSPORT_IWARP: - if (qp->qp_type != IB_QPT_RAW_ETH) + if (qp->qp_type != IB_QPT_RAW_PACKET) return -EINVAL; break; } return qp->device->attach_mcast(qp, gid, lid); } EXPORT_SYMBOL(ib_attach_mcast); int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) { if (!qp->device->detach_mcast) return -ENOSYS; switch (rdma_node_get_transport(qp->device->node_type)) { case RDMA_TRANSPORT_IB: - if (qp->qp_type == IB_QPT_RAW_ETH) { + if (qp->qp_type == IB_QPT_RAW_PACKET) { /* In raw Etherent mgids the 63 msb's should be 0 */ if (gid->global.subnet_prefix & cpu_to_be64(~1ULL)) return -EINVAL; } else if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) return -EINVAL; break; case RDMA_TRANSPORT_IWARP: - if (qp->qp_type != IB_QPT_RAW_ETH) + if (qp->qp_type != IB_QPT_RAW_PACKET) return -EINVAL; break; } return qp->device->detach_mcast(qp, gid, lid); } EXPORT_SYMBOL(ib_detach_mcast); int ib_dealloc_xrcd(struct ib_xrcd *xrcd) { if (atomic_read(&xrcd->usecnt)) return -EBUSY; return xrcd->device->dealloc_xrcd(xrcd); } EXPORT_SYMBOL(ib_dealloc_xrcd); struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device) { struct ib_xrcd *xrcd; if (!device->alloc_xrcd) return ERR_PTR(-ENOSYS); xrcd = device->alloc_xrcd(device, NULL, NULL); if (!IS_ERR(xrcd)) { xrcd->device = device; xrcd->inode = NULL; xrcd->uobject = NULL; atomic_set(&xrcd->usecnt, 0); } return xrcd; } EXPORT_SYMBOL(ib_alloc_xrcd); Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/Kconfig =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/Kconfig (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/Kconfig (revision 273864) @@ -1,8 +1,10 @@ config MLX4_INFINIBAND tristate "Mellanox ConnectX HCA support" + depends on NETDEVICES && ETHERNET && PCI + select NET_VENDOR_MELLANOX select MLX4_CORE ---help--- This driver provides low-level InfiniBand support for Mellanox ConnectX PCI Express host channel adapters (HCAs). This is required to use InfiniBand protocols such as IP-over-IB or SRP with these devices. Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/Makefile =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/Makefile (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/Makefile (revision 273864) @@ -1,4 +1,31 @@ -obj-$(CONFIG_MLX4_INFINIBAND) += mlx4_ib.o +# $FreeBSD$ +#.PATH: ${.CURDIR}/../../ofed/drivers/infiniband/hw/mlx4 +#.PATH: ${.CURDIR}/../../../../include/linux -mlx4_ib-y := ah.o cq.o doorbell.o mad.o main.o mr.o qp.o srq.o -mlx4_ib-y += wc.o +.include + +KMOD = mlx4ib +SRCS = device_if.h bus_if.h pci_if.h vnode_if.h +#SRCS+= linux_compat.c linux_radix.c +SRCS+= ah.c cq.c doorbell.c mad.c main.c mr.c qp.c srq.c wc.c +SRCS+= opt_inet.h opt_inet6.h + +#CFLAGS+= -I${.CURDIR}/../../ofed/include/ +CFLAGS+= -I${.CURDIR}/../../../../include +CFLAGS+= -DCONFIG_INFINIBAND_USER_MEM + +.if !defined(KERNBUILDDIR) +.if ${MK_INET_SUPPORT} != "no" +opt_inet.h: + @echo "#define INET 1" > ${.TARGET} +.endif + +.if ${MK_INET6_SUPPORT} != "no" +opt_inet6.h: + @echo "#define INET6 1" > ${.TARGET} +.endif +.endif + +.include + +CFLAGS+= -Wno-cast-qual -Wno-pointer-arith -fms-extensions Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/ah.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/ah.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/ah.c (revision 273864) @@ -1,205 +1,203 @@ /* * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include "mlx4_ib.h" + +#include +#include +#include #include +#include + +#include #include #include -#include +#include "mlx4_ib.h" + int mlx4_ib_resolve_grh(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah_attr, u8 *mac, int *is_mcast, u8 port) { - struct mlx4_ib_iboe *iboe = &dev->iboe; struct in6_addr in6; *is_mcast = 0; - spin_lock(&iboe->lock); - if (!iboe->netdevs[port - 1]) { - spin_unlock(&iboe->lock); - return -EINVAL; - } - spin_unlock(&iboe->lock); memcpy(&in6, ah_attr->grh.dgid.raw, sizeof in6); if (rdma_link_local_addr(&in6)) rdma_get_ll_mac(&in6, mac); else if (rdma_is_multicast_addr(&in6)) { rdma_get_mcast_mac(&in6, mac); *is_mcast = 1; } else return -EINVAL; return 0; } static struct ib_ah *create_ib_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr, struct mlx4_ib_ah *ah) { struct mlx4_dev *dev = to_mdev(pd->device)->dev; ah->av.ib.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24)); ah->av.ib.g_slid = ah_attr->src_path_bits; if (ah_attr->ah_flags & IB_AH_GRH) { ah->av.ib.g_slid |= 0x80; ah->av.ib.gid_index = ah_attr->grh.sgid_index; ah->av.ib.hop_limit = ah_attr->grh.hop_limit; ah->av.ib.sl_tclass_flowlabel |= cpu_to_be32((ah_attr->grh.traffic_class << 20) | ah_attr->grh.flow_label); memcpy(ah->av.ib.dgid, ah_attr->grh.dgid.raw, 16); } ah->av.ib.dlid = cpu_to_be16(ah_attr->dlid); if (ah_attr->static_rate) { ah->av.ib.stat_rate = ah_attr->static_rate + MLX4_STAT_RATE_OFFSET; while (ah->av.ib.stat_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET && !(1 << ah->av.ib.stat_rate & dev->caps.stat_rate_support)) --ah->av.ib.stat_rate; } ah->av.ib.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 28); return &ah->ibah; } static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr, - struct mlx4_ib_ah *ah) + struct mlx4_ib_ah *ah) { struct mlx4_ib_dev *ibdev = to_mdev(pd->device); struct mlx4_dev *dev = ibdev->dev; + union ib_gid sgid; u8 mac[6]; int err; int is_mcast; u16 vlan_tag; - union ib_gid sgid; err = mlx4_ib_resolve_grh(ibdev, ah_attr, mac, &is_mcast, ah_attr->port_num); if (err) return ERR_PTR(err); memcpy(ah->av.eth.mac, mac, 6); err = ib_get_cached_gid(pd->device, ah_attr->port_num, ah_attr->grh.sgid_index, &sgid); if (err) return ERR_PTR(err); vlan_tag = rdma_get_vlan_id(&sgid); if (vlan_tag < 0x1000) vlan_tag |= (ah_attr->sl & 7) << 13; ah->av.eth.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24)); ah->av.eth.gid_index = ah_attr->grh.sgid_index; ah->av.eth.vlan = cpu_to_be16(vlan_tag); if (ah_attr->static_rate) { ah->av.eth.stat_rate = ah_attr->static_rate + MLX4_STAT_RATE_OFFSET; while (ah->av.eth.stat_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET && !(1 << ah->av.eth.stat_rate & dev->caps.stat_rate_support)) --ah->av.eth.stat_rate; } /* * HW requires multicast LID so we just choose one. */ if (is_mcast) ah->av.ib.dlid = cpu_to_be16(0xc000); memcpy(ah->av.eth.dgid, ah_attr->grh.dgid.raw, 16); - ah->av.eth.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 28); + ah->av.eth.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 29); return &ah->ibah; } struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) { struct mlx4_ib_ah *ah; struct ib_ah *ret; ah = kzalloc(sizeof *ah, GFP_ATOMIC); if (!ah) return ERR_PTR(-ENOMEM); if (rdma_port_get_link_layer(pd->device, ah_attr->port_num) == IB_LINK_LAYER_ETHERNET) { if (!(ah_attr->ah_flags & IB_AH_GRH)) { ret = ERR_PTR(-EINVAL); - goto out; } else { - /* TBD: need to handle the case when we get called - in an atomic context and there we might sleep. We - don't expect this currently since we're working with - link local addresses which we can translate without - going to sleep */ + /* + * TBD: need to handle the case when we get + * called in an atomic context and there we + * might sleep. We don't expect this + * currently since we're working with link + * local addresses which we can translate + * without going to sleep. + */ ret = create_iboe_ah(pd, ah_attr, ah); - if (IS_ERR(ret)) - goto out; - else - return ret; } + + if (IS_ERR(ret)) + kfree(ah); + + return ret; } else return create_ib_ah(pd, ah_attr, ah); /* never fails */ - -out: - kfree(ah); - return ret; } int mlx4_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr) { struct mlx4_ib_ah *ah = to_mah(ibah); enum rdma_link_layer ll; memset(ah_attr, 0, sizeof *ah_attr); ah_attr->sl = be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28; ah_attr->port_num = be32_to_cpu(ah->av.ib.port_pd) >> 24; ll = rdma_port_get_link_layer(ibah->device, ah_attr->port_num); ah_attr->dlid = ll == IB_LINK_LAYER_INFINIBAND ? be16_to_cpu(ah->av.ib.dlid) : 0; if (ah->av.ib.stat_rate) ah_attr->static_rate = ah->av.ib.stat_rate - MLX4_STAT_RATE_OFFSET; ah_attr->src_path_bits = ah->av.ib.g_slid & 0x7F; if (mlx4_ib_ah_grh_present(ah)) { ah_attr->ah_flags = IB_AH_GRH; ah_attr->grh.traffic_class = be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20; ah_attr->grh.flow_label = be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) & 0xfffff; ah_attr->grh.hop_limit = ah->av.ib.hop_limit; ah_attr->grh.sgid_index = ah->av.ib.gid_index; memcpy(ah_attr->grh.dgid.raw, ah->av.ib.dgid, 16); } return 0; } int mlx4_ib_destroy_ah(struct ib_ah *ah) { kfree(to_mah(ah)); return 0; } - Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/alias_GUID.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/alias_GUID.c (nonexistent) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/alias_GUID.c (revision 273864) @@ -0,0 +1,687 @@ +/* + * Copyright (c) 2012 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + /***********************************************************/ +/*This file support the handling of the Alias GUID feature. */ +/***********************************************************/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mlx4_ib.h" + +/* +The driver keeps the current state of all guids, as they are in the HW. +Whenever we receive an smp mad GUIDInfo record, the data will be cached. +*/ + +struct mlx4_alias_guid_work_context { + u8 port; + struct mlx4_ib_dev *dev ; + struct ib_sa_query *sa_query; + struct completion done; + int query_id; + struct list_head list; + int block_num; +}; + +struct mlx4_next_alias_guid_work { + u8 port; + u8 block_num; + struct mlx4_sriov_alias_guid_info_rec_det rec_det; +}; + + +void mlx4_ib_update_cache_on_guid_change(struct mlx4_ib_dev *dev, int block_num, + u8 port_num, u8 *p_data) +{ + int i; + u64 guid_indexes; + int slave_id; + int port_index = port_num - 1; + + if (!mlx4_is_master(dev->dev)) + return; + + guid_indexes = be64_to_cpu((__force __be64) dev->sriov.alias_guid. + ports_guid[port_num - 1]. + all_rec_per_port[block_num].guid_indexes); + pr_debug("port: %d, guid_indexes: 0x%llx\n", port_num, (long long)guid_indexes); + + for (i = 0; i < NUM_ALIAS_GUID_IN_REC; i++) { + /* The location of the specific index starts from bit number 4 + * until bit num 11 */ + if (test_bit(i + 4, (unsigned long *)&guid_indexes)) { + slave_id = (block_num * NUM_ALIAS_GUID_IN_REC) + i ; + if (slave_id >= dev->dev->num_slaves) { + pr_debug("The last slave: %d\n", slave_id); + return; + } + + /* cache the guid: */ + memcpy(&dev->sriov.demux[port_index].guid_cache[slave_id], + &p_data[i * GUID_REC_SIZE], + GUID_REC_SIZE); + } else + pr_debug("Guid number: %d in block: %d" + " was not updated\n", i, block_num); + } +} + +static __be64 get_cached_alias_guid(struct mlx4_ib_dev *dev, int port, int index) +{ + if (index >= NUM_ALIAS_GUID_PER_PORT) { + pr_err("%s: ERROR: asked for index:%d\n", __func__, index); + return (__force __be64) -1; + } + return *(__be64 *)&dev->sriov.demux[port - 1].guid_cache[index]; +} + + +ib_sa_comp_mask mlx4_ib_get_aguid_comp_mask_from_ix(int index) +{ + return IB_SA_COMP_MASK(4 + index); +} + +/* + * Whenever new GUID is set/unset (guid table change) create event and + * notify the relevant slave (master also should be notified). + * If the GUID value is not as we have in the cache the slave will not be + * updated; in this case it waits for the smp_snoop or the port management + * event to call the function and to update the slave. + * block_number - the index of the block (16 blocks available) + * port_number - 1 or 2 + */ +void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev, + int block_num, u8 port_num, + u8 *p_data) +{ + int i; + u64 guid_indexes; + int slave_id; + enum slave_port_state new_state; + enum slave_port_state prev_state; + __be64 tmp_cur_ag, form_cache_ag; + enum slave_port_gen_event gen_event; + + if (!mlx4_is_master(dev->dev)) + return; + + guid_indexes = be64_to_cpu((__force __be64) dev->sriov.alias_guid. + ports_guid[port_num - 1]. + all_rec_per_port[block_num].guid_indexes); + pr_debug("port: %d, guid_indexes: 0x%llx\n", port_num, (long long)guid_indexes); + + /*calculate the slaves and notify them*/ + for (i = 0; i < NUM_ALIAS_GUID_IN_REC; i++) { + /* the location of the specific index runs from bits 4..11 */ + if (!(test_bit(i + 4, (unsigned long *)&guid_indexes))) + continue; + + slave_id = (block_num * NUM_ALIAS_GUID_IN_REC) + i ; + if (slave_id >= dev->dev->num_slaves) + return; + tmp_cur_ag = *(__be64 *)&p_data[i * GUID_REC_SIZE]; + form_cache_ag = get_cached_alias_guid(dev, port_num, + (NUM_ALIAS_GUID_IN_REC * block_num) + i); + /* + * Check if guid is not the same as in the cache, + * If it is different, wait for the snoop_smp or the port mgmt + * change event to update the slave on its port state change + */ + if (tmp_cur_ag != form_cache_ag) + continue; + mlx4_gen_guid_change_eqe(dev->dev, slave_id, port_num); + + /*2 cases: Valid GUID, and Invalid Guid*/ + + if (tmp_cur_ag != MLX4_NOT_SET_GUID) { /*valid GUID*/ + prev_state = mlx4_get_slave_port_state(dev->dev, slave_id, port_num); + new_state = set_and_calc_slave_port_state(dev->dev, slave_id, port_num, + MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID, + &gen_event); + pr_debug("slave: %d, port: %d prev_port_state: %d," + " new_port_state: %d, gen_event: %d\n", + slave_id, port_num, prev_state, new_state, gen_event); + if (gen_event == SLAVE_PORT_GEN_EVENT_UP) { + pr_debug("sending PORT_UP event to slave: %d, port: %d\n", + slave_id, port_num); + mlx4_gen_port_state_change_eqe(dev->dev, slave_id, + port_num, MLX4_PORT_CHANGE_SUBTYPE_ACTIVE); + } + } else { /* request to invalidate GUID */ + set_and_calc_slave_port_state(dev->dev, slave_id, port_num, + MLX4_PORT_STATE_IB_EVENT_GID_INVALID, + &gen_event); + pr_debug("sending PORT DOWN event to slave: %d, port: %d\n", + slave_id, port_num); + mlx4_gen_port_state_change_eqe(dev->dev, slave_id, port_num, + MLX4_PORT_CHANGE_SUBTYPE_DOWN); + } + } +} + +static void aliasguid_query_handler(int status, + struct ib_sa_guidinfo_rec *guid_rec, + void *context) +{ + struct mlx4_ib_dev *dev; + struct mlx4_alias_guid_work_context *cb_ctx = context; + u8 port_index ; + int i; + struct mlx4_sriov_alias_guid_info_rec_det *rec; + unsigned long flags, flags1; + + if (!context) + return; + + dev = cb_ctx->dev; + port_index = cb_ctx->port - 1; + rec = &dev->sriov.alias_guid.ports_guid[port_index]. + all_rec_per_port[cb_ctx->block_num]; + + if (status) { + rec->status = MLX4_GUID_INFO_STATUS_IDLE; + pr_debug("(port: %d) failed: status = %d\n", + cb_ctx->port, status); + goto out; + } + + if (guid_rec->block_num != cb_ctx->block_num) { + pr_err("block num mismatch: %d != %d\n", + cb_ctx->block_num, guid_rec->block_num); + goto out; + } + + pr_debug("lid/port: %d/%d, block_num: %d\n", + be16_to_cpu(guid_rec->lid), cb_ctx->port, + guid_rec->block_num); + + rec = &dev->sriov.alias_guid.ports_guid[port_index]. + all_rec_per_port[guid_rec->block_num]; + + rec->status = MLX4_GUID_INFO_STATUS_SET; + rec->method = MLX4_GUID_INFO_RECORD_SET; + + for (i = 0 ; i < NUM_ALIAS_GUID_IN_REC; i++) { + __be64 tmp_cur_ag; + tmp_cur_ag = *(__be64 *)&guid_rec->guid_info_list[i * GUID_REC_SIZE]; + /* check if the SM didn't assign one of the records. + * if it didn't, if it was not sysadmin request: + * ask the SM to give a new GUID, (instead of the driver request). + */ + if (tmp_cur_ag == MLX4_NOT_SET_GUID) { + mlx4_ib_warn(&dev->ib_dev, "%s:Record num %d in " + "block_num: %d was declined by SM, " + "ownership by %d (0 = driver, 1=sysAdmin," + " 2=None)\n", __func__, i, + guid_rec->block_num, rec->ownership); + if (rec->ownership == MLX4_GUID_DRIVER_ASSIGN) { + /* if it is driver assign, asks for new GUID from SM*/ + *(__be64 *)&rec->all_recs[i * GUID_REC_SIZE] = + MLX4_NOT_SET_GUID; + + /* Mark the record as not assigned, and let it + * be sent again in the next work sched.*/ + rec->status = MLX4_GUID_INFO_STATUS_IDLE; + rec->guid_indexes |= mlx4_ib_get_aguid_comp_mask_from_ix(i); + } + } else { + /* properly assigned record. */ + /* We save the GUID we just got from the SM in the + * admin_guid in order to be persistent, and in the + * request from the sm the process will ask for the same GUID */ + if (rec->ownership == MLX4_GUID_SYSADMIN_ASSIGN && + tmp_cur_ag != *(__be64 *)&rec->all_recs[i * GUID_REC_SIZE]) { + /* the sysadmin assignment failed.*/ + mlx4_ib_warn(&dev->ib_dev, "%s: Failed to set" + " admin guid after SysAdmin " + "configuration. " + "Record num %d in block_num:%d " + "was declined by SM, " + "new val(0x%llx) was kept\n", + __func__, i, + guid_rec->block_num, + (long long)be64_to_cpu(*(__be64 *) & + rec->all_recs[i * GUID_REC_SIZE])); + } else { + memcpy(&rec->all_recs[i * GUID_REC_SIZE], + &guid_rec->guid_info_list[i * GUID_REC_SIZE], + GUID_REC_SIZE); + } + } + } + /* + The func is call here to close the cases when the + sm doesn't send smp, so in the sa response the driver + notifies the slave. + */ + mlx4_ib_notify_slaves_on_guid_change(dev, guid_rec->block_num, + cb_ctx->port, + guid_rec->guid_info_list); +out: + spin_lock_irqsave(&dev->sriov.going_down_lock, flags); + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); + if (!dev->sriov.is_going_down) + queue_delayed_work(dev->sriov.alias_guid.ports_guid[port_index].wq, + &dev->sriov.alias_guid.ports_guid[port_index]. + alias_guid_work, 0); + if (cb_ctx->sa_query) { + list_del(&cb_ctx->list); + kfree(cb_ctx); + } else + complete(&cb_ctx->done); + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); + spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); +} + +static void invalidate_guid_record(struct mlx4_ib_dev *dev, u8 port, int index) +{ + int i; + u64 cur_admin_val; + ib_sa_comp_mask comp_mask = 0; + + dev->sriov.alias_guid.ports_guid[port - 1].all_rec_per_port[index].status + = MLX4_GUID_INFO_STATUS_IDLE; + dev->sriov.alias_guid.ports_guid[port - 1].all_rec_per_port[index].method + = MLX4_GUID_INFO_RECORD_SET; + + /* calculate the comp_mask for that record.*/ + for (i = 0; i < NUM_ALIAS_GUID_IN_REC; i++) { + cur_admin_val = + *(u64 *)&dev->sriov.alias_guid.ports_guid[port - 1]. + all_rec_per_port[index].all_recs[GUID_REC_SIZE * i]; + /* + check the admin value: if it's for delete (~00LL) or + it is the first guid of the first record (hw guid) or + the records is not in ownership of the sysadmin and the sm doesn't + need to assign GUIDs, then don't put it up for assignment. + */ + if (MLX4_GUID_FOR_DELETE_VAL == cur_admin_val || + (!index && !i) || + MLX4_GUID_NONE_ASSIGN == dev->sriov.alias_guid. + ports_guid[port - 1].all_rec_per_port[index].ownership) + continue; + comp_mask |= mlx4_ib_get_aguid_comp_mask_from_ix(i); + } + dev->sriov.alias_guid.ports_guid[port - 1]. + all_rec_per_port[index].guid_indexes = comp_mask; +} + +static int set_guid_rec(struct ib_device *ibdev, + u8 port, int index, + struct mlx4_sriov_alias_guid_info_rec_det *rec_det) +{ + int err; + struct mlx4_ib_dev *dev = to_mdev(ibdev); + struct ib_sa_guidinfo_rec guid_info_rec; + ib_sa_comp_mask comp_mask; + struct ib_port_attr attr; + struct mlx4_alias_guid_work_context *callback_context; + unsigned long resched_delay, flags, flags1; + struct list_head *head = + &dev->sriov.alias_guid.ports_guid[port - 1].cb_list; + + err = __mlx4_ib_query_port(ibdev, port, &attr, 1); + if (err) { + pr_debug("mlx4_ib_query_port failed (err: %d), port: %d\n", + err, port); + return err; + } + /*check the port was configured by the sm, otherwise no need to send */ + if (attr.state != IB_PORT_ACTIVE) { + pr_debug("port %d not active...rescheduling\n", port); + resched_delay = 5 * HZ; + err = -EAGAIN; + goto new_schedule; + } + + callback_context = kmalloc(sizeof *callback_context, GFP_KERNEL); + if (!callback_context) { + err = -ENOMEM; + resched_delay = HZ * 5; + goto new_schedule; + } + callback_context->port = port; + callback_context->dev = dev; + callback_context->block_num = index; + + memset(&guid_info_rec, 0, sizeof (struct ib_sa_guidinfo_rec)); + + guid_info_rec.lid = cpu_to_be16(attr.lid); + guid_info_rec.block_num = index; + + memcpy(guid_info_rec.guid_info_list, rec_det->all_recs, + GUID_REC_SIZE * NUM_ALIAS_GUID_IN_REC); + comp_mask = IB_SA_GUIDINFO_REC_LID | IB_SA_GUIDINFO_REC_BLOCK_NUM | + rec_det->guid_indexes; + + init_completion(&callback_context->done); + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); + list_add_tail(&callback_context->list, head); + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); + + callback_context->query_id = + ib_sa_guid_info_rec_query(dev->sriov.alias_guid.sa_client, + ibdev, port, &guid_info_rec, + comp_mask, rec_det->method, 1000, + GFP_KERNEL, aliasguid_query_handler, + callback_context, + &callback_context->sa_query); + if (callback_context->query_id < 0) { + pr_debug("ib_sa_guid_info_rec_query failed, query_id: " + "%d. will reschedule to the next 1 sec.\n", + callback_context->query_id); + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); + list_del(&callback_context->list); + kfree(callback_context); + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); + resched_delay = 1 * HZ; + err = -EAGAIN; + goto new_schedule; + } + err = 0; + goto out; + +new_schedule: + spin_lock_irqsave(&dev->sriov.going_down_lock, flags); + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); + invalidate_guid_record(dev, port, index); + if (!dev->sriov.is_going_down) { + queue_delayed_work(dev->sriov.alias_guid.ports_guid[port - 1].wq, + &dev->sriov.alias_guid.ports_guid[port - 1].alias_guid_work, + resched_delay); + } + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); + spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); + +out: + return err; +} + +void mlx4_ib_invalidate_all_guid_record(struct mlx4_ib_dev *dev, int port) +{ + int i; + unsigned long flags, flags1; + + pr_debug("port %d\n", port); + + spin_lock_irqsave(&dev->sriov.going_down_lock, flags); + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); + for (i = 0; i < NUM_ALIAS_GUID_REC_IN_PORT; i++) + invalidate_guid_record(dev, port, i); + + if (mlx4_is_master(dev->dev) && !dev->sriov.is_going_down) { + /* + make sure no work waits in the queue, if the work is already + queued(not on the timer) the cancel will fail. That is not a problem + because we just want the work started. + */ + cancel_delayed_work(&dev->sriov.alias_guid. + ports_guid[port - 1].alias_guid_work); + queue_delayed_work(dev->sriov.alias_guid.ports_guid[port - 1].wq, + &dev->sriov.alias_guid.ports_guid[port - 1].alias_guid_work, + 0); + } + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); + spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); +} + +/* The function returns the next record that was + * not configured (or failed to be configured) */ +static int get_next_record_to_update(struct mlx4_ib_dev *dev, u8 port, + struct mlx4_next_alias_guid_work *rec) +{ + int j; + unsigned long flags; + + for (j = 0; j < NUM_ALIAS_GUID_REC_IN_PORT; j++) { + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags); + if (dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[j].status == + MLX4_GUID_INFO_STATUS_IDLE) { + memcpy(&rec->rec_det, + &dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[j], + sizeof (struct mlx4_sriov_alias_guid_info_rec_det)); + rec->port = port; + rec->block_num = j; + dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[j].status = + MLX4_GUID_INFO_STATUS_PENDING; + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags); + return 0; + } + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags); + } + return -ENOENT; +} + +static void set_administratively_guid_record(struct mlx4_ib_dev *dev, int port, + int rec_index, + struct mlx4_sriov_alias_guid_info_rec_det *rec_det) +{ + dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[rec_index].guid_indexes = + rec_det->guid_indexes; + memcpy(dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[rec_index].all_recs, + rec_det->all_recs, NUM_ALIAS_GUID_IN_REC * GUID_REC_SIZE); + dev->sriov.alias_guid.ports_guid[port].all_rec_per_port[rec_index].status = + rec_det->status; +} + +static void set_all_slaves_guids(struct mlx4_ib_dev *dev, int port) +{ + int j; + struct mlx4_sriov_alias_guid_info_rec_det rec_det ; + + for (j = 0 ; j < NUM_ALIAS_GUID_REC_IN_PORT ; j++) { + memset(rec_det.all_recs, 0, NUM_ALIAS_GUID_IN_REC * GUID_REC_SIZE); + rec_det.guid_indexes = (!j ? 0 : IB_SA_GUIDINFO_REC_GID0) | + IB_SA_GUIDINFO_REC_GID1 | IB_SA_GUIDINFO_REC_GID2 | + IB_SA_GUIDINFO_REC_GID3 | IB_SA_GUIDINFO_REC_GID4 | + IB_SA_GUIDINFO_REC_GID5 | IB_SA_GUIDINFO_REC_GID6 | + IB_SA_GUIDINFO_REC_GID7; + rec_det.status = MLX4_GUID_INFO_STATUS_IDLE; + set_administratively_guid_record(dev, port, j, &rec_det); + } +} + +static void alias_guid_work(struct work_struct *work) +{ + struct delayed_work *delay = to_delayed_work(work); + int ret = 0; + struct mlx4_next_alias_guid_work *rec; + struct mlx4_sriov_alias_guid_port_rec_det *sriov_alias_port = + container_of(delay, struct mlx4_sriov_alias_guid_port_rec_det, + alias_guid_work); + struct mlx4_sriov_alias_guid *sriov_alias_guid = sriov_alias_port->parent; + struct mlx4_ib_sriov *ib_sriov = container_of(sriov_alias_guid, + struct mlx4_ib_sriov, + alias_guid); + struct mlx4_ib_dev *dev = container_of(ib_sriov, struct mlx4_ib_dev, sriov); + + rec = kzalloc(sizeof *rec, GFP_KERNEL); + if (!rec) { + pr_err("alias_guid_work: No Memory\n"); + return; + } + + pr_debug("starting [port: %d]...\n", sriov_alias_port->port + 1); + ret = get_next_record_to_update(dev, sriov_alias_port->port, rec); + if (ret) { + pr_debug("No more records to update.\n"); + goto out; + } + + set_guid_rec(&dev->ib_dev, rec->port + 1, rec->block_num, + &rec->rec_det); + +out: + kfree(rec); +} + + +void mlx4_ib_init_alias_guid_work(struct mlx4_ib_dev *dev, int port) +{ + unsigned long flags, flags1; + + if (!mlx4_is_master(dev->dev)) + return; + spin_lock_irqsave(&dev->sriov.going_down_lock, flags); + spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1); + if (!dev->sriov.is_going_down) { + queue_delayed_work(dev->sriov.alias_guid.ports_guid[port].wq, + &dev->sriov.alias_guid.ports_guid[port].alias_guid_work, 0); + } + spin_unlock_irqrestore(&dev->sriov.alias_guid.ag_work_lock, flags1); + spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); +} + +void mlx4_ib_destroy_alias_guid_service(struct mlx4_ib_dev *dev) +{ + int i; + struct mlx4_ib_sriov *sriov = &dev->sriov; + struct mlx4_alias_guid_work_context *cb_ctx; + struct mlx4_sriov_alias_guid_port_rec_det *det; + struct ib_sa_query *sa_query; + unsigned long flags; + + for (i = 0 ; i < dev->num_ports; i++) { + cancel_delayed_work(&dev->sriov.alias_guid.ports_guid[i].alias_guid_work); + det = &sriov->alias_guid.ports_guid[i]; + spin_lock_irqsave(&sriov->alias_guid.ag_work_lock, flags); + while (!list_empty(&det->cb_list)) { + cb_ctx = list_entry(det->cb_list.next, + struct mlx4_alias_guid_work_context, + list); + sa_query = cb_ctx->sa_query; + cb_ctx->sa_query = NULL; + list_del(&cb_ctx->list); + spin_unlock_irqrestore(&sriov->alias_guid.ag_work_lock, flags); + ib_sa_cancel_query(cb_ctx->query_id, sa_query); + wait_for_completion(&cb_ctx->done); + kfree(cb_ctx); + spin_lock_irqsave(&sriov->alias_guid.ag_work_lock, flags); + } + spin_unlock_irqrestore(&sriov->alias_guid.ag_work_lock, flags); + } + for (i = 0 ; i < dev->num_ports; i++) { + flush_workqueue(dev->sriov.alias_guid.ports_guid[i].wq); + destroy_workqueue(dev->sriov.alias_guid.ports_guid[i].wq); + } + ib_sa_unregister_client(dev->sriov.alias_guid.sa_client); + kfree(dev->sriov.alias_guid.sa_client); +} + +int mlx4_ib_init_alias_guid_service(struct mlx4_ib_dev *dev) +{ + char alias_wq_name[15]; + int ret = 0; + int i, j, k; + union ib_gid gid; + + if (!mlx4_is_master(dev->dev)) + return 0; + dev->sriov.alias_guid.sa_client = + kzalloc(sizeof *dev->sriov.alias_guid.sa_client, GFP_KERNEL); + if (!dev->sriov.alias_guid.sa_client) + return -ENOMEM; + + ib_sa_register_client(dev->sriov.alias_guid.sa_client); + + spin_lock_init(&dev->sriov.alias_guid.ag_work_lock); + + for (i = 1; i <= dev->num_ports; ++i) { + if (dev->ib_dev.query_gid(&dev->ib_dev , i, 0, &gid)) { + ret = -EFAULT; + goto err_unregister; + } + } + + for (i = 0 ; i < dev->num_ports; i++) { + memset(&dev->sriov.alias_guid.ports_guid[i], 0, + sizeof (struct mlx4_sriov_alias_guid_port_rec_det)); + /*Check if the SM doesn't need to assign the GUIDs*/ + for (j = 0; j < NUM_ALIAS_GUID_REC_IN_PORT; j++) { + if (mlx4_ib_sm_guid_assign) { + dev->sriov.alias_guid.ports_guid[i]. + all_rec_per_port[j]. + ownership = MLX4_GUID_DRIVER_ASSIGN; + continue; + } + dev->sriov.alias_guid.ports_guid[i].all_rec_per_port[j]. + ownership = MLX4_GUID_NONE_ASSIGN; + /*mark each val as it was deleted, + till the sysAdmin will give it valid val*/ + for (k = 0; k < NUM_ALIAS_GUID_IN_REC; k++) { + *(__be64 *)&dev->sriov.alias_guid.ports_guid[i]. + all_rec_per_port[j].all_recs[GUID_REC_SIZE * k] = + cpu_to_be64(MLX4_GUID_FOR_DELETE_VAL); + } + } + INIT_LIST_HEAD(&dev->sriov.alias_guid.ports_guid[i].cb_list); + /*prepare the records, set them to be allocated by sm*/ + for (j = 0 ; j < NUM_ALIAS_GUID_REC_IN_PORT; j++) + invalidate_guid_record(dev, i + 1, j); + + dev->sriov.alias_guid.ports_guid[i].parent = &dev->sriov.alias_guid; + dev->sriov.alias_guid.ports_guid[i].port = i; + if (mlx4_ib_sm_guid_assign) + set_all_slaves_guids(dev, i); + + snprintf(alias_wq_name, sizeof alias_wq_name, "alias_guid%d", i); + dev->sriov.alias_guid.ports_guid[i].wq = + create_singlethread_workqueue(alias_wq_name); + if (!dev->sriov.alias_guid.ports_guid[i].wq) { + ret = -ENOMEM; + goto err_thread; + } + INIT_DELAYED_WORK(&dev->sriov.alias_guid.ports_guid[i].alias_guid_work, + alias_guid_work); + } + return 0; + +err_thread: + for (--i; i >= 0; i--) { + destroy_workqueue(dev->sriov.alias_guid.ports_guid[i].wq); + dev->sriov.alias_guid.ports_guid[i].wq = NULL; + } + +err_unregister: + ib_sa_unregister_client(dev->sriov.alias_guid.sa_client); + kfree(dev->sriov.alias_guid.sa_client); + dev->sriov.alias_guid.sa_client = NULL; + pr_err("init_alias_guid_service: Failed. (ret:%d)\n", ret); + return ret; +} Property changes on: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/alias_GUID.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/cm.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/cm.c (nonexistent) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/cm.c (revision 273864) @@ -0,0 +1,440 @@ +/* + * Copyright (c) 2012 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include + +#include +#include +#include + +#include "mlx4_ib.h" + +#define CM_CLEANUP_CACHE_TIMEOUT (5 * HZ) + +struct id_map_entry { + struct rb_node node; + + u32 sl_cm_id; + u32 pv_cm_id; + int slave_id; + int scheduled_delete; + struct mlx4_ib_dev *dev; + + struct list_head list; + struct delayed_work timeout; +}; + +struct cm_generic_msg { + struct ib_mad_hdr hdr; + + __be32 local_comm_id; + __be32 remote_comm_id; +}; + +struct cm_req_msg { + unsigned char unused[0x60]; + union ib_gid primary_path_sgid; +}; + + +static void set_local_comm_id(struct ib_mad *mad, u32 cm_id) +{ + struct cm_generic_msg *msg = (struct cm_generic_msg *)mad; + msg->local_comm_id = cpu_to_be32(cm_id); +} + +static u32 get_local_comm_id(struct ib_mad *mad) +{ + struct cm_generic_msg *msg = (struct cm_generic_msg *)mad; + + return be32_to_cpu(msg->local_comm_id); +} + +static void set_remote_comm_id(struct ib_mad *mad, u32 cm_id) +{ + struct cm_generic_msg *msg = (struct cm_generic_msg *)mad; + msg->remote_comm_id = cpu_to_be32(cm_id); +} + +static u32 get_remote_comm_id(struct ib_mad *mad) +{ + struct cm_generic_msg *msg = (struct cm_generic_msg *)mad; + + return be32_to_cpu(msg->remote_comm_id); +} + +static union ib_gid gid_from_req_msg(struct ib_device *ibdev, struct ib_mad *mad) +{ + struct cm_req_msg *msg = (struct cm_req_msg *)mad; + + return msg->primary_path_sgid; +} + +/* Lock should be taken before called */ +static struct id_map_entry * +id_map_find_by_sl_id(struct ib_device *ibdev, u32 slave_id, u32 sl_cm_id) +{ + struct rb_root *sl_id_map = &to_mdev(ibdev)->sriov.sl_id_map; + struct rb_node *node = sl_id_map->rb_node; + + while (node) { + struct id_map_entry *id_map_entry = + rb_entry(node, struct id_map_entry, node); + + if (id_map_entry->sl_cm_id > sl_cm_id) + node = node->rb_left; + else if (id_map_entry->sl_cm_id < sl_cm_id) + node = node->rb_right; + else if (id_map_entry->slave_id > slave_id) + node = node->rb_left; + else if (id_map_entry->slave_id < slave_id) + node = node->rb_right; + else + return id_map_entry; + } + return NULL; +} + +static void id_map_ent_timeout(struct work_struct *work) +{ + struct delayed_work *delay = to_delayed_work(work); + struct id_map_entry *ent = container_of(delay, struct id_map_entry, timeout); + struct id_map_entry *db_ent, *found_ent; + struct mlx4_ib_dev *dev = ent->dev; + struct mlx4_ib_sriov *sriov = &dev->sriov; + struct rb_root *sl_id_map = &sriov->sl_id_map; + int pv_id = (int) ent->pv_cm_id; + + spin_lock(&sriov->id_map_lock); + db_ent = (struct id_map_entry *)idr_find(&sriov->pv_id_table, pv_id); + if (!db_ent) + goto out; + found_ent = id_map_find_by_sl_id(&dev->ib_dev, ent->slave_id, ent->sl_cm_id); + if (found_ent && found_ent == ent) + rb_erase(&found_ent->node, sl_id_map); + idr_remove(&sriov->pv_id_table, pv_id); + +out: + list_del(&ent->list); + spin_unlock(&sriov->id_map_lock); + kfree(ent); +} + +static void id_map_find_del(struct ib_device *ibdev, int pv_cm_id) +{ + struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; + struct rb_root *sl_id_map = &sriov->sl_id_map; + struct id_map_entry *ent, *found_ent; + + spin_lock(&sriov->id_map_lock); + ent = (struct id_map_entry *)idr_find(&sriov->pv_id_table, pv_cm_id); + if (!ent) + goto out; + found_ent = id_map_find_by_sl_id(ibdev, ent->slave_id, ent->sl_cm_id); + if (found_ent && found_ent == ent) + rb_erase(&found_ent->node, sl_id_map); + idr_remove(&sriov->pv_id_table, pv_cm_id); +out: + spin_unlock(&sriov->id_map_lock); +} + +static void sl_id_map_add(struct ib_device *ibdev, struct id_map_entry *new) +{ + struct rb_root *sl_id_map = &to_mdev(ibdev)->sriov.sl_id_map; + struct rb_node **link = &sl_id_map->rb_node, *parent = NULL; + struct id_map_entry *ent; + int slave_id = new->slave_id; + int sl_cm_id = new->sl_cm_id; + + ent = id_map_find_by_sl_id(ibdev, slave_id, sl_cm_id); + if (ent) { + pr_debug("overriding existing sl_id_map entry (cm_id = %x)\n", + sl_cm_id); + + rb_replace_node(&ent->node, &new->node, sl_id_map); + return; + } + + /* Go to the bottom of the tree */ + while (*link) { + parent = *link; + ent = rb_entry(parent, struct id_map_entry, node); + + if (ent->sl_cm_id > sl_cm_id || (ent->sl_cm_id == sl_cm_id && ent->slave_id > slave_id)) + link = &(*link)->rb_left; + else + link = &(*link)->rb_right; + } + + rb_link_node(&new->node, parent, link); + rb_insert_color(&new->node, sl_id_map); +} + +static struct id_map_entry * +id_map_alloc(struct ib_device *ibdev, int slave_id, u32 sl_cm_id) +{ + int ret, id; + static int next_id; + struct id_map_entry *ent; + struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; + + ent = kmalloc(sizeof (struct id_map_entry), GFP_KERNEL); + if (!ent) { + mlx4_ib_warn(ibdev, "Couldn't allocate id cache entry - out of memory\n"); + return ERR_PTR(-ENOMEM); + } + + ent->sl_cm_id = sl_cm_id; + ent->slave_id = slave_id; + ent->scheduled_delete = 0; + ent->dev = to_mdev(ibdev); + INIT_DELAYED_WORK(&ent->timeout, id_map_ent_timeout); + + do { + spin_lock(&to_mdev(ibdev)->sriov.id_map_lock); + ret = idr_get_new_above(&sriov->pv_id_table, ent, + next_id, &id); + if (!ret) { + next_id = ((unsigned) id + 1) & MAX_IDR_MASK; + ent->pv_cm_id = (u32)id; + sl_id_map_add(ibdev, ent); + } + + spin_unlock(&sriov->id_map_lock); + } while (ret == -EAGAIN && idr_pre_get(&sriov->pv_id_table, GFP_KERNEL)); + /*the function idr_get_new_above can return -ENOSPC, so don't insert in that case.*/ + if (!ret) { + spin_lock(&sriov->id_map_lock); + list_add_tail(&ent->list, &sriov->cm_list); + spin_unlock(&sriov->id_map_lock); + return ent; + } + /*error flow*/ + kfree(ent); + mlx4_ib_warn(ibdev, "No more space in the idr (err:0x%x)\n", ret); + return ERR_PTR(-ENOMEM); +} + +static struct id_map_entry * +id_map_get(struct ib_device *ibdev, int *pv_cm_id, int sl_cm_id, int slave_id) +{ + struct id_map_entry *ent; + struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; + + spin_lock(&sriov->id_map_lock); + if (*pv_cm_id == -1) { + ent = id_map_find_by_sl_id(ibdev, sl_cm_id, slave_id); + if (ent) + *pv_cm_id = (int) ent->pv_cm_id; + } else + ent = (struct id_map_entry *)idr_find(&sriov->pv_id_table, *pv_cm_id); + spin_unlock(&sriov->id_map_lock); + + return ent; +} + +static void schedule_delayed(struct ib_device *ibdev, struct id_map_entry *id) +{ + struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; + unsigned long flags; + + spin_lock(&sriov->id_map_lock); + spin_lock_irqsave(&sriov->going_down_lock, flags); + /*make sure that there is no schedule inside the scheduled work.*/ + if (!sriov->is_going_down) { + id->scheduled_delete = 1; + schedule_delayed_work(&id->timeout, CM_CLEANUP_CACHE_TIMEOUT); + } + spin_unlock_irqrestore(&sriov->going_down_lock, flags); + spin_unlock(&sriov->id_map_lock); +} + +int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id, + struct ib_mad *mad) +{ + struct id_map_entry *id; + u32 sl_cm_id; + int pv_cm_id = -1; + + sl_cm_id = get_local_comm_id(mad); + + if (mad->mad_hdr.attr_id == CM_REQ_ATTR_ID || + mad->mad_hdr.attr_id == CM_REP_ATTR_ID) { + id = id_map_alloc(ibdev, slave_id, sl_cm_id); + if (IS_ERR(id)) { + mlx4_ib_warn(ibdev, "%s: id{slave: %d, sl_cm_id: 0x%x} Failed to id_map_alloc\n", + __func__, slave_id, sl_cm_id); + return PTR_ERR(id); + } + } else if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID) { + return 0; + } else { + id = id_map_get(ibdev, &pv_cm_id, slave_id, sl_cm_id); + } + + if (!id) { + pr_debug("id{slave: %d, sl_cm_id: 0x%x} is NULL!\n", + slave_id, sl_cm_id); + return -EINVAL; + } + + set_local_comm_id(mad, id->pv_cm_id); + + if (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID) + schedule_delayed(ibdev, id); + else if (mad->mad_hdr.attr_id == CM_DREP_ATTR_ID) + id_map_find_del(ibdev, pv_cm_id); + + return 0; +} + +int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave, + struct ib_mad *mad, int is_eth) +{ + u32 pv_cm_id; + struct id_map_entry *id; + + if (mad->mad_hdr.attr_id == CM_REQ_ATTR_ID) { + union ib_gid gid; + + if (is_eth) + return 0; + + gid = gid_from_req_msg(ibdev, mad); + *slave = mlx4_ib_find_real_gid(ibdev, port, gid.global.interface_id); + if (*slave < 0) { + mlx4_ib_warn(ibdev, "failed matching slave_id by gid (0x%llx)\n", + (long long)gid.global.interface_id); + return -ENOENT; + } + return 0; + } + + pv_cm_id = get_remote_comm_id(mad); + id = id_map_get(ibdev, (int *)&pv_cm_id, -1, -1); + + if (!id) { + pr_debug("Couldn't find an entry for pv_cm_id 0x%x\n", pv_cm_id); + return -ENOENT; + } + + if (!is_eth) + *slave = id->slave_id; + set_remote_comm_id(mad, id->sl_cm_id); + + if (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID) + schedule_delayed(ibdev, id); + else if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID || + mad->mad_hdr.attr_id == CM_DREP_ATTR_ID) { + id_map_find_del(ibdev, (int) pv_cm_id); + } + + return 0; +} + +void mlx4_ib_cm_paravirt_init(struct mlx4_ib_dev *dev) +{ + spin_lock_init(&dev->sriov.id_map_lock); + INIT_LIST_HEAD(&dev->sriov.cm_list); + dev->sriov.sl_id_map = RB_ROOT; + idr_init(&dev->sriov.pv_id_table); + idr_pre_get(&dev->sriov.pv_id_table, GFP_KERNEL); +} + +/* slave = -1 ==> all slaves */ +/* TBD -- call paravirt clean for single slave. Need for slave RESET event */ +void mlx4_ib_cm_paravirt_clean(struct mlx4_ib_dev *dev, int slave) +{ + struct mlx4_ib_sriov *sriov = &dev->sriov; + struct rb_root *sl_id_map = &sriov->sl_id_map; + struct list_head lh; + struct rb_node *nd; + int need_flush = 1; + struct id_map_entry *map, *tmp_map; + /* cancel all delayed work queue entries */ + INIT_LIST_HEAD(&lh); + spin_lock(&sriov->id_map_lock); + list_for_each_entry_safe(map, tmp_map, &dev->sriov.cm_list, list) { + if (slave < 0 || slave == map->slave_id) { + if (map->scheduled_delete) + need_flush &= !!cancel_delayed_work(&map->timeout); + } + } + + spin_unlock(&sriov->id_map_lock); + + if (!need_flush) + flush_scheduled_work(); /* make sure all timers were flushed */ + + /* now, remove all leftover entries from databases*/ + spin_lock(&sriov->id_map_lock); + if (slave < 0) { + while (rb_first(sl_id_map)) { + struct id_map_entry *ent = + rb_entry(rb_first(sl_id_map), + struct id_map_entry, node); + + rb_erase(&ent->node, sl_id_map); + idr_remove(&sriov->pv_id_table, (int) ent->pv_cm_id); + } + list_splice_init(&dev->sriov.cm_list, &lh); + } else { + /* first, move nodes belonging to slave to db remove list */ + nd = rb_first(sl_id_map); + while (nd) { + struct id_map_entry *ent = + rb_entry(nd, struct id_map_entry, node); + nd = rb_next(nd); + if (ent->slave_id == slave) + list_move_tail(&ent->list, &lh); + } + /* remove those nodes from databases */ + list_for_each_entry_safe(map, tmp_map, &lh, list) { + rb_erase(&map->node, sl_id_map); + idr_remove(&sriov->pv_id_table, (int) map->pv_cm_id); + } + + /* add remaining nodes from cm_list */ + list_for_each_entry_safe(map, tmp_map, &dev->sriov.cm_list, list) { + if (slave == map->slave_id) + list_move_tail(&map->list, &lh); + } + } + + spin_unlock(&sriov->id_map_lock); + + /* free any map entries left behind due to cancel_delayed_work above */ + list_for_each_entry_safe(map, tmp_map, &lh, list) { + list_del(&map->list); + kfree(map); + } +} Property changes on: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/cm.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/cq.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/cq.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/cq.c (revision 273864) @@ -1,861 +1,897 @@ /* * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include -#include +#include #include "mlx4_ib.h" #include "user.h" /* Which firmware version adds support for Resize CQ */ #define MLX4_FW_VER_RESIZE_CQ mlx4_fw_ver(2, 5, 0) +#define MLX4_FW_VER_IGNORE_OVERRUN_CQ mlx4_fw_ver(2, 7, 8200) static void mlx4_ib_cq_comp(struct mlx4_cq *cq) { struct ib_cq *ibcq = &to_mibcq(cq)->ibcq; ibcq->comp_handler(ibcq, ibcq->cq_context); } static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type) { struct ib_event event; struct ib_cq *ibcq; if (type != MLX4_EVENT_TYPE_CQ_ERROR) { - printk(KERN_WARNING "mlx4_ib: Unexpected event type %d " + pr_warn("Unexpected event type %d " "on CQ %06x\n", type, cq->cqn); return; } ibcq = &to_mibcq(cq)->ibcq; if (ibcq->event_handler) { event.device = ibcq->device; event.event = IB_EVENT_CQ_ERR; event.element.cq = ibcq; ibcq->event_handler(&event, ibcq->cq_context); } } static void *get_cqe_from_buf(struct mlx4_ib_cq_buf *buf, int n) { - return mlx4_buf_offset(&buf->buf, n * sizeof (struct mlx4_cqe)); + return mlx4_buf_offset(&buf->buf, n * buf->entry_size); } static void *get_cqe(struct mlx4_ib_cq *cq, int n) { return get_cqe_from_buf(&cq->buf, n); } static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n) { struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe); + struct mlx4_cqe *tcqe = ((cq->buf.entry_size == 64) ? (cqe + 1) : cqe); - return (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^ + return (!!(tcqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^ !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe; } static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq) { return get_sw_cqe(cq, cq->mcq.cons_index); } int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) { struct mlx4_ib_cq *mcq = to_mcq(cq); struct mlx4_ib_dev *dev = to_mdev(cq->device); return mlx4_cq_modify(dev->dev, &mcq->mcq, cq_count, cq_period); } static int mlx4_ib_alloc_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int nent) { int err; - err = mlx4_buf_alloc(dev->dev, nent * sizeof(struct mlx4_cqe), + err = mlx4_buf_alloc(dev->dev, nent * dev->dev->caps.cqe_size, PAGE_SIZE * 2, &buf->buf); if (err) goto out; + buf->entry_size = dev->dev->caps.cqe_size; err = mlx4_mtt_init(dev->dev, buf->buf.npages, buf->buf.page_shift, &buf->mtt); if (err) goto err_buf; err = mlx4_buf_write_mtt(dev->dev, &buf->mtt, &buf->buf); if (err) goto err_mtt; return 0; err_mtt: mlx4_mtt_cleanup(dev->dev, &buf->mtt); err_buf: - mlx4_buf_free(dev->dev, nent * sizeof(struct mlx4_cqe), - &buf->buf); + mlx4_buf_free(dev->dev, nent * buf->entry_size, &buf->buf); out: return err; } static void mlx4_ib_free_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int cqe) { - mlx4_buf_free(dev->dev, (cqe + 1) * sizeof(struct mlx4_cqe), &buf->buf); + mlx4_buf_free(dev->dev, (cqe + 1) * buf->entry_size, &buf->buf); } static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_ucontext *context, struct mlx4_ib_cq_buf *buf, struct ib_umem **umem, u64 buf_addr, int cqe) { int err; + int cqe_size = dev->dev->caps.cqe_size; + int shift; + int n; - *umem = ib_umem_get(context, buf_addr, cqe * sizeof (struct mlx4_cqe), + *umem = ib_umem_get(context, buf_addr, cqe * cqe_size, IB_ACCESS_LOCAL_WRITE, 1); if (IS_ERR(*umem)) return PTR_ERR(*umem); - err = mlx4_mtt_init(dev->dev, ib_umem_page_count(*umem), - ilog2((*umem)->page_size), &buf->mtt); + n = ib_umem_page_count(*umem); + shift = mlx4_ib_umem_calc_optimal_mtt_size(*umem, 0, &n); + err = mlx4_mtt_init(dev->dev, n, shift, &buf->mtt); + if (err) goto err_buf; err = mlx4_ib_umem_write_mtt(dev, &buf->mtt, *umem); if (err) goto err_mtt; return 0; err_mtt: mlx4_mtt_cleanup(dev->dev, &buf->mtt); err_buf: ib_umem_release(*umem); return err; } struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector, struct ib_ucontext *context, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(ibdev); struct mlx4_ib_cq *cq; struct mlx4_uar *uar; int err; - if (entries < 1 || entries > dev->dev->caps.max_cqes) { - mlx4_ib_dbg("invalid num of entries: %d", entries); + if (entries < 1 || entries > dev->dev->caps.max_cqes) return ERR_PTR(-EINVAL); - } - cq = kzalloc(sizeof *cq, GFP_KERNEL); + cq = kmalloc(sizeof *cq, GFP_KERNEL); if (!cq) return ERR_PTR(-ENOMEM); entries = roundup_pow_of_two(entries + 1); cq->ibcq.cqe = entries - 1; mutex_init(&cq->resize_mutex); spin_lock_init(&cq->lock); cq->resize_buf = NULL; cq->resize_umem = NULL; if (context) { struct mlx4_ib_create_cq ucmd; if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { err = -EFAULT; goto err_cq; } err = mlx4_ib_get_cq_umem(dev, context, &cq->buf, &cq->umem, ucmd.buf_addr, entries); if (err) goto err_cq; err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr, &cq->db); if (err) goto err_mtt; uar = &to_mucontext(context)->uar; } else { err = mlx4_db_alloc(dev->dev, &cq->db, 1); if (err) goto err_cq; cq->mcq.set_ci_db = cq->db.db; cq->mcq.arm_db = cq->db.db + 1; *cq->mcq.set_ci_db = 0; *cq->mcq.arm_db = 0; err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries); if (err) goto err_db; uar = &dev->priv_uar; } + if (dev->eq_table) + vector = dev->eq_table[vector % ibdev->num_comp_vectors]; + err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar, - cq->db.dma, &cq->mcq, - vector == IB_CQ_VECTOR_LEAST_ATTACHED ? - MLX4_LEAST_ATTACHED_VECTOR : vector, 0); + cq->db.dma, &cq->mcq, vector, 0, 0); if (err) goto err_dbmap; cq->mcq.comp = mlx4_ib_cq_comp; cq->mcq.event = mlx4_ib_cq_event; if (context) if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) { err = -EFAULT; goto err_dbmap; } return &cq->ibcq; err_dbmap: if (context) mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db); err_mtt: mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt); if (context) ib_umem_release(cq->umem); else mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); err_db: if (!context) mlx4_db_free(dev->dev, &cq->db); err_cq: kfree(cq); return ERR_PTR(err); } static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, int entries) { int err; if (cq->resize_buf) return -EBUSY; cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC); if (!cq->resize_buf) return -ENOMEM; err = mlx4_ib_alloc_cq_buf(dev, &cq->resize_buf->buf, entries); if (err) { kfree(cq->resize_buf); cq->resize_buf = NULL; return err; } cq->resize_buf->cqe = entries - 1; return 0; } static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, int entries, struct ib_udata *udata) { struct mlx4_ib_resize_cq ucmd; int err; if (cq->resize_umem) return -EBUSY; if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) return -EFAULT; cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC); if (!cq->resize_buf) return -ENOMEM; err = mlx4_ib_get_cq_umem(dev, cq->umem->context, &cq->resize_buf->buf, &cq->resize_umem, ucmd.buf_addr, entries); if (err) { kfree(cq->resize_buf); cq->resize_buf = NULL; return err; } cq->resize_buf->cqe = entries - 1; return 0; } static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq) { u32 i; i = cq->mcq.cons_index; while (get_sw_cqe(cq, i & cq->ibcq.cqe)) ++i; return i - cq->mcq.cons_index; } static void mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq) { struct mlx4_cqe *cqe, *new_cqe; int i; + int cqe_size = cq->buf.entry_size; + int cqe_inc = cqe_size == 64 ? 1 : 0; i = cq->mcq.cons_index; cqe = get_cqe(cq, i & cq->ibcq.cqe); + cqe += cqe_inc; + while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) { new_cqe = get_cqe_from_buf(&cq->resize_buf->buf, (i + 1) & cq->resize_buf->cqe); - memcpy(new_cqe, get_cqe(cq, i & cq->ibcq.cqe), sizeof(struct mlx4_cqe)); + memcpy(new_cqe, get_cqe(cq, i & cq->ibcq.cqe), cqe_size); + new_cqe += cqe_inc; + new_cqe->owner_sr_opcode = (cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK) | (((i + 1) & (cq->resize_buf->cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0); cqe = get_cqe(cq, ++i & cq->ibcq.cqe); + cqe += cqe_inc; } ++cq->mcq.cons_index; } int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(ibcq->device); struct mlx4_ib_cq *cq = to_mcq(ibcq); struct mlx4_mtt mtt; int outst_cqe; int err; if (dev->dev->caps.fw_ver < MLX4_FW_VER_RESIZE_CQ) return -ENOSYS; mutex_lock(&cq->resize_mutex); if (entries < 1 || entries > dev->dev->caps.max_cqes) { err = -EINVAL; goto out; } entries = roundup_pow_of_two(entries + 1); if (entries == ibcq->cqe + 1) { err = 0; goto out; } if (ibcq->uobject) { err = mlx4_alloc_resize_umem(dev, cq, entries, udata); if (err) goto out; } else { /* Can't be smaller than the number of outstanding CQEs */ outst_cqe = mlx4_ib_get_outstanding_cqes(cq); if (entries < outst_cqe + 1) { err = 0; goto out; } err = mlx4_alloc_resize_buf(dev, cq, entries); if (err) goto out; } mtt = cq->buf.mtt; err = mlx4_cq_resize(dev->dev, &cq->mcq, entries, &cq->resize_buf->buf.mtt); if (err) goto err_buf; mlx4_mtt_cleanup(dev->dev, &mtt); if (ibcq->uobject) { cq->buf = cq->resize_buf->buf; cq->ibcq.cqe = cq->resize_buf->cqe; ib_umem_release(cq->umem); cq->umem = cq->resize_umem; kfree(cq->resize_buf); cq->resize_buf = NULL; cq->resize_umem = NULL; } else { struct mlx4_ib_cq_buf tmp_buf; int tmp_cqe = 0; - + spin_lock_irq(&cq->lock); if (cq->resize_buf) { mlx4_ib_cq_resize_copy_cqes(cq); tmp_buf = cq->buf; tmp_cqe = cq->ibcq.cqe; cq->buf = cq->resize_buf->buf; cq->ibcq.cqe = cq->resize_buf->cqe; kfree(cq->resize_buf); cq->resize_buf = NULL; } spin_unlock_irq(&cq->lock); if (tmp_cqe) mlx4_ib_free_cq_buf(dev, &tmp_buf, tmp_cqe); } goto out; err_buf: mlx4_mtt_cleanup(dev->dev, &cq->resize_buf->buf.mtt); if (!ibcq->uobject) mlx4_ib_free_cq_buf(dev, &cq->resize_buf->buf, cq->resize_buf->cqe); kfree(cq->resize_buf); cq->resize_buf = NULL; if (cq->resize_umem) { ib_umem_release(cq->resize_umem); cq->resize_umem = NULL; } out: mutex_unlock(&cq->resize_mutex); + return err; } +int mlx4_ib_ignore_overrun_cq(struct ib_cq *ibcq) +{ + struct mlx4_ib_dev *dev = to_mdev(ibcq->device); + struct mlx4_ib_cq *cq = to_mcq(ibcq); + + if (dev->dev->caps.fw_ver < MLX4_FW_VER_IGNORE_OVERRUN_CQ) + return -ENOSYS; + + return mlx4_cq_ignore_overrun(dev->dev, &cq->mcq); +} + int mlx4_ib_destroy_cq(struct ib_cq *cq) { struct mlx4_ib_dev *dev = to_mdev(cq->device); struct mlx4_ib_cq *mcq = to_mcq(cq); mlx4_cq_free(dev->dev, &mcq->mcq); mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt); if (cq->uobject) { mlx4_ib_db_unmap_user(to_mucontext(cq->uobject->context), &mcq->db); ib_umem_release(mcq->umem); } else { mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe); mlx4_db_free(dev->dev, &mcq->db); } kfree(mcq); return 0; } static void dump_cqe(void *cqe) { __be32 *buf = cqe; - printk(KERN_DEBUG "CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n", + pr_debug("CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n", be32_to_cpu(buf[0]), be32_to_cpu(buf[1]), be32_to_cpu(buf[2]), be32_to_cpu(buf[3]), be32_to_cpu(buf[4]), be32_to_cpu(buf[5]), be32_to_cpu(buf[6]), be32_to_cpu(buf[7])); } static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe, struct ib_wc *wc) { if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) { - printk(KERN_DEBUG "local QP operation err " + pr_debug("local QP operation err " "(QPN %06x, WQE index %x, vendor syndrome %02x, " "opcode = %02x)\n", be32_to_cpu(cqe->my_qpn), be16_to_cpu(cqe->wqe_index), cqe->vendor_err_syndrome, cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); dump_cqe(cqe); } switch (cqe->syndrome) { case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR: wc->status = IB_WC_LOC_LEN_ERR; break; case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR: wc->status = IB_WC_LOC_QP_OP_ERR; break; case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR: wc->status = IB_WC_LOC_PROT_ERR; break; case MLX4_CQE_SYNDROME_WR_FLUSH_ERR: wc->status = IB_WC_WR_FLUSH_ERR; break; case MLX4_CQE_SYNDROME_MW_BIND_ERR: wc->status = IB_WC_MW_BIND_ERR; break; case MLX4_CQE_SYNDROME_BAD_RESP_ERR: wc->status = IB_WC_BAD_RESP_ERR; break; case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR: wc->status = IB_WC_LOC_ACCESS_ERR; break; case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: wc->status = IB_WC_REM_INV_REQ_ERR; break; case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR: wc->status = IB_WC_REM_ACCESS_ERR; break; case MLX4_CQE_SYNDROME_REMOTE_OP_ERR: wc->status = IB_WC_REM_OP_ERR; break; case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: wc->status = IB_WC_RETRY_EXC_ERR; break; case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR: wc->status = IB_WC_RNR_RETRY_EXC_ERR; break; case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR: wc->status = IB_WC_REM_ABORT_ERR; break; default: wc->status = IB_WC_GENERAL_ERR; break; } wc->vendor_err = cqe->vendor_err_syndrome; } static int mlx4_ib_ipoib_csum_ok(__be16 status, __be16 checksum) { return ((status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 | MLX4_CQE_STATUS_IPV4F | MLX4_CQE_STATUS_IPV4OPT | MLX4_CQE_STATUS_IPV6 | MLX4_CQE_STATUS_IPOK)) == cpu_to_be16(MLX4_CQE_STATUS_IPV4 | MLX4_CQE_STATUS_IPOK)) && (status & cpu_to_be16(MLX4_CQE_STATUS_UDP | MLX4_CQE_STATUS_TCP)) && checksum == cpu_to_be16(0xffff); } +static int use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc, + unsigned tail, struct mlx4_cqe *cqe) +{ + struct mlx4_ib_proxy_sqp_hdr *hdr; + + ib_dma_sync_single_for_cpu(qp->ibqp.device, + qp->sqp_proxy_rcv[tail].map, + sizeof (struct mlx4_ib_proxy_sqp_hdr), + DMA_FROM_DEVICE); + hdr = (struct mlx4_ib_proxy_sqp_hdr *) (qp->sqp_proxy_rcv[tail].addr); + wc->pkey_index = be16_to_cpu(hdr->tun.pkey_index); + wc->slid = be16_to_cpu(hdr->tun.slid_mac_47_32); + wc->sl = (u8) (be16_to_cpu(hdr->tun.sl_vid) >> 12); + wc->src_qp = be32_to_cpu(hdr->tun.flags_src_qp) & 0xFFFFFF; + wc->wc_flags |= (hdr->tun.g_ml_path & 0x80) ? (IB_WC_GRH) : 0; + wc->dlid_path_bits = 0; + + return 0; +} + static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq, struct mlx4_ib_qp **cur_qp, struct ib_wc *wc) { struct mlx4_cqe *cqe; struct mlx4_qp *mqp; struct mlx4_ib_wq *wq; struct mlx4_ib_srq *srq; - struct mlx4_srq *msrq; int is_send; int is_error; u32 g_mlpath_rqpn; - int is_xrc_recv = 0; u16 wqe_ctr; + unsigned tail = 0; repoll: cqe = next_cqe_sw(cq); if (!cqe) return -EAGAIN; + if (cq->buf.entry_size == 64) + cqe++; + ++cq->mcq.cons_index; /* * Make sure we read CQ entry contents after we've checked the * ownership bit. */ rmb(); is_send = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK; is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_ERROR; if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_OPCODE_NOP && is_send)) { - printk(KERN_WARNING "Completion for NOP opcode detected!\n"); + pr_warn("Completion for NOP opcode detected!\n"); return -EINVAL; } /* Resize CQ in progress */ if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_RESIZE)) { if (cq->resize_buf) { struct mlx4_ib_dev *dev = to_mdev(cq->ibcq.device); mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); cq->buf = cq->resize_buf->buf; cq->ibcq.cqe = cq->resize_buf->cqe; kfree(cq->resize_buf); cq->resize_buf = NULL; } goto repoll; } - if ((be32_to_cpu(cqe->vlan_my_qpn) & (1 << 23)) && !is_send) { - /* - * We do not have to take the XRC SRQ table lock here, - * because CQs will be locked while XRC SRQs are removed - * from the table. - */ - msrq = __mlx4_srq_lookup(to_mdev(cq->ibcq.device)->dev, - be32_to_cpu(cqe->g_mlpath_rqpn) & - 0xffffff); - if (unlikely(!msrq)) { - printk(KERN_WARNING "CQ %06x with entry for unknown " - "XRC SRQ %06x\n", cq->mcq.cqn, - be32_to_cpu(cqe->g_mlpath_rqpn) & 0xffffff); - return -EINVAL; - } - is_xrc_recv = 1; - srq = to_mibsrq(msrq); - } else if (!*cur_qp || + if (!*cur_qp || (be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) != (*cur_qp)->mqp.qpn) { /* * We do not have to take the QP table lock here, * because CQs will be locked while QPs are removed * from the table. */ mqp = __mlx4_qp_lookup(to_mdev(cq->ibcq.device)->dev, be32_to_cpu(cqe->vlan_my_qpn)); if (unlikely(!mqp)) { - printk(KERN_WARNING "CQ %06x with entry for unknown QPN %06x\n", + pr_warn("CQ %06x with entry for unknown QPN %06x\n", cq->mcq.cqn, be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK); return -EINVAL; } *cur_qp = to_mibqp(mqp); } - wc->qp = is_xrc_recv ? NULL: &(*cur_qp)->ibqp; + wc->qp = &(*cur_qp)->ibqp; if (is_send) { wq = &(*cur_qp)->sq; if (!(*cur_qp)->sq_signal_bits) { wqe_ctr = be16_to_cpu(cqe->wqe_index); wq->tail += (u16) (wqe_ctr - (u16) wq->tail); } wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; ++wq->tail; - } else if (is_xrc_recv) { - wqe_ctr = be16_to_cpu(cqe->wqe_index); - wc->wr_id = srq->wrid[wqe_ctr]; - mlx4_ib_free_srq_wqe(srq, wqe_ctr); } else if ((*cur_qp)->ibqp.srq) { srq = to_msrq((*cur_qp)->ibqp.srq); wqe_ctr = be16_to_cpu(cqe->wqe_index); wc->wr_id = srq->wrid[wqe_ctr]; mlx4_ib_free_srq_wqe(srq, wqe_ctr); } else { wq = &(*cur_qp)->rq; - wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + tail = wq->tail & (wq->wqe_cnt - 1); + wc->wr_id = wq->wrid[tail]; ++wq->tail; } if (unlikely(is_error)) { mlx4_ib_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc); return 0; } wc->status = IB_WC_SUCCESS; if (is_send) { wc->wc_flags = 0; switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { case MLX4_OPCODE_RDMA_WRITE_IMM: wc->wc_flags |= IB_WC_WITH_IMM; case MLX4_OPCODE_RDMA_WRITE: wc->opcode = IB_WC_RDMA_WRITE; break; case MLX4_OPCODE_SEND_IMM: wc->wc_flags |= IB_WC_WITH_IMM; case MLX4_OPCODE_SEND: case MLX4_OPCODE_SEND_INVAL: wc->opcode = IB_WC_SEND; break; case MLX4_OPCODE_RDMA_READ: wc->opcode = IB_WC_RDMA_READ; wc->byte_len = be32_to_cpu(cqe->byte_cnt); break; case MLX4_OPCODE_ATOMIC_CS: wc->opcode = IB_WC_COMP_SWAP; wc->byte_len = 8; break; case MLX4_OPCODE_ATOMIC_FA: wc->opcode = IB_WC_FETCH_ADD; wc->byte_len = 8; break; case MLX4_OPCODE_MASKED_ATOMIC_CS: wc->opcode = IB_WC_MASKED_COMP_SWAP; wc->byte_len = 8; break; case MLX4_OPCODE_MASKED_ATOMIC_FA: wc->opcode = IB_WC_MASKED_FETCH_ADD; wc->byte_len = 8; break; case MLX4_OPCODE_BIND_MW: wc->opcode = IB_WC_BIND_MW; break; case MLX4_OPCODE_LSO: wc->opcode = IB_WC_LSO; break; case MLX4_OPCODE_FMR: wc->opcode = IB_WC_FAST_REG_MR; break; case MLX4_OPCODE_LOCAL_INVAL: wc->opcode = IB_WC_LOCAL_INV; break; } } else { wc->byte_len = be32_to_cpu(cqe->byte_cnt); switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { case MLX4_RECV_OPCODE_RDMA_WRITE_IMM: wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; wc->wc_flags = IB_WC_WITH_IMM; wc->ex.imm_data = cqe->immed_rss_invalid; break; case MLX4_RECV_OPCODE_SEND_INVAL: wc->opcode = IB_WC_RECV; wc->wc_flags = IB_WC_WITH_INVALIDATE; wc->ex.invalidate_rkey = be32_to_cpu(cqe->immed_rss_invalid); break; case MLX4_RECV_OPCODE_SEND: wc->opcode = IB_WC_RECV; wc->wc_flags = 0; break; case MLX4_RECV_OPCODE_SEND_IMM: wc->opcode = IB_WC_RECV; wc->wc_flags = IB_WC_WITH_IMM; wc->ex.imm_data = cqe->immed_rss_invalid; break; } + if (mlx4_is_mfunc(to_mdev(cq->ibcq.device)->dev)) { + if ((*cur_qp)->mlx4_ib_qp_type & + (MLX4_IB_QPT_PROXY_SMI_OWNER | + MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) + return use_tunnel_data(*cur_qp, cq, wc, tail, cqe); + } + wc->slid = be16_to_cpu(cqe->rlid); - wc->sl = be16_to_cpu(cqe->sl_vid) >> 12; g_mlpath_rqpn = be32_to_cpu(cqe->g_mlpath_rqpn); wc->src_qp = g_mlpath_rqpn & 0xffffff; wc->dlid_path_bits = (g_mlpath_rqpn >> 24) & 0x7f; wc->wc_flags |= g_mlpath_rqpn & 0x80000000 ? IB_WC_GRH : 0; wc->pkey_index = be32_to_cpu(cqe->immed_rss_invalid) & 0x7f; - wc->csum_ok = mlx4_ib_ipoib_csum_ok(cqe->status, cqe->checksum); + wc->wc_flags |= mlx4_ib_ipoib_csum_ok(cqe->status, + cqe->checksum) ? IB_WC_IP_CSUM_OK : 0; + if (rdma_port_get_link_layer(wc->qp->device, + (*cur_qp)->port) == IB_LINK_LAYER_ETHERNET) + wc->sl = be16_to_cpu(cqe->sl_vid) >> 13; + else + wc->sl = be16_to_cpu(cqe->sl_vid) >> 12; } return 0; } int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) { struct mlx4_ib_cq *cq = to_mcq(ibcq); struct mlx4_ib_qp *cur_qp = NULL; unsigned long flags; int npolled; int err = 0; spin_lock_irqsave(&cq->lock, flags); for (npolled = 0; npolled < num_entries; ++npolled) { err = mlx4_ib_poll_one(cq, &cur_qp, wc + npolled); if (err) break; } - if (npolled) - mlx4_cq_set_ci(&cq->mcq); + mlx4_cq_set_ci(&cq->mcq); spin_unlock_irqrestore(&cq->lock, flags); if (err == 0 || err == -EAGAIN) return npolled; else return err; } int mlx4_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { mlx4_cq_arm(&to_mcq(ibcq)->mcq, (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT, to_mdev(ibcq->device)->priv_uar.map, MLX4_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->uar_lock)); return 0; } void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) { u32 prod_index; int nfreed = 0; struct mlx4_cqe *cqe, *dest; u8 owner_bit; - int is_xrc_srq = 0; + int cqe_inc = cq->buf.entry_size == 64 ? 1 : 0; - if (srq && srq->ibsrq.xrc_cq) - is_xrc_srq = 1; - /* * First we need to find the current producer index, so we * know where to start cleaning from. It doesn't matter if HW * adds new entries after this loop -- the QP we're worried * about is already in RESET, so the new entries won't come * from our QP and therefore don't need to be checked. */ for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index) if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe) break; /* * Now sweep backwards through the CQ, removing CQ entries * that match our QP by copying older entries on top of them. */ while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); - if (((be32_to_cpu(cqe->vlan_my_qpn) & 0xffffff) == qpn) || - (is_xrc_srq && - (be32_to_cpu(cqe->g_mlpath_rqpn) & 0xffffff) == - srq->msrq.srqn)) { + cqe += cqe_inc; + + if ((be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) == qpn) { if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index)); ++nfreed; } else if (nfreed) { dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe); + dest += cqe_inc; + owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK; memcpy(dest, cqe, sizeof *cqe); dest->owner_sr_opcode = owner_bit | (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); } } if (nfreed) { cq->mcq.cons_index += nfreed; /* * Make sure update of buffer contents is done before * updating consumer index. */ wmb(); mlx4_cq_set_ci(&cq->mcq); } } void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) { spin_lock_irq(&cq->lock); __mlx4_ib_cq_clean(cq, qpn, srq); spin_unlock_irq(&cq->lock); } Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/mad.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/mad.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/mad.c (revision 273864) @@ -1,452 +1,2294 @@ /* * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include +#include +#include +#include #include +#include +#include #include "mlx4_ib.h" enum { MLX4_IB_VENDOR_CLASS1 = 0x9, MLX4_IB_VENDOR_CLASS2 = 0xa }; -int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int ignore_mkey, int ignore_bkey, +#define MLX4_TUN_SEND_WRID_SHIFT 34 +#define MLX4_TUN_QPN_SHIFT 32 +#define MLX4_TUN_WRID_RECV (((u64) 1) << MLX4_TUN_SEND_WRID_SHIFT) +#define MLX4_TUN_SET_WRID_QPN(a) (((u64) ((a) & 0x3)) << MLX4_TUN_QPN_SHIFT) + +#define MLX4_TUN_IS_RECV(a) (((a) >> MLX4_TUN_SEND_WRID_SHIFT) & 0x1) +#define MLX4_TUN_WRID_QPN(a) (((a) >> MLX4_TUN_QPN_SHIFT) & 0x3) + + /* Port mgmt change event handling */ + +#define GET_BLK_PTR_FROM_EQE(eqe) be32_to_cpu(eqe->event.port_mgmt_change.params.tbl_change_info.block_ptr) +#define GET_MASK_FROM_EQE(eqe) be32_to_cpu(eqe->event.port_mgmt_change.params.tbl_change_info.tbl_entries_mask) +#define NUM_IDX_IN_PKEY_TBL_BLK 32 +#define GUID_TBL_ENTRY_SIZE 8 /* size in bytes */ +#define GUID_TBL_BLK_NUM_ENTRIES 8 +#define GUID_TBL_BLK_SIZE (GUID_TBL_ENTRY_SIZE * GUID_TBL_BLK_NUM_ENTRIES) + +struct mlx4_mad_rcv_buf { + struct ib_grh grh; + u8 payload[256]; +} __packed; + +struct mlx4_mad_snd_buf { + u8 payload[256]; +} __packed; + +struct mlx4_tunnel_mad { + struct ib_grh grh; + struct mlx4_ib_tunnel_header hdr; + struct ib_mad mad; +} __packed; + +struct mlx4_rcv_tunnel_mad { + struct mlx4_rcv_tunnel_hdr hdr; + struct ib_grh grh; + struct ib_mad mad; +} __packed; + +static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u8 port_num); +static void handle_lid_change_event(struct mlx4_ib_dev *dev, u8 port_num); +static void __propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num, + int block, u32 change_bitmap); + +__be64 mlx4_ib_gen_node_guid(void) +{ +#define NODE_GUID_HI ((u64) (((u64)IB_OPENIB_OUI) << 40)) + return cpu_to_be64(NODE_GUID_HI | random()); +} + +__be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx) +{ + return cpu_to_be64(atomic_inc_return(&ctx->tid)) | + cpu_to_be64(0xff00000000000000LL); +} + +int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int mad_ifc_flags, int port, struct ib_wc *in_wc, struct ib_grh *in_grh, void *in_mad, void *response_mad) { struct mlx4_cmd_mailbox *inmailbox, *outmailbox; void *inbox; int err; u32 in_modifier = port; u8 op_modifier = 0; inmailbox = mlx4_alloc_cmd_mailbox(dev->dev); if (IS_ERR(inmailbox)) return PTR_ERR(inmailbox); inbox = inmailbox->buf; outmailbox = mlx4_alloc_cmd_mailbox(dev->dev); if (IS_ERR(outmailbox)) { mlx4_free_cmd_mailbox(dev->dev, inmailbox); return PTR_ERR(outmailbox); } memcpy(inbox, in_mad, 256); /* * Key check traps can't be generated unless we have in_wc to * tell us where to send the trap. */ - if (ignore_mkey || !in_wc) + if ((mad_ifc_flags & MLX4_MAD_IFC_IGNORE_MKEY) || !in_wc) op_modifier |= 0x1; - if (ignore_bkey || !in_wc) + if ((mad_ifc_flags & MLX4_MAD_IFC_IGNORE_BKEY) || !in_wc) op_modifier |= 0x2; + if (mlx4_is_mfunc(dev->dev) && + (mad_ifc_flags & MLX4_MAD_IFC_NET_VIEW || in_wc)) + op_modifier |= 0x8; if (in_wc) { struct { __be32 my_qpn; u32 reserved1; __be32 rqpn; u8 sl; u8 g_path; u16 reserved2[2]; __be16 pkey; u32 reserved3[11]; u8 grh[40]; } *ext_info; memset(inbox + 256, 0, 256); ext_info = inbox + 256; ext_info->my_qpn = cpu_to_be32(in_wc->qp->qp_num); ext_info->rqpn = cpu_to_be32(in_wc->src_qp); ext_info->sl = in_wc->sl << 4; ext_info->g_path = in_wc->dlid_path_bits | (in_wc->wc_flags & IB_WC_GRH ? 0x80 : 0); ext_info->pkey = cpu_to_be16(in_wc->pkey_index); if (in_grh) memcpy(ext_info->grh, in_grh, 40); op_modifier |= 0x4; in_modifier |= in_wc->slid << 16; } - err = mlx4_cmd_box(dev->dev, inmailbox->dma, outmailbox->dma, - in_modifier, op_modifier, - MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C); + err = mlx4_cmd_box(dev->dev, inmailbox->dma, outmailbox->dma, in_modifier, + mlx4_is_master(dev->dev) ? (op_modifier & ~0x8) : op_modifier, + MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C, + (op_modifier & 0x8) ? MLX4_CMD_NATIVE : MLX4_CMD_WRAPPED); if (!err) memcpy(response_mad, outmailbox->buf, 256); mlx4_free_cmd_mailbox(dev->dev, inmailbox); mlx4_free_cmd_mailbox(dev->dev, outmailbox); return err; } static void update_sm_ah(struct mlx4_ib_dev *dev, u8 port_num, u16 lid, u8 sl) { struct ib_ah *new_ah; struct ib_ah_attr ah_attr; + unsigned long flags; if (!dev->send_agent[port_num - 1][0]) return; memset(&ah_attr, 0, sizeof ah_attr); ah_attr.dlid = lid; ah_attr.sl = sl; ah_attr.port_num = port_num; new_ah = ib_create_ah(dev->send_agent[port_num - 1][0]->qp->pd, &ah_attr); if (IS_ERR(new_ah)) return; - spin_lock(&dev->sm_lock); + spin_lock_irqsave(&dev->sm_lock, flags); if (dev->sm_ah[port_num - 1]) ib_destroy_ah(dev->sm_ah[port_num - 1]); dev->sm_ah[port_num - 1] = new_ah; - spin_unlock(&dev->sm_lock); + spin_unlock_irqrestore(&dev->sm_lock, flags); } /* - * Snoop SM MADs for port info and P_Key table sets, so we can - * synthesize LID change and P_Key change events. + * Snoop SM MADs for port info, GUID info, and P_Key table sets, so we can + * synthesize LID change, Client-Rereg, GID change, and P_Key change events. */ static void smp_snoop(struct ib_device *ibdev, u8 port_num, struct ib_mad *mad, - u16 prev_lid) + u16 prev_lid) { - struct ib_event event; + struct ib_port_info *pinfo; + u16 lid; + __be16 *base; + u32 bn, pkey_change_bitmap; + int i; + + struct mlx4_ib_dev *dev = to_mdev(ibdev); if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && - mad->mad_hdr.method == IB_MGMT_METHOD_SET) { - if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO) { - struct ib_port_info *pinfo = - (struct ib_port_info *) ((struct ib_smp *) mad)->data; - u16 lid = be16_to_cpu(pinfo->lid); + mad->mad_hdr.method == IB_MGMT_METHOD_SET) + switch (mad->mad_hdr.attr_id) { + case IB_SMP_ATTR_PORT_INFO: + pinfo = (struct ib_port_info *) ((struct ib_smp *) mad)->data; + lid = be16_to_cpu(pinfo->lid); - update_sm_ah(to_mdev(ibdev), port_num, + update_sm_ah(dev, port_num, be16_to_cpu(pinfo->sm_lid), pinfo->neighbormtu_mastersmsl & 0xf); - event.device = ibdev; - event.element.port_num = port_num; + if (pinfo->clientrereg_resv_subnetto & 0x80) + handle_client_rereg_event(dev, port_num); - if (pinfo->clientrereg_resv_subnetto & 0x80) { - event.event = IB_EVENT_CLIENT_REREGISTER; - ib_dispatch_event(&event); + if (prev_lid != lid) + handle_lid_change_event(dev, port_num); + break; + + case IB_SMP_ATTR_PKEY_TABLE: + if (!mlx4_is_mfunc(dev->dev)) { + mlx4_ib_dispatch_event(dev, port_num, + IB_EVENT_PKEY_CHANGE); + break; } - if (prev_lid != lid) { - event.event = IB_EVENT_LID_CHANGE; - ib_dispatch_event(&event); + /* at this point, we are running in the master. + * Slaves do not receive SMPs. + */ + bn = be32_to_cpu(((struct ib_smp *)mad)->attr_mod) & 0xFFFF; + base = (__be16 *) &(((struct ib_smp *)mad)->data[0]); + pkey_change_bitmap = 0; + for (i = 0; i < 32; i++) { + pr_debug("PKEY[%d] = x%x\n", + i + bn*32, be16_to_cpu(base[i])); + if (be16_to_cpu(base[i]) != + dev->pkeys.phys_pkey_cache[port_num - 1][i + bn*32]) { + pkey_change_bitmap |= (1 << i); + dev->pkeys.phys_pkey_cache[port_num - 1][i + bn*32] = + be16_to_cpu(base[i]); + } } + pr_debug("PKEY Change event: port=%d, " + "block=0x%x, change_bitmap=0x%x\n", + port_num, bn, pkey_change_bitmap); + + if (pkey_change_bitmap) { + mlx4_ib_dispatch_event(dev, port_num, + IB_EVENT_PKEY_CHANGE); + if (!dev->sriov.is_going_down) + __propagate_pkey_ev(dev, port_num, bn, + pkey_change_bitmap); + } + break; + + case IB_SMP_ATTR_GUID_INFO: + /* paravirtualized master's guid is guid 0 -- does not change */ + if (!mlx4_is_master(dev->dev)) + mlx4_ib_dispatch_event(dev, port_num, + IB_EVENT_GID_CHANGE); + /*if master, notify relevant slaves*/ + if (mlx4_is_master(dev->dev) && + !dev->sriov.is_going_down) { + bn = be32_to_cpu(((struct ib_smp *)mad)->attr_mod); + mlx4_ib_update_cache_on_guid_change(dev, bn, port_num, + (u8 *)(&((struct ib_smp *)mad)->data)); + mlx4_ib_notify_slaves_on_guid_change(dev, bn, port_num, + (u8 *)(&((struct ib_smp *)mad)->data)); + } + break; + + default: + break; } +} - if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PKEY_TABLE) { - event.device = ibdev; - event.event = IB_EVENT_PKEY_CHANGE; - event.element.port_num = port_num; - ib_dispatch_event(&event); +static void __propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num, + int block, u32 change_bitmap) +{ + int i, ix, slave, err; + int have_event = 0; + + for (slave = 0; slave < dev->dev->caps.sqp_demux; slave++) { + if (slave == mlx4_master_func_num(dev->dev)) + continue; + if (!mlx4_is_slave_active(dev->dev, slave)) + continue; + + have_event = 0; + for (i = 0; i < 32; i++) { + if (!(change_bitmap & (1 << i))) + continue; + for (ix = 0; + ix < dev->dev->caps.pkey_table_len[port_num]; ix++) { + if (dev->pkeys.virt2phys_pkey[slave][port_num - 1] + [ix] == i + 32 * block) { + err = mlx4_gen_pkey_eqe(dev->dev, slave, port_num); + pr_debug("propagate_pkey_ev: slave %d," + " port %d, ix %d (%d)\n", + slave, port_num, ix, err); + have_event = 1; + break; + } + } + if (have_event) + break; } } } static void node_desc_override(struct ib_device *dev, struct ib_mad *mad) { + unsigned long flags; + if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && mad->mad_hdr.method == IB_MGMT_METHOD_GET_RESP && mad->mad_hdr.attr_id == IB_SMP_ATTR_NODE_DESC) { - spin_lock(&to_mdev(dev)->sm_lock); + spin_lock_irqsave(&to_mdev(dev)->sm_lock, flags); memcpy(((struct ib_smp *) mad)->data, dev->node_desc, 64); - spin_unlock(&to_mdev(dev)->sm_lock); + spin_unlock_irqrestore(&to_mdev(dev)->sm_lock, flags); } } static void forward_trap(struct mlx4_ib_dev *dev, u8 port_num, struct ib_mad *mad) { int qpn = mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED; struct ib_mad_send_buf *send_buf; struct ib_mad_agent *agent = dev->send_agent[port_num - 1][qpn]; int ret; + unsigned long flags; if (agent) { send_buf = ib_create_send_mad(agent, qpn, 0, 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA, GFP_ATOMIC); + if (IS_ERR(send_buf)) + return; /* * We rely here on the fact that MLX QPs don't use the * address handle after the send is posted (this is * wrong following the IB spec strictly, but we know * it's OK for our devices). */ - spin_lock(&dev->sm_lock); + spin_lock_irqsave(&dev->sm_lock, flags); memcpy(send_buf->mad, mad, sizeof *mad); if ((send_buf->ah = dev->sm_ah[port_num - 1])) ret = ib_post_send_mad(send_buf, NULL); else ret = -EINVAL; - spin_unlock(&dev->sm_lock); + spin_unlock_irqrestore(&dev->sm_lock, flags); if (ret) ib_free_send_mad(send_buf); } } -static int is_vendor_id(__be16 attr_id) +static int mlx4_ib_demux_sa_handler(struct ib_device *ibdev, int port, int slave, + struct ib_sa_mad *sa_mad) { - return (attr_id & IB_SMP_ATTR_VENDOR_MASK) == IB_SMP_ATTR_VENDOR_MASK; + int ret = 0; + + /* dispatch to different sa handlers */ + switch (be16_to_cpu(sa_mad->mad_hdr.attr_id)) { + case IB_SA_ATTR_MC_MEMBER_REC: + ret = mlx4_ib_mcg_demux_handler(ibdev, port, slave, sa_mad); + break; + default: + break; + } + return ret; } -static int supported_vendor_id(__be16 attr_id) +int mlx4_ib_find_real_gid(struct ib_device *ibdev, u8 port, __be64 guid) { - return 1; + struct mlx4_ib_dev *dev = to_mdev(ibdev); + int i; + + for (i = 0; i < dev->dev->caps.sqp_demux; i++) { + if (dev->sriov.demux[port - 1].guid_cache[i] == guid) + return i; + } + return -1; } + +static int find_slave_port_pkey_ix(struct mlx4_ib_dev *dev, int slave, + u8 port, u16 pkey, u16 *ix) +{ + int i, ret; + u8 unassigned_pkey_ix, pkey_ix, partial_ix = 0xFF; + u16 slot_pkey; + + if (slave == mlx4_master_func_num(dev->dev)) + return ib_find_cached_pkey(&dev->ib_dev, port, pkey, ix); + + unassigned_pkey_ix = dev->dev->phys_caps.pkey_phys_table_len[port] - 1; + + for (i = 0; i < dev->dev->caps.pkey_table_len[port]; i++) { + if (dev->pkeys.virt2phys_pkey[slave][port - 1][i] == unassigned_pkey_ix) + continue; + + pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][i]; + + ret = ib_get_cached_pkey(&dev->ib_dev, port, pkey_ix, &slot_pkey); + if (ret) + continue; + if ((slot_pkey & 0x7FFF) == (pkey & 0x7FFF)) { + if (slot_pkey & 0x8000) { + *ix = (u16) pkey_ix; + return 0; + } else { + /* take first partial pkey index found */ + if (partial_ix == 0xFF) + partial_ix = pkey_ix; + } + } + } + + if (partial_ix < 0xFF) { + *ix = (u16) partial_ix; + return 0; + } + + return -EINVAL; +} + +int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port, + enum ib_qp_type dest_qpt, struct ib_wc *wc, + struct ib_grh *grh, struct ib_mad *mad) +{ + struct ib_sge list; + struct ib_send_wr wr, *bad_wr; + struct mlx4_ib_demux_pv_ctx *tun_ctx; + struct mlx4_ib_demux_pv_qp *tun_qp; + struct mlx4_rcv_tunnel_mad *tun_mad; + struct ib_ah_attr attr; + struct ib_ah *ah; + struct ib_qp *src_qp = NULL; + unsigned tun_tx_ix = 0; + int dqpn; + int ret = 0; + u16 tun_pkey_ix; + u16 cached_pkey; + u8 is_eth = dev->dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH; + + if (dest_qpt > IB_QPT_GSI) + return -EINVAL; + + tun_ctx = dev->sriov.demux[port-1].tun[slave]; + + /* check if proxy qp created */ + if (!tun_ctx || tun_ctx->state != DEMUX_PV_STATE_ACTIVE) + return -EAGAIN; + + /* QP0 forwarding only for Dom0 */ + if (!dest_qpt && (mlx4_master_func_num(dev->dev) != slave)) + return -EINVAL; + + if (!dest_qpt) + tun_qp = &tun_ctx->qp[0]; + else + tun_qp = &tun_ctx->qp[1]; + + /* compute P_Key index to put in tunnel header for slave */ + if (dest_qpt) { + u16 pkey_ix; + ret = ib_get_cached_pkey(&dev->ib_dev, port, wc->pkey_index, &cached_pkey); + if (ret) + return -EINVAL; + + ret = find_slave_port_pkey_ix(dev, slave, port, cached_pkey, &pkey_ix); + if (ret) + return -EINVAL; + tun_pkey_ix = pkey_ix; + } else + tun_pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][0]; + + dqpn = dev->dev->phys_caps.base_proxy_sqpn + 8 * slave + port + (dest_qpt * 2) - 1; + + /* get tunnel tx data buf for slave */ + src_qp = tun_qp->qp; + + /* create ah. Just need an empty one with the port num for the post send. + * The driver will set the force loopback bit in post_send */ + memset(&attr, 0, sizeof attr); + attr.port_num = port; + if (is_eth) { + memcpy(&attr.grh.dgid.raw[0], &grh->dgid.raw[0], 16); + attr.ah_flags = IB_AH_GRH; + } + ah = ib_create_ah(tun_ctx->pd, &attr); + if (IS_ERR(ah)) + return -ENOMEM; + + /* allocate tunnel tx buf after pass failure returns */ + spin_lock(&tun_qp->tx_lock); + if (tun_qp->tx_ix_head - tun_qp->tx_ix_tail >= + (MLX4_NUM_TUNNEL_BUFS - 1)) + ret = -EAGAIN; + else + tun_tx_ix = (++tun_qp->tx_ix_head) & (MLX4_NUM_TUNNEL_BUFS - 1); + spin_unlock(&tun_qp->tx_lock); + if (ret) + goto out; + + tun_mad = (struct mlx4_rcv_tunnel_mad *) (tun_qp->tx_ring[tun_tx_ix].buf.addr); + if (tun_qp->tx_ring[tun_tx_ix].ah) + ib_destroy_ah(tun_qp->tx_ring[tun_tx_ix].ah); + tun_qp->tx_ring[tun_tx_ix].ah = ah; + ib_dma_sync_single_for_cpu(&dev->ib_dev, + tun_qp->tx_ring[tun_tx_ix].buf.map, + sizeof (struct mlx4_rcv_tunnel_mad), + DMA_TO_DEVICE); + + /* copy over to tunnel buffer */ + if (grh) + memcpy(&tun_mad->grh, grh, sizeof *grh); + memcpy(&tun_mad->mad, mad, sizeof *mad); + + /* adjust tunnel data */ + tun_mad->hdr.pkey_index = cpu_to_be16(tun_pkey_ix); + tun_mad->hdr.sl_vid = cpu_to_be16(((u16)(wc->sl)) << 12); + tun_mad->hdr.slid_mac_47_32 = cpu_to_be16(wc->slid); + tun_mad->hdr.flags_src_qp = cpu_to_be32(wc->src_qp & 0xFFFFFF); + tun_mad->hdr.g_ml_path = (grh && (wc->wc_flags & IB_WC_GRH)) ? 0x80 : 0; + + ib_dma_sync_single_for_device(&dev->ib_dev, + tun_qp->tx_ring[tun_tx_ix].buf.map, + sizeof (struct mlx4_rcv_tunnel_mad), + DMA_TO_DEVICE); + + list.addr = tun_qp->tx_ring[tun_tx_ix].buf.map; + list.length = sizeof (struct mlx4_rcv_tunnel_mad); + list.lkey = tun_ctx->mr->lkey; + + wr.wr.ud.ah = ah; + wr.wr.ud.port_num = port; + wr.wr.ud.remote_qkey = IB_QP_SET_QKEY; + wr.wr.ud.remote_qpn = dqpn; + wr.next = NULL; + wr.wr_id = ((u64) tun_tx_ix) | MLX4_TUN_SET_WRID_QPN(dest_qpt); + wr.sg_list = &list; + wr.num_sge = 1; + wr.opcode = IB_WR_SEND; + wr.send_flags = IB_SEND_SIGNALED; + + ret = ib_post_send(src_qp, &wr, &bad_wr); +out: + if (ret) + ib_destroy_ah(ah); + return ret; +} + +static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port, + struct ib_wc *wc, struct ib_grh *grh, + struct ib_mad *mad) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + int err; + int slave; + u8 *slave_id; + int is_eth = 0; + + if (rdma_port_get_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND) + is_eth = 0; + else + is_eth = 1; + + if (is_eth) { + if (!wc->wc_flags & IB_WC_GRH) { + mlx4_ib_warn(ibdev, "RoCE grh not present.\n"); + return -EINVAL; + } + if (mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_CM) { + mlx4_ib_warn(ibdev, "RoCE mgmt class is not CM\n"); + return -EINVAL; + } + if (mlx4_get_slave_from_roce_gid(dev->dev, port, grh->dgid.raw, &slave)) { + mlx4_ib_warn(ibdev, "failed matching grh\n"); + return -ENOENT; + } + if (slave >= dev->dev->caps.sqp_demux) { + mlx4_ib_warn(ibdev, "slave id: %d is bigger than allowed:%d\n", + slave, dev->dev->caps.sqp_demux); + return -ENOENT; + } + + if (mlx4_ib_demux_cm_handler(ibdev, port, &slave, mad, is_eth)) + return 0; + + err = mlx4_ib_send_to_slave(dev, slave, port, wc->qp->qp_type, wc, grh, mad); + if (err) + pr_debug("failed sending to slave %d via tunnel qp (%d)\n", + slave, err); + return 0; + } + + /* Initially assume that this mad is for us */ + slave = mlx4_master_func_num(dev->dev); + + /* See if the slave id is encoded in a response mad */ + if (mad->mad_hdr.method & 0x80) { + slave_id = (u8 *) &mad->mad_hdr.tid; + slave = *slave_id; + if (slave != 255) /*255 indicates the dom0*/ + *slave_id = 0; /* remap tid */ + } + + /* If a grh is present, we demux according to it */ + if (wc->wc_flags & IB_WC_GRH) { + slave = mlx4_ib_find_real_gid(ibdev, port, grh->dgid.global.interface_id); + if (slave < 0) { + mlx4_ib_warn(ibdev, "failed matching grh\n"); + return -ENOENT; + } + } + /* Class-specific handling */ + switch (mad->mad_hdr.mgmt_class) { + case IB_MGMT_CLASS_SUBN_ADM: + if (mlx4_ib_demux_sa_handler(ibdev, port, slave, + (struct ib_sa_mad *) mad)) + return 0; + break; + case IB_MGMT_CLASS_CM: + if (mlx4_ib_demux_cm_handler(ibdev, port, &slave, mad, is_eth)) + return 0; + break; + case IB_MGMT_CLASS_DEVICE_MGMT: + if (mad->mad_hdr.method != IB_MGMT_METHOD_GET_RESP) + return 0; + break; + default: + /* Drop unsupported classes for slaves in tunnel mode */ + if (slave != mlx4_master_func_num(dev->dev)) { + pr_debug("dropping unsupported ingress mad from class:%d " + "for slave:%d\n", mad->mad_hdr.mgmt_class, slave); + return 0; + } + } + /*make sure that no slave==255 was not handled yet.*/ + if (slave >= dev->dev->caps.sqp_demux) { + mlx4_ib_warn(ibdev, "slave id: %d is bigger than allowed:%d\n", + slave, dev->dev->caps.sqp_demux); + return -ENOENT; + } + + err = mlx4_ib_send_to_slave(dev, slave, port, wc->qp->qp_type, wc, grh, mad); + if (err) + pr_debug("failed sending to slave %d via tunnel qp (%d)\n", + slave, err); + return 0; +} + static int ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, - struct ib_wc *in_wc, struct ib_grh *in_grh, - struct ib_mad *in_mad, struct ib_mad *out_mad) + struct ib_wc *in_wc, struct ib_grh *in_grh, + struct ib_mad *in_mad, struct ib_mad *out_mad) { u16 slid, prev_lid = 0; int err; struct ib_port_attr pattr; + if (in_wc && in_wc->qp->qp_num) { + pr_debug("received MAD: slid:%d sqpn:%d " + "dlid_bits:%d dqpn:%d wc_flags:0x%x, cls %x, mtd %x, atr %x\n", + in_wc->slid, in_wc->src_qp, + in_wc->dlid_path_bits, + in_wc->qp->qp_num, + in_wc->wc_flags, + in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.method, + be16_to_cpu(in_mad->mad_hdr.attr_id)); + if (in_wc->wc_flags & IB_WC_GRH) { + pr_debug("sgid_hi:0x%016llx sgid_lo:0x%016llx\n", + (long long)be64_to_cpu(in_grh->sgid.global.subnet_prefix), + (long long) + be64_to_cpu(in_grh->sgid.global.interface_id)); + pr_debug("dgid_hi:0x%016llx dgid_lo:0x%016llx\n", + (long long)be64_to_cpu(in_grh->dgid.global.subnet_prefix), + (long long)be64_to_cpu(in_grh->dgid.global.interface_id)); + } + } + slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE); if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && slid == 0) { forward_trap(to_mdev(ibdev), port_num, in_mad); return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; } if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && in_mad->mad_hdr.method != IB_MGMT_METHOD_SET && in_mad->mad_hdr.method != IB_MGMT_METHOD_TRAP_REPRESS) return IB_MAD_RESULT_SUCCESS; /* - * Don't process SMInfo queries or vendor-specific - * MADs -- the SMA can't handle them. + * Don't process SMInfo queries -- the SMA can't handle them. */ - if (in_mad->mad_hdr.attr_id == IB_SMP_ATTR_SM_INFO || - (is_vendor_id(in_mad->mad_hdr.attr_id) && - !supported_vendor_id(in_mad->mad_hdr.attr_id))) + if (in_mad->mad_hdr.attr_id == IB_SMP_ATTR_SM_INFO) return IB_MAD_RESULT_SUCCESS; } else if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT || in_mad->mad_hdr.mgmt_class == MLX4_IB_VENDOR_CLASS1 || in_mad->mad_hdr.mgmt_class == MLX4_IB_VENDOR_CLASS2 || in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_CONG_MGMT) { if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && in_mad->mad_hdr.method != IB_MGMT_METHOD_SET) return IB_MAD_RESULT_SUCCESS; } else return IB_MAD_RESULT_SUCCESS; if ((in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && in_mad->mad_hdr.method == IB_MGMT_METHOD_SET && in_mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO && !ib_query_port(ibdev, port_num, &pattr)) prev_lid = pattr.lid; err = mlx4_MAD_IFC(to_mdev(ibdev), - mad_flags & IB_MAD_IGNORE_MKEY, - mad_flags & IB_MAD_IGNORE_BKEY, + (mad_flags & IB_MAD_IGNORE_MKEY ? MLX4_MAD_IFC_IGNORE_MKEY : 0) | + (mad_flags & IB_MAD_IGNORE_BKEY ? MLX4_MAD_IFC_IGNORE_BKEY : 0) | + MLX4_MAD_IFC_NET_VIEW, port_num, in_wc, in_grh, in_mad, out_mad); if (err) return IB_MAD_RESULT_FAILURE; if (!out_mad->mad_hdr.status) { - smp_snoop(ibdev, port_num, in_mad, prev_lid); - node_desc_override(ibdev, out_mad); + if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV)) + smp_snoop(ibdev, port_num, in_mad, prev_lid); + /* slaves get node desc from FW */ + if (!mlx4_is_slave(to_mdev(ibdev)->dev)) + node_desc_override(ibdev, out_mad); } /* set return bit in status of directed route responses */ if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) out_mad->mad_hdr.status |= cpu_to_be16(1 << 15); if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP_REPRESS) /* no response for trap repress */ return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; } -static __be32 be64_to_be32(__be64 b64) +static void edit_counter_ext(struct mlx4_if_stat_extended *cnt, void *counters, + __be16 attr_id) { - return cpu_to_be32(be64_to_cpu(b64) & 0xffffffff); -} + switch (attr_id) { + case IB_PMA_PORT_COUNTERS: + { + struct ib_pma_portcounters *pma_cnt = + (struct ib_pma_portcounters *)counters; + pma_cnt->port_xmit_data = + cpu_to_be32((be64_to_cpu(cnt->counters[0]. + IfTxUnicastOctets) + + be64_to_cpu(cnt->counters[0]. + IfTxMulticastOctets) + + be64_to_cpu(cnt->counters[0]. + IfTxBroadcastOctets) + + be64_to_cpu(cnt->counters[0]. + IfTxDroppedOctets)) >> 2); + pma_cnt->port_rcv_data = + cpu_to_be32((be64_to_cpu(cnt->counters[0]. + IfRxUnicastOctets) + + be64_to_cpu(cnt->counters[0]. + IfRxMulticastOctets) + + be64_to_cpu(cnt->counters[0]. + IfRxBroadcastOctets) + + be64_to_cpu(cnt->counters[0]. + IfRxNoBufferOctets) + + be64_to_cpu(cnt->counters[0]. + IfRxErrorOctets)) >> 2); + pma_cnt->port_xmit_packets = + cpu_to_be32(be64_to_cpu(cnt->counters[0]. + IfTxUnicastFrames) + + be64_to_cpu(cnt->counters[0]. + IfTxMulticastFrames) + + be64_to_cpu(cnt->counters[0]. + IfTxBroadcastFrames) + + be64_to_cpu(cnt->counters[0]. + IfTxDroppedFrames)); + pma_cnt->port_rcv_packets = + cpu_to_be32(be64_to_cpu(cnt->counters[0]. + IfRxUnicastFrames) + + be64_to_cpu(cnt->counters[0]. + IfRxMulticastFrames) + + be64_to_cpu(cnt->counters[0]. + IfRxBroadcastFrames) + + be64_to_cpu(cnt->counters[0]. + IfRxNoBufferFrames) + + be64_to_cpu(cnt->counters[0]. + IfRxErrorFrames)); + pma_cnt->port_rcv_errors = cpu_to_be32(be64_to_cpu(cnt-> + counters[0]. + IfRxErrorFrames)); + break; + } -static void edit_counters(struct mlx4_counters *cnt, void *data) -{ - *(__be32 *)(data + 40 + 24) = be64_to_be32(cnt->tx_bytes); - *(__be32 *)(data + 40 + 28) = be64_to_be32(cnt->rx_bytes); - *(__be32 *)(data + 40 + 32) = be64_to_be32(cnt->tx_frames); - *(__be32 *)(data + 40 + 36) = be64_to_be32(cnt->rx_frames); + case IB_PMA_PORT_COUNTERS_EXT: + { + struct ib_pma_portcounters_ext *pma_cnt_ext = + (struct ib_pma_portcounters_ext *)counters; + + pma_cnt_ext->port_xmit_data = + cpu_to_be64((be64_to_cpu(cnt->counters[0]. + IfTxUnicastOctets) + + be64_to_cpu(cnt->counters[0]. + IfTxMulticastOctets) + + be64_to_cpu(cnt->counters[0]. + IfTxBroadcastOctets) + + be64_to_cpu(cnt->counters[0]. + IfTxDroppedOctets)) >> 2); + pma_cnt_ext->port_rcv_data = + cpu_to_be64((be64_to_cpu(cnt->counters[0]. + IfRxUnicastOctets) + + be64_to_cpu(cnt->counters[0]. + IfRxMulticastOctets) + + be64_to_cpu(cnt->counters[0]. + IfRxBroadcastOctets) + + be64_to_cpu(cnt->counters[0]. + IfRxNoBufferOctets) + + be64_to_cpu(cnt->counters[0]. + IfRxErrorOctets)) >> 2); + pma_cnt_ext->port_xmit_packets = + cpu_to_be64(be64_to_cpu(cnt->counters[0]. + IfTxUnicastFrames) + + be64_to_cpu(cnt->counters[0]. + IfTxMulticastFrames) + + be64_to_cpu(cnt->counters[0]. + IfTxBroadcastFrames) + + be64_to_cpu(cnt->counters[0]. + IfTxDroppedFrames)); + pma_cnt_ext->port_rcv_packets = + cpu_to_be64(be64_to_cpu(cnt->counters[0]. + IfRxUnicastFrames) + + be64_to_cpu(cnt->counters[0]. + IfRxMulticastFrames) + + be64_to_cpu(cnt->counters[0]. + IfRxBroadcastFrames) + + be64_to_cpu(cnt->counters[0]. + IfRxNoBufferFrames) + + be64_to_cpu(cnt->counters[0]. + IfRxErrorFrames)); + pma_cnt_ext->port_unicast_xmit_packets = cnt->counters[0]. + IfTxUnicastFrames; + pma_cnt_ext->port_unicast_rcv_packets = cnt->counters[0]. + IfRxUnicastFrames; + pma_cnt_ext->port_multicast_xmit_packets = + cpu_to_be64(be64_to_cpu(cnt->counters[0]. + IfTxMulticastFrames) + + be64_to_cpu(cnt->counters[0]. + IfTxBroadcastFrames)); + pma_cnt_ext->port_multicast_rcv_packets = + cpu_to_be64(be64_to_cpu(cnt->counters[0]. + IfTxMulticastFrames) + + be64_to_cpu(cnt->counters[0]. + IfTxBroadcastFrames)); + + break; + } + + default: + pr_warn("Unsupported attr_id 0x%x\n", attr_id); + break; + } + } -static void edit_ext_counters(struct mlx4_counters_ext *cnt, void *data) +static void edit_counter(struct mlx4_if_stat_basic *cnt, void *counters, + __be16 attr_id) { - *(__be32 *)(data + 40 + 24) = be64_to_be32(cnt->tx_uni_bytes); - *(__be32 *)(data + 40 + 28) = be64_to_be32(cnt->rx_uni_bytes); - *(__be32 *)(data + 40 + 32) = be64_to_be32(cnt->tx_uni_frames); - *(__be32 *)(data + 40 + 36) = be64_to_be32(cnt->rx_uni_frames); - *(__be32 *)(data + 40 + 8) = be64_to_be32(cnt->rx_err_frames); + switch (attr_id) { + case IB_PMA_PORT_COUNTERS: + { + struct ib_pma_portcounters *pma_cnt = + (struct ib_pma_portcounters *) counters; + pma_cnt->port_xmit_data = + cpu_to_be32(be64_to_cpu( + cnt->counters[0].IfTxOctets) >> 2); + pma_cnt->port_rcv_data = + cpu_to_be32(be64_to_cpu( + cnt->counters[0].IfRxOctets) >> 2); + pma_cnt->port_xmit_packets = + cpu_to_be32(be64_to_cpu(cnt->counters[0].IfTxFrames)); + pma_cnt->port_rcv_packets = + cpu_to_be32(be64_to_cpu(cnt->counters[0].IfRxFrames)); + break; + } + case IB_PMA_PORT_COUNTERS_EXT: + { + struct ib_pma_portcounters_ext *pma_cnt_ext = + (struct ib_pma_portcounters_ext *) counters; + + pma_cnt_ext->port_xmit_data = + cpu_to_be64((be64_to_cpu(cnt->counters[0]. + IfTxOctets) >> 2)); + pma_cnt_ext->port_rcv_data = + cpu_to_be64((be64_to_cpu(cnt->counters[0]. + IfRxOctets) >> 2)); + pma_cnt_ext->port_xmit_packets = cnt->counters[0].IfTxFrames; + pma_cnt_ext->port_rcv_packets = cnt->counters[0].IfRxFrames; + break; + } + default: + pr_warn("Unsupported attr_id 0x%x\n", attr_id); + break; + } } -static int rdmaoe_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, - struct ib_wc *in_wc, struct ib_grh *in_grh, - struct ib_mad *in_mad, struct ib_mad *out_mad) +int mlx4_ib_query_if_stat(struct mlx4_ib_dev *dev, u32 counter_index, + union mlx4_counter *counter, u8 clear) { struct mlx4_cmd_mailbox *mailbox; - struct mlx4_ib_dev *dev = to_mdev(ibdev); int err; - u32 inmod = dev->counters[port_num - 1] & 0xffff; - int mode; + u32 inmod = counter_index | ((clear & 1) << 31); - if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_PERF_MGMT) - return -EINVAL; - mailbox = mlx4_alloc_cmd_mailbox(dev->dev); if (IS_ERR(mailbox)) return IB_MAD_RESULT_FAILURE; err = mlx4_cmd_box(dev->dev, 0, mailbox->dma, inmod, 0, - MLX4_CMD_QUERY_IF_STAT, MLX4_CMD_TIME_CLASS_C); - if (err) + MLX4_CMD_QUERY_IF_STAT, MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_WRAPPED); + if (!err) + memcpy(counter, mailbox->buf, MLX4_IF_STAT_SZ(1)); + + mlx4_free_cmd_mailbox(dev->dev, mailbox); + + return err; +} + +static int iboe_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, + struct ib_wc *in_wc, struct ib_grh *in_grh, + struct ib_mad *in_mad, struct ib_mad *out_mad) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + int err; + u32 counter_index = dev->counters[port_num - 1] & 0xffff; + u8 mode; + char counter_buf[MLX4_IF_STAT_SZ(1)]; + union mlx4_counter *counter = (union mlx4_counter *) + counter_buf; + + if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_PERF_MGMT) + return -EINVAL; + + if (mlx4_ib_query_if_stat(dev, counter_index, counter, 0)) { err = IB_MAD_RESULT_FAILURE; - else { + } else { memset(out_mad->data, 0, sizeof out_mad->data); - mode = be32_to_cpu(((struct mlx4_counters *)mailbox->buf)->counter_mode) & 0xf; - switch (mode) { + mode = counter->control.cnt_mode & 0xFF; + err = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; + switch (mode & 0xf) { case 0: - edit_counters(mailbox->buf, out_mad->data); - err = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; + edit_counter((void *)counter, + (void *)(out_mad->data + 40), + in_mad->mad_hdr.attr_id); break; case 1: - edit_ext_counters(mailbox->buf, out_mad->data); - err = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; + edit_counter_ext((void *)counter, + (void *)(out_mad->data + 40), + in_mad->mad_hdr.attr_id); break; default: err = IB_MAD_RESULT_FAILURE; } } - mlx4_free_cmd_mailbox(dev->dev, mailbox); return err; } -int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, +int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, struct ib_wc *in_wc, struct ib_grh *in_grh, struct ib_mad *in_mad, struct ib_mad *out_mad) { switch (rdma_port_get_link_layer(ibdev, port_num)) { case IB_LINK_LAYER_INFINIBAND: return ib_process_mad(ibdev, mad_flags, port_num, in_wc, in_grh, in_mad, out_mad); case IB_LINK_LAYER_ETHERNET: - return rdmaoe_process_mad(ibdev, mad_flags, port_num, in_wc, + return iboe_process_mad(ibdev, mad_flags, port_num, in_wc, in_grh, in_mad, out_mad); default: return -EINVAL; } } static void send_handler(struct ib_mad_agent *agent, struct ib_mad_send_wc *mad_send_wc) { + if (mad_send_wc->send_buf->context[0]) + ib_destroy_ah(mad_send_wc->send_buf->context[0]); ib_free_send_mad(mad_send_wc->send_buf); } int mlx4_ib_mad_init(struct mlx4_ib_dev *dev) { struct ib_mad_agent *agent; int p, q; int ret; enum rdma_link_layer ll; for (p = 0; p < dev->num_ports; ++p) { ll = rdma_port_get_link_layer(&dev->ib_dev, p + 1); for (q = 0; q <= 1; ++q) { if (ll == IB_LINK_LAYER_INFINIBAND) { agent = ib_register_mad_agent(&dev->ib_dev, p + 1, q ? IB_QPT_GSI : IB_QPT_SMI, NULL, 0, send_handler, NULL, NULL); if (IS_ERR(agent)) { ret = PTR_ERR(agent); goto err; } dev->send_agent[p][q] = agent; } else dev->send_agent[p][q] = NULL; } } return 0; err: for (p = 0; p < dev->num_ports; ++p) for (q = 0; q <= 1; ++q) if (dev->send_agent[p][q]) ib_unregister_mad_agent(dev->send_agent[p][q]); return ret; } void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev) { struct ib_mad_agent *agent; int p, q; for (p = 0; p < dev->num_ports; ++p) { for (q = 0; q <= 1; ++q) { agent = dev->send_agent[p][q]; if (agent) { dev->send_agent[p][q] = NULL; ib_unregister_mad_agent(agent); } } if (dev->sm_ah[p]) ib_destroy_ah(dev->sm_ah[p]); + } +} + +static void handle_lid_change_event(struct mlx4_ib_dev *dev, u8 port_num) +{ + mlx4_ib_dispatch_event(dev, port_num, IB_EVENT_LID_CHANGE); + + if (mlx4_is_master(dev->dev) && !dev->sriov.is_going_down) + mlx4_gen_slaves_port_mgt_ev(dev->dev, port_num, + MLX4_EQ_PORT_INFO_LID_CHANGE_MASK, 0, 0); +} + +static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u8 port_num) +{ + /* re-configure the alias-guid and mcg's */ + if (mlx4_is_master(dev->dev)) { + mlx4_ib_invalidate_all_guid_record(dev, port_num); + + if (!dev->sriov.is_going_down) { + mlx4_ib_mcg_port_cleanup(&dev->sriov.demux[port_num - 1], 0); + mlx4_gen_slaves_port_mgt_ev(dev->dev, port_num, + MLX4_EQ_PORT_INFO_CLIENT_REREG_MASK, 0, 0); + } + } + mlx4_ib_dispatch_event(dev, port_num, IB_EVENT_CLIENT_REREGISTER); +} + +static void propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num, + struct mlx4_eqe *eqe) +{ + __propagate_pkey_ev(dev, port_num, GET_BLK_PTR_FROM_EQE(eqe), + GET_MASK_FROM_EQE(eqe)); +} + +static void handle_slaves_guid_change(struct mlx4_ib_dev *dev, u8 port_num, + u32 guid_tbl_blk_num, u32 change_bitmap) +{ + struct ib_smp *in_mad = NULL; + struct ib_smp *out_mad = NULL; + u16 i; + + if (!mlx4_is_mfunc(dev->dev) || !mlx4_is_master(dev->dev)) + return; + + in_mad = kmalloc(sizeof *in_mad, GFP_KERNEL); + out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); + if (!in_mad || !out_mad) { + mlx4_ib_warn(&dev->ib_dev, "failed to allocate memory for guid info mads\n"); + goto out; + } + + guid_tbl_blk_num *= 4; + + for (i = 0; i < 4; i++) { + if (change_bitmap && (!((change_bitmap >> (8 * i)) & 0xff))) + continue; + memset(in_mad, 0, sizeof *in_mad); + memset(out_mad, 0, sizeof *out_mad); + + in_mad->base_version = 1; + in_mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; + in_mad->class_version = 1; + in_mad->method = IB_MGMT_METHOD_GET; + in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; + in_mad->attr_mod = cpu_to_be32(guid_tbl_blk_num + i); + + if (mlx4_MAD_IFC(dev, + MLX4_MAD_IFC_IGNORE_KEYS | MLX4_MAD_IFC_NET_VIEW, + port_num, NULL, NULL, in_mad, out_mad)) { + mlx4_ib_warn(&dev->ib_dev, "Failed in get GUID INFO MAD_IFC\n"); + goto out; + } + + mlx4_ib_update_cache_on_guid_change(dev, guid_tbl_blk_num + i, + port_num, + (u8 *)(&((struct ib_smp *)out_mad)->data)); + mlx4_ib_notify_slaves_on_guid_change(dev, guid_tbl_blk_num + i, + port_num, + (u8 *)(&((struct ib_smp *)out_mad)->data)); + } + +out: + kfree(in_mad); + kfree(out_mad); + return; +} + +void handle_port_mgmt_change_event(struct work_struct *work) +{ + struct ib_event_work *ew = container_of(work, struct ib_event_work, work); + struct mlx4_ib_dev *dev = ew->ib_dev; + struct mlx4_eqe *eqe = &(ew->ib_eqe); + u8 port = eqe->event.port_mgmt_change.port; + u32 changed_attr; + u32 tbl_block; + u32 change_bitmap; + + switch (eqe->subtype) { + case MLX4_DEV_PMC_SUBTYPE_PORT_INFO: + changed_attr = be32_to_cpu(eqe->event.port_mgmt_change.params.port_info.changed_attr); + + /* Update the SM ah - This should be done before handling + the other changed attributes so that MADs can be sent to the SM */ + if (changed_attr & MSTR_SM_CHANGE_MASK) { + u16 lid = be16_to_cpu(eqe->event.port_mgmt_change.params.port_info.mstr_sm_lid); + u8 sl = eqe->event.port_mgmt_change.params.port_info.mstr_sm_sl & 0xf; + update_sm_ah(dev, port, lid, sl); + } + + /* Check if it is a lid change event */ + if (changed_attr & MLX4_EQ_PORT_INFO_LID_CHANGE_MASK) + handle_lid_change_event(dev, port); + + /* Generate GUID changed event */ + if (changed_attr & MLX4_EQ_PORT_INFO_GID_PFX_CHANGE_MASK) { + mlx4_ib_dispatch_event(dev, port, IB_EVENT_GID_CHANGE); + /*if master, notify all slaves*/ + if (mlx4_is_master(dev->dev)) + mlx4_gen_slaves_port_mgt_ev(dev->dev, port, + MLX4_EQ_PORT_INFO_GID_PFX_CHANGE_MASK, 0, 0); + } + + if (changed_attr & MLX4_EQ_PORT_INFO_CLIENT_REREG_MASK) + handle_client_rereg_event(dev, port); + break; + + case MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE: + mlx4_ib_dispatch_event(dev, port, IB_EVENT_PKEY_CHANGE); + if (mlx4_is_master(dev->dev) && !dev->sriov.is_going_down) + propagate_pkey_ev(dev, port, eqe); + break; + case MLX4_DEV_PMC_SUBTYPE_GUID_INFO: + /* paravirtualized master's guid is guid 0 -- does not change */ + if (!mlx4_is_master(dev->dev)) + mlx4_ib_dispatch_event(dev, port, IB_EVENT_GID_CHANGE); + /*if master, notify relevant slaves*/ + else if (!dev->sriov.is_going_down) { + tbl_block = GET_BLK_PTR_FROM_EQE(eqe); + change_bitmap = GET_MASK_FROM_EQE(eqe); + handle_slaves_guid_change(dev, port, tbl_block, change_bitmap); + } + break; + default: + pr_warn("Unsupported subtype 0x%x for " + "Port Management Change event\n", eqe->subtype); + } + + kfree(ew); +} + +void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u8 port_num, + enum ib_event_type type) +{ + struct ib_event event; + + event.device = &dev->ib_dev; + event.element.port_num = port_num; + event.event = type; + + ib_dispatch_event(&event); +} + +static void mlx4_ib_tunnel_comp_handler(struct ib_cq *cq, void *arg) +{ + unsigned long flags; + struct mlx4_ib_demux_pv_ctx *ctx = cq->cq_context; + struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev); + spin_lock_irqsave(&dev->sriov.going_down_lock, flags); + if (!dev->sriov.is_going_down && ctx->state == DEMUX_PV_STATE_ACTIVE) + queue_work(ctx->wq, &ctx->work); + spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); +} + +static int mlx4_ib_post_pv_qp_buf(struct mlx4_ib_demux_pv_ctx *ctx, + struct mlx4_ib_demux_pv_qp *tun_qp, + int index) +{ + struct ib_sge sg_list; + struct ib_recv_wr recv_wr, *bad_recv_wr; + int size; + + size = (tun_qp->qp->qp_type == IB_QPT_UD) ? + sizeof (struct mlx4_tunnel_mad) : sizeof (struct mlx4_mad_rcv_buf); + + sg_list.addr = tun_qp->ring[index].map; + sg_list.length = size; + sg_list.lkey = ctx->mr->lkey; + + recv_wr.next = NULL; + recv_wr.sg_list = &sg_list; + recv_wr.num_sge = 1; + recv_wr.wr_id = (u64) index | MLX4_TUN_WRID_RECV | + MLX4_TUN_SET_WRID_QPN(tun_qp->proxy_qpt); + ib_dma_sync_single_for_device(ctx->ib_dev, tun_qp->ring[index].map, + size, DMA_FROM_DEVICE); + return ib_post_recv(tun_qp->qp, &recv_wr, &bad_recv_wr); +} + +static int mlx4_ib_multiplex_sa_handler(struct ib_device *ibdev, int port, + int slave, struct ib_sa_mad *sa_mad) +{ + int ret = 0; + + /* dispatch to different sa handlers */ + switch (be16_to_cpu(sa_mad->mad_hdr.attr_id)) { + case IB_SA_ATTR_MC_MEMBER_REC: + ret = mlx4_ib_mcg_multiplex_handler(ibdev, port, slave, sa_mad); + break; + default: + break; + } + return ret; +} + +static int is_proxy_qp0(struct mlx4_ib_dev *dev, int qpn, int slave) +{ + int proxy_start = dev->dev->phys_caps.base_proxy_sqpn + 8 * slave; + + return (qpn >= proxy_start && qpn <= proxy_start + 1); +} + + +int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port, + enum ib_qp_type dest_qpt, u16 pkey_index, u32 remote_qpn, + u32 qkey, struct ib_ah_attr *attr, struct ib_mad *mad) +{ + struct ib_sge list; + struct ib_send_wr wr, *bad_wr; + struct mlx4_ib_demux_pv_ctx *sqp_ctx; + struct mlx4_ib_demux_pv_qp *sqp; + struct mlx4_mad_snd_buf *sqp_mad; + struct ib_ah *ah; + struct ib_qp *send_qp = NULL; + unsigned wire_tx_ix = 0; + int ret = 0; + u16 wire_pkey_ix; + int src_qpnum; + u8 sgid_index; + + + sqp_ctx = dev->sriov.sqps[port-1]; + + /* check if proxy qp created */ + if (!sqp_ctx || sqp_ctx->state != DEMUX_PV_STATE_ACTIVE) + return -EAGAIN; + + /* QP0 forwarding only for Dom0 */ + if (dest_qpt == IB_QPT_SMI && (mlx4_master_func_num(dev->dev) != slave)) + return -EINVAL; + + if (dest_qpt == IB_QPT_SMI) { + src_qpnum = 0; + sqp = &sqp_ctx->qp[0]; + wire_pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][0]; + } else { + src_qpnum = 1; + sqp = &sqp_ctx->qp[1]; + wire_pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][pkey_index]; + } + + send_qp = sqp->qp; + + /* create ah */ + sgid_index = attr->grh.sgid_index; + attr->grh.sgid_index = 0; + ah = ib_create_ah(sqp_ctx->pd, attr); + if (IS_ERR(ah)) + return -ENOMEM; + attr->grh.sgid_index = sgid_index; + to_mah(ah)->av.ib.gid_index = sgid_index; + /* get rid of force-loopback bit */ + to_mah(ah)->av.ib.port_pd &= cpu_to_be32(0x7FFFFFFF); + spin_lock(&sqp->tx_lock); + if (sqp->tx_ix_head - sqp->tx_ix_tail >= + (MLX4_NUM_TUNNEL_BUFS - 1)) + ret = -EAGAIN; + else + wire_tx_ix = (++sqp->tx_ix_head) & (MLX4_NUM_TUNNEL_BUFS - 1); + spin_unlock(&sqp->tx_lock); + if (ret) + goto out; + + sqp_mad = (struct mlx4_mad_snd_buf *) (sqp->tx_ring[wire_tx_ix].buf.addr); + if (sqp->tx_ring[wire_tx_ix].ah) + ib_destroy_ah(sqp->tx_ring[wire_tx_ix].ah); + sqp->tx_ring[wire_tx_ix].ah = ah; + ib_dma_sync_single_for_cpu(&dev->ib_dev, + sqp->tx_ring[wire_tx_ix].buf.map, + sizeof (struct mlx4_mad_snd_buf), + DMA_TO_DEVICE); + + memcpy(&sqp_mad->payload, mad, sizeof *mad); + + ib_dma_sync_single_for_device(&dev->ib_dev, + sqp->tx_ring[wire_tx_ix].buf.map, + sizeof (struct mlx4_mad_snd_buf), + DMA_TO_DEVICE); + + list.addr = sqp->tx_ring[wire_tx_ix].buf.map; + list.length = sizeof (struct mlx4_mad_snd_buf); + list.lkey = sqp_ctx->mr->lkey; + + wr.wr.ud.ah = ah; + wr.wr.ud.port_num = port; + wr.wr.ud.pkey_index = wire_pkey_ix; + wr.wr.ud.remote_qkey = qkey; + wr.wr.ud.remote_qpn = remote_qpn; + wr.next = NULL; + wr.wr_id = ((u64) wire_tx_ix) | MLX4_TUN_SET_WRID_QPN(src_qpnum); + wr.sg_list = &list; + wr.num_sge = 1; + wr.opcode = IB_WR_SEND; + wr.send_flags = IB_SEND_SIGNALED; + + ret = ib_post_send(send_qp, &wr, &bad_wr); +out: + if (ret) + ib_destroy_ah(ah); + return ret; +} + +static int get_slave_base_gid_ix(struct mlx4_ib_dev *dev, int slave, int port) +{ + int gids; + int vfs; + + if (rdma_port_get_link_layer(&dev->ib_dev, port) == IB_LINK_LAYER_INFINIBAND) + return slave; + + gids = MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS; + vfs = dev->dev->num_vfs; + + if (slave == 0) + return 0; + if (slave <= gids % vfs) + return MLX4_ROCE_PF_GIDS + ((gids / vfs) + 1) * (slave - 1); + + return MLX4_ROCE_PF_GIDS + (gids % vfs) + ((gids / vfs) * (slave - 1)); +} + +static int get_real_sgid_index(struct mlx4_ib_dev *dev, int slave, int port, + struct ib_ah_attr *ah_attr) +{ + if (rdma_port_get_link_layer(&dev->ib_dev, port) == IB_LINK_LAYER_INFINIBAND) { + ah_attr->grh.sgid_index = slave; + return 0; + } + ah_attr->grh.sgid_index += get_slave_base_gid_ix(dev, slave, port); + return 0; +} + +static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc *wc) +{ + struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev); + struct mlx4_ib_demux_pv_qp *tun_qp = &ctx->qp[MLX4_TUN_WRID_QPN(wc->wr_id)]; + int wr_ix = wc->wr_id & (MLX4_NUM_TUNNEL_BUFS - 1); + struct mlx4_tunnel_mad *tunnel = tun_qp->ring[wr_ix].addr; + struct mlx4_ib_ah ah; + struct ib_ah_attr ah_attr; + u8 *slave_id; + int slave; + + /* Get slave that sent this packet */ + if (wc->src_qp < dev->dev->phys_caps.base_proxy_sqpn || + wc->src_qp >= dev->dev->phys_caps.base_proxy_sqpn + 8 * MLX4_MFUNC_MAX || + (wc->src_qp & 0x1) != ctx->port - 1 || + wc->src_qp & 0x4) { + mlx4_ib_warn(ctx->ib_dev, "can't multiplex bad sqp:%d\n", wc->src_qp); + return; + } + slave = ((wc->src_qp & ~0x7) - dev->dev->phys_caps.base_proxy_sqpn) / 8; + if (slave != ctx->slave) { + mlx4_ib_warn(ctx->ib_dev, "can't multiplex bad sqp:%d: " + "belongs to another slave\n", wc->src_qp); + return; + } + if (slave != mlx4_master_func_num(dev->dev) && !(wc->src_qp & 0x2)) { + mlx4_ib_warn(ctx->ib_dev, "can't multiplex bad sqp:%d: " + "non-master trying to send QP0 packets\n", wc->src_qp); + return; + } + + /* Map transaction ID */ + ib_dma_sync_single_for_cpu(ctx->ib_dev, tun_qp->ring[wr_ix].map, + sizeof (struct mlx4_tunnel_mad), + DMA_FROM_DEVICE); + switch (tunnel->mad.mad_hdr.method) { + case IB_MGMT_METHOD_SET: + case IB_MGMT_METHOD_GET: + case IB_MGMT_METHOD_REPORT: + case IB_SA_METHOD_GET_TABLE: + case IB_SA_METHOD_DELETE: + case IB_SA_METHOD_GET_MULTI: + case IB_SA_METHOD_GET_TRACE_TBL: + slave_id = (u8 *) &tunnel->mad.mad_hdr.tid; + if (*slave_id) { + mlx4_ib_warn(ctx->ib_dev, "egress mad has non-null tid msb:%d " + "class:%d slave:%d\n", *slave_id, + tunnel->mad.mad_hdr.mgmt_class, slave); + return; + } else + *slave_id = slave; + default: + /* nothing */; + } + + /* Class-specific handling */ + switch (tunnel->mad.mad_hdr.mgmt_class) { + case IB_MGMT_CLASS_SUBN_ADM: + if (mlx4_ib_multiplex_sa_handler(ctx->ib_dev, ctx->port, slave, + (struct ib_sa_mad *) &tunnel->mad)) + return; + break; + case IB_MGMT_CLASS_CM: + if (mlx4_ib_multiplex_cm_handler(ctx->ib_dev, ctx->port, slave, + (struct ib_mad *) &tunnel->mad)) + return; + break; + case IB_MGMT_CLASS_DEVICE_MGMT: + if (tunnel->mad.mad_hdr.method != IB_MGMT_METHOD_GET && + tunnel->mad.mad_hdr.method != IB_MGMT_METHOD_SET) + return; + break; + default: + /* Drop unsupported classes for slaves in tunnel mode */ + if (slave != mlx4_master_func_num(dev->dev)) { + mlx4_ib_warn(ctx->ib_dev, "dropping unsupported egress mad from class:%d " + "for slave:%d\n", tunnel->mad.mad_hdr.mgmt_class, slave); + return; + } + } + + /* We are using standard ib_core services to send the mad, so generate a + * stadard address handle by decoding the tunnelled mlx4_ah fields */ + memcpy(&ah.av, &tunnel->hdr.av, sizeof (struct mlx4_av)); + ah.ibah.device = ctx->ib_dev; + mlx4_ib_query_ah(&ah.ibah, &ah_attr); + if (ah_attr.ah_flags & IB_AH_GRH) + if (get_real_sgid_index(dev, slave, ctx->port, &ah_attr)) + return; + + mlx4_ib_send_to_wire(dev, slave, ctx->port, + is_proxy_qp0(dev, wc->src_qp, slave) ? + IB_QPT_SMI : IB_QPT_GSI, + be16_to_cpu(tunnel->hdr.pkey_index), + be32_to_cpu(tunnel->hdr.remote_qpn), + be32_to_cpu(tunnel->hdr.qkey), + &ah_attr, &tunnel->mad); +} + +static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx, + enum ib_qp_type qp_type, int is_tun) +{ + int i; + struct mlx4_ib_demux_pv_qp *tun_qp; + int rx_buf_size, tx_buf_size; + + if (qp_type > IB_QPT_GSI) + return -EINVAL; + + tun_qp = &ctx->qp[qp_type]; + + tun_qp->ring = kzalloc(sizeof (struct mlx4_ib_buf) * MLX4_NUM_TUNNEL_BUFS, + GFP_KERNEL); + if (!tun_qp->ring) + return -ENOMEM; + + tun_qp->tx_ring = kcalloc(MLX4_NUM_TUNNEL_BUFS, + sizeof (struct mlx4_ib_tun_tx_buf), + GFP_KERNEL); + if (!tun_qp->tx_ring) { + kfree(tun_qp->ring); + tun_qp->ring = NULL; + return -ENOMEM; + } + + if (is_tun) { + rx_buf_size = sizeof (struct mlx4_tunnel_mad); + tx_buf_size = sizeof (struct mlx4_rcv_tunnel_mad); + } else { + rx_buf_size = sizeof (struct mlx4_mad_rcv_buf); + tx_buf_size = sizeof (struct mlx4_mad_snd_buf); + } + + for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + tun_qp->ring[i].addr = kmalloc(rx_buf_size, GFP_KERNEL); + if (!tun_qp->ring[i].addr) + goto err; + tun_qp->ring[i].map = ib_dma_map_single(ctx->ib_dev, + tun_qp->ring[i].addr, + rx_buf_size, + DMA_FROM_DEVICE); + } + + for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + tun_qp->tx_ring[i].buf.addr = + kmalloc(tx_buf_size, GFP_KERNEL); + if (!tun_qp->tx_ring[i].buf.addr) + goto tx_err; + tun_qp->tx_ring[i].buf.map = + ib_dma_map_single(ctx->ib_dev, + tun_qp->tx_ring[i].buf.addr, + tx_buf_size, + DMA_TO_DEVICE); + tun_qp->tx_ring[i].ah = NULL; + } + spin_lock_init(&tun_qp->tx_lock); + tun_qp->tx_ix_head = 0; + tun_qp->tx_ix_tail = 0; + tun_qp->proxy_qpt = qp_type; + + return 0; + +tx_err: + while (i > 0) { + --i; + ib_dma_unmap_single(ctx->ib_dev, tun_qp->tx_ring[i].buf.map, + tx_buf_size, DMA_TO_DEVICE); + kfree(tun_qp->tx_ring[i].buf.addr); + } + kfree(tun_qp->tx_ring); + tun_qp->tx_ring = NULL; + i = MLX4_NUM_TUNNEL_BUFS; +err: + while (i > 0) { + --i; + ib_dma_unmap_single(ctx->ib_dev, tun_qp->ring[i].map, + rx_buf_size, DMA_FROM_DEVICE); + kfree(tun_qp->ring[i].addr); + } + kfree(tun_qp->ring); + tun_qp->ring = NULL; + return -ENOMEM; +} + +static void mlx4_ib_free_pv_qp_bufs(struct mlx4_ib_demux_pv_ctx *ctx, + enum ib_qp_type qp_type, int is_tun) +{ + int i; + struct mlx4_ib_demux_pv_qp *tun_qp; + int rx_buf_size, tx_buf_size; + + if (qp_type > IB_QPT_GSI) + return; + + tun_qp = &ctx->qp[qp_type]; + if (is_tun) { + rx_buf_size = sizeof (struct mlx4_tunnel_mad); + tx_buf_size = sizeof (struct mlx4_rcv_tunnel_mad); + } else { + rx_buf_size = sizeof (struct mlx4_mad_rcv_buf); + tx_buf_size = sizeof (struct mlx4_mad_snd_buf); + } + + + for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + ib_dma_unmap_single(ctx->ib_dev, tun_qp->ring[i].map, + rx_buf_size, DMA_FROM_DEVICE); + kfree(tun_qp->ring[i].addr); + } + + for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + ib_dma_unmap_single(ctx->ib_dev, tun_qp->tx_ring[i].buf.map, + tx_buf_size, DMA_TO_DEVICE); + kfree(tun_qp->tx_ring[i].buf.addr); + if (tun_qp->tx_ring[i].ah) + ib_destroy_ah(tun_qp->tx_ring[i].ah); + } + kfree(tun_qp->tx_ring); + kfree(tun_qp->ring); +} + +static void mlx4_ib_tunnel_comp_worker(struct work_struct *work) +{ + struct mlx4_ib_demux_pv_ctx *ctx; + struct mlx4_ib_demux_pv_qp *tun_qp; + struct ib_wc wc; + int ret; + ctx = container_of(work, struct mlx4_ib_demux_pv_ctx, work); + ib_req_notify_cq(ctx->cq, IB_CQ_NEXT_COMP); + + while (ib_poll_cq(ctx->cq, 1, &wc) == 1) { + tun_qp = &ctx->qp[MLX4_TUN_WRID_QPN(wc.wr_id)]; + if (wc.status == IB_WC_SUCCESS) { + switch (wc.opcode) { + case IB_WC_RECV: + mlx4_ib_multiplex_mad(ctx, &wc); + ret = mlx4_ib_post_pv_qp_buf(ctx, tun_qp, + wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)); + if (ret) + pr_err("Failed reposting tunnel " + "buf:%lld\n", (long long)wc.wr_id); + break; + case IB_WC_SEND: + pr_debug("received tunnel send completion:" + "wrid=0x%llx, status=0x%x\n", + (long long)wc.wr_id, wc.status); + ib_destroy_ah(tun_qp->tx_ring[wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)].ah); + tun_qp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah + = NULL; + spin_lock(&tun_qp->tx_lock); + tun_qp->tx_ix_tail++; + spin_unlock(&tun_qp->tx_lock); + + break; + default: + break; + } + } else { + pr_debug("mlx4_ib: completion error in tunnel: %d." + " status = %d, wrid = 0x%llx\n", + ctx->slave, wc.status, (long long)wc.wr_id); + if (!MLX4_TUN_IS_RECV(wc.wr_id)) { + ib_destroy_ah(tun_qp->tx_ring[wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)].ah); + tun_qp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah + = NULL; + spin_lock(&tun_qp->tx_lock); + tun_qp->tx_ix_tail++; + spin_unlock(&tun_qp->tx_lock); + } + } + } +} + +static void pv_qp_event_handler(struct ib_event *event, void *qp_context) +{ + struct mlx4_ib_demux_pv_ctx *sqp = qp_context; + + /* It's worse than that! He's dead, Jim! */ + pr_err("Fatal error (%d) on a MAD QP on port %d\n", + event->event, sqp->port); +} + +static int create_pv_sqp(struct mlx4_ib_demux_pv_ctx *ctx, + enum ib_qp_type qp_type, int create_tun) +{ + int i, ret; + struct mlx4_ib_demux_pv_qp *tun_qp; + struct mlx4_ib_qp_tunnel_init_attr qp_init_attr; + struct ib_qp_attr attr; + int qp_attr_mask_INIT; + + if (qp_type > IB_QPT_GSI) + return -EINVAL; + + tun_qp = &ctx->qp[qp_type]; + + memset(&qp_init_attr, 0, sizeof qp_init_attr); + qp_init_attr.init_attr.send_cq = ctx->cq; + qp_init_attr.init_attr.recv_cq = ctx->cq; + qp_init_attr.init_attr.sq_sig_type = IB_SIGNAL_ALL_WR; + qp_init_attr.init_attr.cap.max_send_wr = MLX4_NUM_TUNNEL_BUFS; + qp_init_attr.init_attr.cap.max_recv_wr = MLX4_NUM_TUNNEL_BUFS; + qp_init_attr.init_attr.cap.max_send_sge = 1; + qp_init_attr.init_attr.cap.max_recv_sge = 1; + if (create_tun) { + qp_init_attr.init_attr.qp_type = IB_QPT_UD; + qp_init_attr.init_attr.create_flags = (enum ib_qp_create_flags)MLX4_IB_SRIOV_TUNNEL_QP; + qp_init_attr.port = ctx->port; + qp_init_attr.slave = ctx->slave; + qp_init_attr.proxy_qp_type = qp_type; + qp_attr_mask_INIT = IB_QP_STATE | IB_QP_PKEY_INDEX | + IB_QP_QKEY | IB_QP_PORT; + } else { + qp_init_attr.init_attr.qp_type = qp_type; + qp_init_attr.init_attr.create_flags = (enum ib_qp_create_flags)MLX4_IB_SRIOV_SQP; + qp_attr_mask_INIT = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_QKEY; + } + qp_init_attr.init_attr.port_num = ctx->port; + qp_init_attr.init_attr.qp_context = ctx; + qp_init_attr.init_attr.event_handler = pv_qp_event_handler; + tun_qp->qp = ib_create_qp(ctx->pd, &qp_init_attr.init_attr); + if (IS_ERR(tun_qp->qp)) { + ret = PTR_ERR(tun_qp->qp); + tun_qp->qp = NULL; + pr_err("Couldn't create %s QP (%d)\n", + create_tun ? "tunnel" : "special", ret); + return ret; + } + + memset(&attr, 0, sizeof attr); + attr.qp_state = IB_QPS_INIT; + attr.pkey_index = + to_mdev(ctx->ib_dev)->pkeys.virt2phys_pkey[ctx->slave][ctx->port - 1][0]; + attr.qkey = IB_QP1_QKEY; + attr.port_num = ctx->port; + ret = ib_modify_qp(tun_qp->qp, &attr, qp_attr_mask_INIT); + if (ret) { + pr_err("Couldn't change %s qp state to INIT (%d)\n", + create_tun ? "tunnel" : "special", ret); + goto err_qp; + } + attr.qp_state = IB_QPS_RTR; + ret = ib_modify_qp(tun_qp->qp, &attr, IB_QP_STATE); + if (ret) { + pr_err("Couldn't change %s qp state to RTR (%d)\n", + create_tun ? "tunnel" : "special", ret); + goto err_qp; + } + attr.qp_state = IB_QPS_RTS; + attr.sq_psn = 0; + ret = ib_modify_qp(tun_qp->qp, &attr, IB_QP_STATE | IB_QP_SQ_PSN); + if (ret) { + pr_err("Couldn't change %s qp state to RTS (%d)\n", + create_tun ? "tunnel" : "special", ret); + goto err_qp; + } + + for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) { + ret = mlx4_ib_post_pv_qp_buf(ctx, tun_qp, i); + if (ret) { + pr_err(" mlx4_ib_post_pv_buf error" + " (err = %d, i = %d)\n", ret, i); + goto err_qp; + } + } + return 0; + +err_qp: + ib_destroy_qp(tun_qp->qp); + tun_qp->qp = NULL; + return ret; +} + +/* + * IB MAD completion callback for real SQPs + */ +static void mlx4_ib_sqp_comp_worker(struct work_struct *work) +{ + struct mlx4_ib_demux_pv_ctx *ctx; + struct mlx4_ib_demux_pv_qp *sqp; + struct ib_wc wc; + struct ib_grh *grh; + struct ib_mad *mad; + + ctx = container_of(work, struct mlx4_ib_demux_pv_ctx, work); + ib_req_notify_cq(ctx->cq, IB_CQ_NEXT_COMP); + + while (mlx4_ib_poll_cq(ctx->cq, 1, &wc) == 1) { + sqp = &ctx->qp[MLX4_TUN_WRID_QPN(wc.wr_id)]; + if (wc.status == IB_WC_SUCCESS) { + switch (wc.opcode) { + case IB_WC_SEND: + ib_destroy_ah(sqp->tx_ring[wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)].ah); + sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah + = NULL; + spin_lock(&sqp->tx_lock); + sqp->tx_ix_tail++; + spin_unlock(&sqp->tx_lock); + break; + case IB_WC_RECV: + mad = (struct ib_mad *) &(((struct mlx4_mad_rcv_buf *) + (sqp->ring[wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)].addr))->payload); + grh = &(((struct mlx4_mad_rcv_buf *) + (sqp->ring[wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)].addr))->grh); + mlx4_ib_demux_mad(ctx->ib_dev, ctx->port, &wc, grh, mad); + if (mlx4_ib_post_pv_qp_buf(ctx, sqp, wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1))) + pr_err("Failed reposting SQP " + "buf:%lld\n", (long long)wc.wr_id); + break; + default: + BUG_ON(1); + break; + } + } else { + pr_debug("mlx4_ib: completion error in tunnel: %d." + " status = %d, wrid = 0x%llx\n", + ctx->slave, wc.status, (long long)wc.wr_id); + if (!MLX4_TUN_IS_RECV(wc.wr_id)) { + ib_destroy_ah(sqp->tx_ring[wc.wr_id & + (MLX4_NUM_TUNNEL_BUFS - 1)].ah); + sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah + = NULL; + spin_lock(&sqp->tx_lock); + sqp->tx_ix_tail++; + spin_unlock(&sqp->tx_lock); + } + } + } +} + +static int alloc_pv_object(struct mlx4_ib_dev *dev, int slave, int port, + struct mlx4_ib_demux_pv_ctx **ret_ctx) +{ + struct mlx4_ib_demux_pv_ctx *ctx; + + *ret_ctx = NULL; + ctx = kzalloc(sizeof (struct mlx4_ib_demux_pv_ctx), GFP_KERNEL); + if (!ctx) { + pr_err("failed allocating pv resource context " + "for port %d, slave %d\n", port, slave); + return -ENOMEM; + } + + ctx->ib_dev = &dev->ib_dev; + ctx->port = port; + ctx->slave = slave; + *ret_ctx = ctx; + return 0; +} + +static void free_pv_object(struct mlx4_ib_dev *dev, int slave, int port) +{ + if (dev->sriov.demux[port - 1].tun[slave]) { + kfree(dev->sriov.demux[port - 1].tun[slave]); + dev->sriov.demux[port - 1].tun[slave] = NULL; + } +} + +static int create_pv_resources(struct ib_device *ibdev, int slave, int port, + int create_tun, struct mlx4_ib_demux_pv_ctx *ctx) +{ + int ret, cq_size; + + if (ctx->state != DEMUX_PV_STATE_DOWN) + return -EEXIST; + + ctx->state = DEMUX_PV_STATE_STARTING; + /* have QP0 only on port owner, and only if link layer is IB */ + if (ctx->slave == mlx4_master_func_num(to_mdev(ctx->ib_dev)->dev) && + rdma_port_get_link_layer(ibdev, ctx->port) == IB_LINK_LAYER_INFINIBAND) + ctx->has_smi = 1; + + if (ctx->has_smi) { + ret = mlx4_ib_alloc_pv_bufs(ctx, IB_QPT_SMI, create_tun); + if (ret) { + pr_err("Failed allocating qp0 tunnel bufs (%d)\n", ret); + goto err_out; + } + } + + ret = mlx4_ib_alloc_pv_bufs(ctx, IB_QPT_GSI, create_tun); + if (ret) { + pr_err("Failed allocating qp1 tunnel bufs (%d)\n", ret); + goto err_out_qp0; + } + + cq_size = 2 * MLX4_NUM_TUNNEL_BUFS; + if (ctx->has_smi) + cq_size *= 2; + + ctx->cq = ib_create_cq(ctx->ib_dev, mlx4_ib_tunnel_comp_handler, + NULL, ctx, cq_size, 0); + if (IS_ERR(ctx->cq)) { + ret = PTR_ERR(ctx->cq); + pr_err("Couldn't create tunnel CQ (%d)\n", ret); + goto err_buf; + } + + ctx->pd = ib_alloc_pd(ctx->ib_dev); + if (IS_ERR(ctx->pd)) { + ret = PTR_ERR(ctx->pd); + pr_err("Couldn't create tunnel PD (%d)\n", ret); + goto err_cq; + } + + ctx->mr = ib_get_dma_mr(ctx->pd, IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(ctx->mr)) { + ret = PTR_ERR(ctx->mr); + pr_err("Couldn't get tunnel DMA MR (%d)\n", ret); + goto err_pd; + } + + if (ctx->has_smi) { + ret = create_pv_sqp(ctx, IB_QPT_SMI, create_tun); + if (ret) { + pr_err("Couldn't create %s QP0 (%d)\n", + create_tun ? "tunnel for" : "", ret); + goto err_mr; + } + } + + ret = create_pv_sqp(ctx, IB_QPT_GSI, create_tun); + if (ret) { + pr_err("Couldn't create %s QP1 (%d)\n", + create_tun ? "tunnel for" : "", ret); + goto err_qp0; + } + + if (create_tun) + INIT_WORK(&ctx->work, mlx4_ib_tunnel_comp_worker); + else + INIT_WORK(&ctx->work, mlx4_ib_sqp_comp_worker); + + ctx->wq = to_mdev(ibdev)->sriov.demux[port - 1].wq; + + ret = ib_req_notify_cq(ctx->cq, IB_CQ_NEXT_COMP); + if (ret) { + pr_err("Couldn't arm tunnel cq (%d)\n", ret); + goto err_wq; + } + ctx->state = DEMUX_PV_STATE_ACTIVE; + return 0; + +err_wq: + ctx->wq = NULL; + ib_destroy_qp(ctx->qp[1].qp); + ctx->qp[1].qp = NULL; + + +err_qp0: + if (ctx->has_smi) + ib_destroy_qp(ctx->qp[0].qp); + ctx->qp[0].qp = NULL; + +err_mr: + ib_dereg_mr(ctx->mr); + ctx->mr = NULL; + +err_pd: + ib_dealloc_pd(ctx->pd); + ctx->pd = NULL; + +err_cq: + ib_destroy_cq(ctx->cq); + ctx->cq = NULL; + +err_buf: + mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_GSI, create_tun); + +err_out_qp0: + if (ctx->has_smi) + mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_SMI, create_tun); +err_out: + ctx->state = DEMUX_PV_STATE_DOWN; + return ret; +} + +static void destroy_pv_resources(struct mlx4_ib_dev *dev, int slave, int port, + struct mlx4_ib_demux_pv_ctx *ctx, int flush) +{ + if (!ctx) + return; + if (ctx->state > DEMUX_PV_STATE_DOWN) { + ctx->state = DEMUX_PV_STATE_DOWNING; + if (flush) + flush_workqueue(ctx->wq); + if (ctx->has_smi) { + ib_destroy_qp(ctx->qp[0].qp); + ctx->qp[0].qp = NULL; + mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_SMI, 1); + } + ib_destroy_qp(ctx->qp[1].qp); + ctx->qp[1].qp = NULL; + mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_GSI, 1); + ib_dereg_mr(ctx->mr); + ctx->mr = NULL; + ib_dealloc_pd(ctx->pd); + ctx->pd = NULL; + ib_destroy_cq(ctx->cq); + ctx->cq = NULL; + ctx->state = DEMUX_PV_STATE_DOWN; + } +} + +static int mlx4_ib_tunnels_update(struct mlx4_ib_dev *dev, int slave, + int port, int do_init) +{ + int ret = 0; + + if (!do_init) { + clean_vf_mcast(&dev->sriov.demux[port - 1], slave); + /* for master, destroy real sqp resources */ + if (slave == mlx4_master_func_num(dev->dev)) + destroy_pv_resources(dev, slave, port, + dev->sriov.sqps[port - 1], 1); + /* destroy the tunnel qp resources */ + destroy_pv_resources(dev, slave, port, + dev->sriov.demux[port - 1].tun[slave], 1); + return 0; + } + + /* create the tunnel qp resources */ + ret = create_pv_resources(&dev->ib_dev, slave, port, 1, + dev->sriov.demux[port - 1].tun[slave]); + + /* for master, create the real sqp resources */ + if (!ret && slave == mlx4_master_func_num(dev->dev)) + ret = create_pv_resources(&dev->ib_dev, slave, port, 0, + dev->sriov.sqps[port - 1]); + return ret; +} + +void mlx4_ib_tunnels_update_work(struct work_struct *work) +{ + struct mlx4_ib_demux_work *dmxw; + + dmxw = container_of(work, struct mlx4_ib_demux_work, work); + mlx4_ib_tunnels_update(dmxw->dev, dmxw->slave, (int) dmxw->port, + dmxw->do_init); + kfree(dmxw); + return; +} + +static int mlx4_ib_alloc_demux_ctx(struct mlx4_ib_dev *dev, + struct mlx4_ib_demux_ctx *ctx, + int port) +{ + char name[12]; + int ret = 0; + int i; + + ctx->tun = kcalloc(dev->dev->caps.sqp_demux, + sizeof (struct mlx4_ib_demux_pv_ctx *), GFP_KERNEL); + if (!ctx->tun) + return -ENOMEM; + + ctx->dev = dev; + ctx->port = port; + ctx->ib_dev = &dev->ib_dev; + + for (i = 0; i < dev->dev->caps.sqp_demux; i++) { + ret = alloc_pv_object(dev, i, port, &ctx->tun[i]); + if (ret) { + ret = -ENOMEM; + goto err_mcg; + } + } + + ret = mlx4_ib_mcg_port_init(ctx); + if (ret) { + pr_err("Failed initializing mcg para-virt (%d)\n", ret); + goto err_mcg; + } + + snprintf(name, sizeof name, "mlx4_ibt%d", port); + ctx->wq = create_singlethread_workqueue(name); + if (!ctx->wq) { + pr_err("Failed to create tunnelling WQ for port %d\n", port); + ret = -ENOMEM; + goto err_wq; + } + + snprintf(name, sizeof name, "mlx4_ibud%d", port); + ctx->ud_wq = create_singlethread_workqueue(name); + if (!ctx->ud_wq) { + pr_err("Failed to create up/down WQ for port %d\n", port); + ret = -ENOMEM; + goto err_udwq; + } + + return 0; + +err_udwq: + destroy_workqueue(ctx->wq); + ctx->wq = NULL; + +err_wq: + mlx4_ib_mcg_port_cleanup(ctx, 1); +err_mcg: + for (i = 0; i < dev->dev->caps.sqp_demux; i++) + free_pv_object(dev, i, port); + kfree(ctx->tun); + ctx->tun = NULL; + return ret; +} + +static void mlx4_ib_free_sqp_ctx(struct mlx4_ib_demux_pv_ctx *sqp_ctx) +{ + if (sqp_ctx->state > DEMUX_PV_STATE_DOWN) { + sqp_ctx->state = DEMUX_PV_STATE_DOWNING; + flush_workqueue(sqp_ctx->wq); + if (sqp_ctx->has_smi) { + ib_destroy_qp(sqp_ctx->qp[0].qp); + sqp_ctx->qp[0].qp = NULL; + mlx4_ib_free_pv_qp_bufs(sqp_ctx, IB_QPT_SMI, 0); + } + ib_destroy_qp(sqp_ctx->qp[1].qp); + sqp_ctx->qp[1].qp = NULL; + mlx4_ib_free_pv_qp_bufs(sqp_ctx, IB_QPT_GSI, 0); + ib_dereg_mr(sqp_ctx->mr); + sqp_ctx->mr = NULL; + ib_dealloc_pd(sqp_ctx->pd); + sqp_ctx->pd = NULL; + ib_destroy_cq(sqp_ctx->cq); + sqp_ctx->cq = NULL; + sqp_ctx->state = DEMUX_PV_STATE_DOWN; + } +} + +static void mlx4_ib_free_demux_ctx(struct mlx4_ib_demux_ctx *ctx) +{ + int i; + if (ctx) { + struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev); + mlx4_ib_mcg_port_cleanup(ctx, 1); + for (i = 0; i < dev->dev->caps.sqp_demux; i++) { + if (!ctx->tun[i]) + continue; + if (ctx->tun[i]->state > DEMUX_PV_STATE_DOWN) + ctx->tun[i]->state = DEMUX_PV_STATE_DOWNING; + } + flush_workqueue(ctx->wq); + for (i = 0; i < dev->dev->caps.sqp_demux; i++) { + destroy_pv_resources(dev, i, ctx->port, ctx->tun[i], 0); + free_pv_object(dev, i, ctx->port); + } + kfree(ctx->tun); + destroy_workqueue(ctx->ud_wq); + destroy_workqueue(ctx->wq); + } +} + +static void mlx4_ib_master_tunnels(struct mlx4_ib_dev *dev, int do_init) +{ + int i; + + if (!mlx4_is_master(dev->dev)) + return; + /* initialize or tear down tunnel QPs for the master */ + for (i = 0; i < dev->dev->caps.num_ports; i++) + mlx4_ib_tunnels_update(dev, mlx4_master_func_num(dev->dev), i + 1, do_init); + return; +} + +int mlx4_ib_init_sriov(struct mlx4_ib_dev *dev) +{ + int i = 0; + int err; + + if (!mlx4_is_mfunc(dev->dev)) + return 0; + + dev->sriov.is_going_down = 0; + spin_lock_init(&dev->sriov.going_down_lock); + mlx4_ib_cm_paravirt_init(dev); + + mlx4_ib_warn(&dev->ib_dev, "multi-function enabled\n"); + + if (mlx4_is_slave(dev->dev)) { + mlx4_ib_warn(&dev->ib_dev, "operating in qp1 tunnel mode\n"); + return 0; + } + + for (i = 0; i < dev->dev->caps.sqp_demux; i++) { + if (i == mlx4_master_func_num(dev->dev)) + mlx4_put_slave_node_guid(dev->dev, i, dev->ib_dev.node_guid); + else + mlx4_put_slave_node_guid(dev->dev, i, mlx4_ib_gen_node_guid()); + } + + err = mlx4_ib_init_alias_guid_service(dev); + if (err) { + mlx4_ib_warn(&dev->ib_dev, "Failed init alias guid process.\n"); + goto paravirt_err; + } + err = mlx4_ib_device_register_sysfs(dev); + if (err) { + mlx4_ib_warn(&dev->ib_dev, "Failed to register sysfs\n"); + goto sysfs_err; + } + + mlx4_ib_warn(&dev->ib_dev, "initializing demux service for %d qp1 clients\n", + dev->dev->caps.sqp_demux); + for (i = 0; i < dev->num_ports; i++) { + union ib_gid gid; + err = __mlx4_ib_query_gid(&dev->ib_dev, i + 1, 0, &gid, 1); + if (err) + goto demux_err; + dev->sriov.demux[i].guid_cache[0] = gid.global.interface_id; + err = alloc_pv_object(dev, mlx4_master_func_num(dev->dev), i + 1, + &dev->sriov.sqps[i]); + if (err) + goto demux_err; + err = mlx4_ib_alloc_demux_ctx(dev, &dev->sriov.demux[i], i + 1); + if (err) + goto demux_err; + } + mlx4_ib_master_tunnels(dev, 1); + return 0; + +demux_err: + while (i > 0) { + free_pv_object(dev, mlx4_master_func_num(dev->dev), i + 1); + mlx4_ib_free_demux_ctx(&dev->sriov.demux[i]); + --i; + } + mlx4_ib_device_unregister_sysfs(dev); + +sysfs_err: + mlx4_ib_destroy_alias_guid_service(dev); + +paravirt_err: + mlx4_ib_cm_paravirt_clean(dev, -1); + + return err; +} + +void mlx4_ib_close_sriov(struct mlx4_ib_dev *dev) +{ + int i; + unsigned long flags; + + if (!mlx4_is_mfunc(dev->dev)) + return; + + spin_lock_irqsave(&dev->sriov.going_down_lock, flags); + dev->sriov.is_going_down = 1; + spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags); + if (mlx4_is_master(dev->dev)) { + for (i = 0; i < dev->num_ports; i++) { + flush_workqueue(dev->sriov.demux[i].ud_wq); + mlx4_ib_free_sqp_ctx(dev->sriov.sqps[i]); + kfree(dev->sriov.sqps[i]); + dev->sriov.sqps[i] = NULL; + mlx4_ib_free_demux_ctx(&dev->sriov.demux[i]); + } + + mlx4_ib_cm_paravirt_clean(dev, -1); + mlx4_ib_destroy_alias_guid_service(dev); + mlx4_ib_device_unregister_sysfs(dev); } } Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/main.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/main.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/main.c (revision 273864) @@ -1,1580 +1,2417 @@ /* * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include -#include + +#ifdef __linux__ +#include +#endif + +#include #include #include #include -#include #include +#include +#include +#include #include #include #include #include #include - +#include #include "mlx4_ib.h" #include "user.h" #include "wc.h" #define DRV_NAME MLX4_IB_DRV_NAME -#define DRV_VERSION "1.0-ofed1.5.2" -#define DRV_RELDATE "August 4, 2010" +#define DRV_VERSION "1.0" +#define DRV_RELDATE "April 4, 2008" +#define MLX4_IB_DRIVER_PROC_DIR_NAME "driver/mlx4_ib" +#define MLX4_IB_MRS_PROC_DIR_NAME "mrs" + MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("Mellanox ConnectX HCA InfiniBand driver"); MODULE_LICENSE("Dual BSD/GPL"); MODULE_VERSION(DRV_VERSION); -#ifdef CONFIG_MLX4_DEBUG +int mlx4_ib_sm_guid_assign = 1; -int mlx4_ib_debug_level = 0; -module_param_named(debug_level, mlx4_ib_debug_level, int, 0644); -MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0"); +#ifdef __linux__ +struct proc_dir_entry *mlx4_mrs_dir_entry; +static struct proc_dir_entry *mlx4_ib_driver_dir_entry; +#endif -#endif /* CONFIG_MLX4_DEBUG */ +module_param_named(sm_guid_assign, mlx4_ib_sm_guid_assign, int, 0444); +MODULE_PARM_DESC(sm_guid_assign, "Enable SM alias_GUID assignment if sm_guid_assign > 0 (Default: 1)"); +static char dev_assign_str[512]; +//module_param_string(dev_assign_str, dev_assign_str, sizeof(dev_assign_str), 0644); +MODULE_PARM_DESC(dev_assign_str, "Map all device function numbers to " + "IB device numbers following the pattern: " + "bb:dd.f-0,bb:dd.f-1,... (all numbers are hexadecimals)." + " Max supported devices - 32"); + static const char mlx4_ib_version[] = DRV_NAME ": Mellanox ConnectX InfiniBand driver v" DRV_VERSION " (" DRV_RELDATE ")\n"; -static void *get_ibdev(struct mlx4_dev *dev, void *ctx, u8 port) -{ - struct mlx4_ib_dev *mlxibdev = ctx; - return &mlxibdev->ib_dev; -} - struct update_gid_work { - struct work_struct work; - union ib_gid gids[128]; - int port; - struct mlx4_ib_dev *dev; + struct work_struct work; + union ib_gid gids[128]; + struct mlx4_ib_dev *dev; + int port; }; +struct dev_rec { + int bus; + int dev; + int func; + int nr; +}; + +#define MAX_DR 32 +static struct dev_rec dr[MAX_DR]; + +static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init); + static struct workqueue_struct *wq; static void init_query_mad(struct ib_smp *mad) { mad->base_version = 1; mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; mad->class_version = 1; mad->method = IB_MGMT_METHOD_GET; } static union ib_gid zgid; static int mlx4_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props) { struct mlx4_ib_dev *dev = to_mdev(ibdev); struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; int err = -ENOMEM; in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); if (!in_mad || !out_mad) goto out; init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; - err = mlx4_MAD_IFC(to_mdev(ibdev), 1, 1, 1, NULL, NULL, in_mad, out_mad); + err = mlx4_MAD_IFC(to_mdev(ibdev), MLX4_MAD_IFC_IGNORE_KEYS, + 1, NULL, NULL, in_mad, out_mad); if (err) goto out; memset(props, 0, sizeof *props); props->fw_ver = dev->dev->caps.fw_ver; props->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT | IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_SYS_IMAGE_GUID | IB_DEVICE_RC_RNR_NAK_GEN | - IB_DEVICE_BLOCK_MULTICAST_LOOPBACK; + IB_DEVICE_BLOCK_MULTICAST_LOOPBACK | + IB_DEVICE_SHARED_MR; + if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR) props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR; if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR) props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR; if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_APM) props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG; if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UD_AV_PORT) props->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE; if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_IPOIB_CSUM) props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM; if (dev->dev->caps.max_gso_sz && dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BLH) props->device_cap_flags |= IB_DEVICE_UD_TSO; if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_RESERVED_LKEY) props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY; if ((dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_LOCAL_INV) && (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_REMOTE_INV) && (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_FAST_REG_WR)) props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) props->device_cap_flags |= IB_DEVICE_XRC; - if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_RAW_ETY) - props->max_raw_ethy_qp = dev->ib_dev.phys_port_cnt; + props->device_cap_flags |= IB_DEVICE_QPG; + if (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS) { + props->device_cap_flags |= IB_DEVICE_UD_RSS; + props->max_rss_tbl_sz = dev->dev->caps.max_rss_tbl_sz; + } props->vendor_id = be32_to_cpup((__be32 *) (out_mad->data + 36)) & 0xffffff; - props->vendor_part_id = be16_to_cpup((__be16 *) (out_mad->data + 30)); + props->vendor_part_id = dev->dev->pdev->device; props->hw_ver = be32_to_cpup((__be32 *) (out_mad->data + 32)); memcpy(&props->sys_image_guid, out_mad->data + 4, 8); props->max_mr_size = ~0ull; props->page_size_cap = dev->dev->caps.page_size_cap; - props->max_qp = dev->dev->caps.num_qps - dev->dev->caps.reserved_qps; + props->max_qp = dev->dev->quotas.qp; props->max_qp_wr = dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE; props->max_sge = min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg); - props->max_cq = dev->dev->caps.num_cqs - dev->dev->caps.reserved_cqs; + props->max_cq = dev->dev->quotas.cq; props->max_cqe = dev->dev->caps.max_cqes; - props->max_mr = dev->dev->caps.num_mpts - dev->dev->caps.reserved_mrws; + props->max_mr = dev->dev->quotas.mpt; props->max_pd = dev->dev->caps.num_pds - dev->dev->caps.reserved_pds; props->max_qp_rd_atom = dev->dev->caps.max_qp_dest_rdma; props->max_qp_init_rd_atom = dev->dev->caps.max_qp_init_rdma; props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; - props->max_srq = dev->dev->caps.num_srqs - dev->dev->caps.reserved_srqs; + props->max_srq = dev->dev->quotas.srq; props->max_srq_wr = dev->dev->caps.max_srq_wqes - 1; props->max_srq_sge = dev->dev->caps.max_srq_sge; - props->max_fast_reg_page_list_len = MAX_FAST_REG_PAGES; + props->max_fast_reg_page_list_len = MLX4_MAX_FAST_REG_PAGES; props->local_ca_ack_delay = dev->dev->caps.local_ca_ack_delay; props->atomic_cap = dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_ATOMIC ? IB_ATOMIC_HCA : IB_ATOMIC_NONE; - props->masked_atomic_cap = IB_ATOMIC_HCA; + props->masked_atomic_cap = props->atomic_cap; props->max_pkeys = dev->dev->caps.pkey_table_len[1]; props->max_mcast_grp = dev->dev->caps.num_mgms + dev->dev->caps.num_amgms; props->max_mcast_qp_attach = dev->dev->caps.num_qp_per_mgm; props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * props->max_mcast_grp; - props->max_map_per_fmr = (1 << (32 - ilog2(dev->dev->caps.num_mpts))) - 1; + props->max_map_per_fmr = dev->dev->caps.max_fmr_maps; out: kfree(in_mad); kfree(out_mad); return err; } static enum rdma_link_layer mlx4_ib_port_link_layer(struct ib_device *device, u8 port_num) { struct mlx4_dev *dev = to_mdev(device)->dev; return dev->caps.port_mask[port_num] == MLX4_PORT_TYPE_IB ? IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET; } -static void ib_link_query_port(struct ib_device *ibdev, u8 port, - struct ib_port_attr *props, - struct ib_smp *out_mad) +static int ib_link_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props, int netw_view) { + struct ib_smp *in_mad = NULL; + struct ib_smp *out_mad = NULL; + int ext_active_speed; + int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS; + int err = -ENOMEM; + + in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); + out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); + if (!in_mad || !out_mad) + goto out; + + init_query_mad(in_mad); + in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; + in_mad->attr_mod = cpu_to_be32(port); + + if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view) + mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW; + + err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL, + in_mad, out_mad); + if (err) + goto out; + + props->lid = be16_to_cpup((__be16 *) (out_mad->data + 16)); props->lmc = out_mad->data[34] & 0x7; props->sm_lid = be16_to_cpup((__be16 *) (out_mad->data + 18)); props->sm_sl = out_mad->data[36] & 0xf; props->state = out_mad->data[32] & 0xf; props->phys_state = out_mad->data[33] >> 4; props->port_cap_flags = be32_to_cpup((__be32 *) (out_mad->data + 20)); - props->gid_tbl_len = to_mdev(ibdev)->dev->caps.gid_table_len[port]; + if (netw_view) + props->gid_tbl_len = out_mad->data[50]; + else + props->gid_tbl_len = to_mdev(ibdev)->dev->caps.gid_table_len[port]; props->max_msg_sz = to_mdev(ibdev)->dev->caps.max_msg_sz; props->pkey_tbl_len = to_mdev(ibdev)->dev->caps.pkey_table_len[port]; props->bad_pkey_cntr = be16_to_cpup((__be16 *) (out_mad->data + 46)); props->qkey_viol_cntr = be16_to_cpup((__be16 *) (out_mad->data + 48)); props->active_width = out_mad->data[31] & 0xf; props->active_speed = out_mad->data[35] >> 4; props->max_mtu = out_mad->data[41] & 0xf; props->active_mtu = out_mad->data[36] >> 4; props->subnet_timeout = out_mad->data[51] & 0x1f; props->max_vl_num = out_mad->data[37] >> 4; props->init_type_reply = out_mad->data[41] >> 4; - props->link_layer = IB_LINK_LAYER_INFINIBAND; -} -#ifdef notyet -static int eth_to_ib_width(int w) -{ - switch (w) { - case 4: - return IB_WIDTH_4X; - case 8: - case 16: - return IB_WIDTH_8X; - case 32: - return IB_WIDTH_12X; - default: - return IB_WIDTH_1X; + /* Check if extended speeds (EDR/FDR/...) are supported */ + if (props->port_cap_flags & IB_PORT_EXTENDED_SPEEDS_SUP) { + ext_active_speed = out_mad->data[62] >> 4; + + switch (ext_active_speed) { + case 1: + props->active_speed = IB_SPEED_FDR; + break; + case 2: + props->active_speed = IB_SPEED_EDR; + break; + } } -} -static int eth_to_ib_speed(int s) -{ - switch (s) { - case 256: - return 1; - case 512: - return 2; - case 1024: - return 4; - default: - return 1; + /* If reported active speed is QDR, check if is FDR-10 */ + if (props->active_speed == IB_SPEED_QDR) { + init_query_mad(in_mad); + in_mad->attr_id = MLX4_ATTR_EXTENDED_PORT_INFO; + in_mad->attr_mod = cpu_to_be32(port); + + err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, + NULL, NULL, in_mad, out_mad); + if (err) + goto out; + + /* Checking LinkSpeedActive for FDR-10 */ + if (out_mad->data[15] & 0x1) + props->active_speed = IB_SPEED_FDR10; } + + /* Avoid wrong speed value returned by FW if the IB link is down. */ + if (props->state == IB_PORT_DOWN) + props->active_speed = IB_SPEED_SDR; + +out: + kfree(in_mad); + kfree(out_mad); + return err; } -#endif static u8 state_to_phys_state(enum ib_port_state state) { return state == IB_PORT_ACTIVE ? 5 : 3; } static int eth_link_query_port(struct ib_device *ibdev, u8 port, - struct ib_port_attr *props, - struct ib_smp *out_mad) + struct ib_port_attr *props, int netw_view) { - struct mlx4_ib_iboe *iboe = &to_mdev(ibdev)->iboe; + + struct mlx4_ib_dev *mdev = to_mdev(ibdev); + struct mlx4_ib_iboe *iboe = &mdev->iboe; struct net_device *ndev; enum ib_mtu tmp; + struct mlx4_cmd_mailbox *mailbox; + int err = 0; - props->active_width = IB_WIDTH_4X; - props->active_speed = 1; + mailbox = mlx4_alloc_cmd_mailbox(mdev->dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, port, 0, + MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); + if (err) + goto out; + + props->active_width = (((u8 *)mailbox->buf)[5] == 0x40) ? + IB_WIDTH_4X : IB_WIDTH_1X; + props->active_speed = IB_SPEED_QDR; props->port_cap_flags = IB_PORT_CM_SUP; - props->gid_tbl_len = to_mdev(ibdev)->dev->caps.gid_table_len[port]; - props->max_msg_sz = to_mdev(ibdev)->dev->caps.max_msg_sz; + if (netw_view) + props->gid_tbl_len = MLX4_ROCE_MAX_GIDS; + else + props->gid_tbl_len = mdev->dev->caps.gid_table_len[port]; + + props->max_msg_sz = mdev->dev->caps.max_msg_sz; props->pkey_tbl_len = 1; - props->bad_pkey_cntr = be16_to_cpup((__be16 *) (out_mad->data + 46)); - props->qkey_viol_cntr = be16_to_cpup((__be16 *) (out_mad->data + 48)); - props->max_mtu = IB_MTU_2048; - props->subnet_timeout = 0; - props->max_vl_num = out_mad->data[37] >> 4; - props->init_type_reply = 0; - props->link_layer = IB_LINK_LAYER_ETHERNET; + props->max_mtu = IB_MTU_4096; + props->max_vl_num = 2; props->state = IB_PORT_DOWN; props->phys_state = state_to_phys_state(props->state); props->active_mtu = IB_MTU_256; spin_lock(&iboe->lock); ndev = iboe->netdevs[port - 1]; if (!ndev) - goto out; + goto out_unlock; -#ifdef __linux__ - tmp = iboe_get_mtu(ndev->mtu); -#else tmp = iboe_get_mtu(ndev->if_mtu); -#endif props->active_mtu = tmp ? min(props->max_mtu, tmp) : IB_MTU_256; - props->state = netif_carrier_ok(ndev) && netif_oper_up(ndev) ? + + props->state = (netif_running(ndev) && netif_carrier_ok(ndev)) ? IB_PORT_ACTIVE : IB_PORT_DOWN; props->phys_state = state_to_phys_state(props->state); - -out: +out_unlock: spin_unlock(&iboe->lock); - return 0; +out: + mlx4_free_cmd_mailbox(mdev->dev, mailbox); + return err; } -static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port, - struct ib_port_attr *props) +int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props, int netw_view) { - struct ib_smp *in_mad = NULL; - struct ib_smp *out_mad = NULL; - int err = -ENOMEM; + int err; - in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) - goto out; - memset(props, 0, sizeof *props); - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; - in_mad->attr_mod = cpu_to_be32(port); + err = mlx4_ib_port_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND ? + ib_link_query_port(ibdev, port, props, netw_view) : + eth_link_query_port(ibdev, port, props, netw_view); - err = mlx4_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad); - if (err) - goto out; - - mlx4_ib_port_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND ? - ib_link_query_port(ibdev, port, props, out_mad) : - eth_link_query_port(ibdev, port, props, out_mad); - -out: - kfree(in_mad); - kfree(out_mad); - return err; } -static int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, - union ib_gid *gid) +static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props) { + /* returns host view */ + return __mlx4_ib_query_port(ibdev, port, props, 0); +} + +int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, + union ib_gid *gid, int netw_view) +{ struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; int err = -ENOMEM; + struct mlx4_ib_dev *dev = to_mdev(ibdev); + int clear = 0; + int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS; in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); if (!in_mad || !out_mad) goto out; init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; in_mad->attr_mod = cpu_to_be32(port); - err = mlx4_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad); + if (mlx4_is_mfunc(dev->dev) && netw_view) + mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW; + + err = mlx4_MAD_IFC(dev, mad_ifc_flags, port, NULL, NULL, in_mad, out_mad); if (err) goto out; memcpy(gid->raw, out_mad->data + 8, 8); + if (mlx4_is_mfunc(dev->dev) && !netw_view) { + if (index) { + /* For any index > 0, return the null guid */ + err = 0; + clear = 1; + goto out; + } + } + init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; in_mad->attr_mod = cpu_to_be32(index / 8); - err = mlx4_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad); + err = mlx4_MAD_IFC(dev, mad_ifc_flags, port, + NULL, NULL, in_mad, out_mad); if (err) goto out; memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8); out: + if (clear) + memset(gid->raw + 8, 0, 8); kfree(in_mad); kfree(out_mad); return err; } static int iboe_query_gid(struct ib_device *ibdev, u8 port, int index, - union ib_gid *gid) + union ib_gid *gid) { struct mlx4_ib_dev *dev = to_mdev(ibdev); *gid = dev->iboe.gid_table[port - 1][index]; return 0; } static int mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid) { if (rdma_port_get_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND) - return __mlx4_ib_query_gid(ibdev, port, index, gid); + return __mlx4_ib_query_gid(ibdev, port, index, gid, 0); else return iboe_query_gid(ibdev, port, index, gid); } -static int mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, - u16 *pkey) +int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, + u16 *pkey, int netw_view) { struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; + int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS; int err = -ENOMEM; in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); if (!in_mad || !out_mad) goto out; init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE; in_mad->attr_mod = cpu_to_be32(index / 32); - err = mlx4_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad); + if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view) + mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW; + + err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL, + in_mad, out_mad); if (err) goto out; *pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]); out: kfree(in_mad); kfree(out_mad); return err; } +static int mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) +{ + return __mlx4_ib_query_pkey(ibdev, port, index, pkey, 0); +} + static int mlx4_ib_modify_device(struct ib_device *ibdev, int mask, struct ib_device_modify *props) { struct mlx4_cmd_mailbox *mailbox; - int err; + unsigned long flags; if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) return -EOPNOTSUPP; if (!(mask & IB_DEVICE_MODIFY_NODE_DESC)) return 0; - spin_lock(&to_mdev(ibdev)->sm_lock); + if (mlx4_is_slave(to_mdev(ibdev)->dev)) + return -EOPNOTSUPP; + + spin_lock_irqsave(&to_mdev(ibdev)->sm_lock, flags); memcpy(ibdev->node_desc, props->node_desc, 64); - spin_unlock(&to_mdev(ibdev)->sm_lock); + spin_unlock_irqrestore(&to_mdev(ibdev)->sm_lock, flags); - /* if possible, pass node desc to FW, so it can generate - * a 144 trap. If cmd fails, just ignore. + /* + * If possible, pass node desc to FW, so it can generate + * a 144 trap. If cmd fails, just ignore. */ mailbox = mlx4_alloc_cmd_mailbox(to_mdev(ibdev)->dev); if (IS_ERR(mailbox)) return 0; memset(mailbox->buf, 0, 256); memcpy(mailbox->buf, props->node_desc, 64); - err = mlx4_cmd(to_mdev(ibdev)->dev, mailbox->dma, 1, 0, - MLX4_CMD_SET_NODE, MLX4_CMD_TIME_CLASS_A); - if (err) - mlx4_ib_dbg("SET_NODE command failed (%d)", err); + mlx4_cmd(to_mdev(ibdev)->dev, mailbox->dma, 1, 0, + MLX4_CMD_SET_NODE, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); mlx4_free_cmd_mailbox(to_mdev(ibdev)->dev, mailbox); return 0; } static int mlx4_SET_PORT(struct mlx4_ib_dev *dev, u8 port, int reset_qkey_viols, u32 cap_mask) { struct mlx4_cmd_mailbox *mailbox; int err; u8 is_eth = dev->dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH; mailbox = mlx4_alloc_cmd_mailbox(dev->dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); memset(mailbox->buf, 0, 256); if (dev->dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { *(u8 *) mailbox->buf = !!reset_qkey_viols << 6; ((__be32 *) mailbox->buf)[2] = cpu_to_be32(cap_mask); } else { ((u8 *) mailbox->buf)[3] = !!reset_qkey_viols; ((__be32 *) mailbox->buf)[1] = cpu_to_be32(cap_mask); } err = mlx4_cmd(dev->dev, mailbox->dma, port, is_eth, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B); + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); mlx4_free_cmd_mailbox(dev->dev, mailbox); return err; } static int mlx4_ib_modify_port(struct ib_device *ibdev, u8 port, int mask, struct ib_port_modify *props) { struct ib_port_attr attr; u32 cap_mask; int err; mutex_lock(&to_mdev(ibdev)->cap_mask_mutex); err = mlx4_ib_query_port(ibdev, port, &attr); if (err) goto out; cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) & ~props->clr_port_cap_mask; err = mlx4_SET_PORT(to_mdev(ibdev), port, !!(mask & IB_PORT_RESET_QKEY_CNTR), cap_mask); out: mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex); return err; } static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(ibdev); struct mlx4_ib_ucontext *context; + struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3; struct mlx4_ib_alloc_ucontext_resp resp; int err; if (!dev->ib_active) return ERR_PTR(-EAGAIN); - resp.qp_tab_size = dev->dev->caps.num_qps; - - if (mlx4_wc_enabled()) { - resp.bf_reg_size = dev->dev->caps.bf_reg_size; - resp.bf_regs_per_page = dev->dev->caps.bf_regs_per_page; + if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) { + resp_v3.qp_tab_size = dev->dev->caps.num_qps; + if (mlx4_wc_enabled()) { + resp_v3.bf_reg_size = dev->dev->caps.bf_reg_size; + resp_v3.bf_regs_per_page = dev->dev->caps.bf_regs_per_page; + } else { + resp_v3.bf_reg_size = 0; + resp_v3.bf_regs_per_page = 0; + } } else { - resp.bf_reg_size = 0; - resp.bf_regs_per_page = 0; + resp.dev_caps = dev->dev->caps.userspace_caps; + resp.qp_tab_size = dev->dev->caps.num_qps; + if (mlx4_wc_enabled()) { + resp.bf_reg_size = dev->dev->caps.bf_reg_size; + resp.bf_regs_per_page = dev->dev->caps.bf_regs_per_page; + } else { + resp.bf_reg_size = 0; + resp.bf_regs_per_page = 0; + } + resp.cqe_size = dev->dev->caps.cqe_size; } - context = kzalloc(sizeof *context, GFP_KERNEL); + context = kmalloc(sizeof *context, GFP_KERNEL); if (!context) return ERR_PTR(-ENOMEM); err = mlx4_uar_alloc(to_mdev(ibdev)->dev, &context->uar); if (err) { kfree(context); return ERR_PTR(err); } INIT_LIST_HEAD(&context->db_page_list); mutex_init(&context->db_page_mutex); - err = ib_copy_to_udata(udata, &resp, sizeof resp); + if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) + err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3)); + else + err = ib_copy_to_udata(udata, &resp, sizeof(resp)); + if (err) { mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar); kfree(context); return ERR_PTR(-EFAULT); } return &context->ibucontext; } static int mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) { struct mlx4_ib_ucontext *context = to_mucontext(ibcontext); mlx4_uar_free(to_mdev(ibcontext->device)->dev, &context->uar); kfree(context); return 0; } +#ifdef __linux__ +static unsigned long mlx4_ib_get_unmapped_area(struct file *file, + unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long start_addr; + unsigned long page_size_order; + unsigned long command; + mm = current->mm; + if (addr) + return current->mm->get_unmapped_area(file, addr, len, + pgoff, flags); + + /* Last 8 bits hold the command others are data per that command */ + command = pgoff & MLX4_IB_MMAP_CMD_MASK; + if (command != MLX4_IB_MMAP_GET_CONTIGUOUS_PAGES) + return current->mm->get_unmapped_area(file, addr, len, + pgoff, flags); + + page_size_order = pgoff >> MLX4_IB_MMAP_CMD_BITS; + /* code is based on the huge-pages get_unmapped_area code */ + start_addr = mm->free_area_cache; + + if (len <= mm->cached_hole_size) + start_addr = TASK_UNMAPPED_BASE; + + +full_search: + addr = ALIGN(start_addr, 1 << page_size_order); + + for (vma = find_vma(mm, addr); ; vma = vma->vm_next) { + /* At this point: (!vma || addr < vma->vm_end). */ + if (TASK_SIZE - len < addr) { + /* + * Start a new search - just in case we missed + * some holes. + */ + if (start_addr != TASK_UNMAPPED_BASE) { + start_addr = TASK_UNMAPPED_BASE; + goto full_search; + } + return -ENOMEM; + } + + if (!vma || addr + len <= vma->vm_start) + return addr; + addr = ALIGN(vma->vm_end, 1 << page_size_order); + } +} +#endif + static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) { struct mlx4_ib_dev *dev = to_mdev(context->device); + int err; - if (vma->vm_end - vma->vm_start != PAGE_SIZE) - return -EINVAL; + /* Last 8 bits hold the command others are data per that command */ + unsigned long command = vma->vm_pgoff & MLX4_IB_MMAP_CMD_MASK; - if (vma->vm_pgoff == 0) { + if (command < MLX4_IB_MMAP_GET_CONTIGUOUS_PAGES) { + /* compatability handling for commands 0 & 1*/ + if (vma->vm_end - vma->vm_start != PAGE_SIZE) + return -EINVAL; + } + if (command == MLX4_IB_MMAP_UAR_PAGE) { vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); if (io_remap_pfn_range(vma, vma->vm_start, to_mucontext(context)->uar.pfn, PAGE_SIZE, vma->vm_page_prot)) return -EAGAIN; - } else if (vma->vm_pgoff == 1 && dev->dev->caps.bf_reg_size != 0) { + } else if (command == MLX4_IB_MMAP_BLUE_FLAME_PAGE && + dev->dev->caps.bf_reg_size != 0) { vma->vm_page_prot = pgprot_wc(vma->vm_page_prot); if (io_remap_pfn_range(vma, vma->vm_start, to_mucontext(context)->uar.pfn + dev->dev->caps.num_uars, PAGE_SIZE, vma->vm_page_prot)) return -EAGAIN; + } else if (command == MLX4_IB_MMAP_GET_CONTIGUOUS_PAGES) { + /* Getting contiguous physical pages */ + unsigned long total_size = vma->vm_end - vma->vm_start; + unsigned long page_size_order = (vma->vm_pgoff) >> + MLX4_IB_MMAP_CMD_BITS; + struct ib_cmem *ib_cmem; + ib_cmem = ib_cmem_alloc_contiguous_pages(context, total_size, + page_size_order); + if (IS_ERR(ib_cmem)) { + err = PTR_ERR(ib_cmem); + return err; + } + + err = ib_cmem_map_contiguous_pages_to_vma(ib_cmem, vma); + if (err) { + ib_cmem_release_contiguous_pages(ib_cmem); + return err; + } + return 0; } else return -EINVAL; return 0; } static struct ib_pd *mlx4_ib_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context, struct ib_udata *udata) { struct mlx4_ib_pd *pd; int err; - pd = kzalloc(sizeof *pd, GFP_KERNEL); + pd = kmalloc(sizeof *pd, GFP_KERNEL); if (!pd) return ERR_PTR(-ENOMEM); err = mlx4_pd_alloc(to_mdev(ibdev)->dev, &pd->pdn); if (err) { kfree(pd); return ERR_PTR(err); } if (context) if (ib_copy_to_udata(udata, &pd->pdn, sizeof (__u32))) { mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn); kfree(pd); return ERR_PTR(-EFAULT); } return &pd->ibpd; } static int mlx4_ib_dealloc_pd(struct ib_pd *pd) { mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn); kfree(pd); return 0; } +static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev, + struct ib_ucontext *context, + struct ib_udata *udata) +{ + struct mlx4_ib_xrcd *xrcd; + int err; + + if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) + return ERR_PTR(-ENOSYS); + + xrcd = kmalloc(sizeof *xrcd, GFP_KERNEL); + if (!xrcd) + return ERR_PTR(-ENOMEM); + + err = mlx4_xrcd_alloc(to_mdev(ibdev)->dev, &xrcd->xrcdn); + if (err) + goto err1; + + xrcd->pd = ib_alloc_pd(ibdev); + if (IS_ERR(xrcd->pd)) { + err = PTR_ERR(xrcd->pd); + goto err2; + } + + xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, 1, 0); + if (IS_ERR(xrcd->cq)) { + err = PTR_ERR(xrcd->cq); + goto err3; + } + + return &xrcd->ibxrcd; + +err3: + ib_dealloc_pd(xrcd->pd); +err2: + mlx4_xrcd_free(to_mdev(ibdev)->dev, xrcd->xrcdn); +err1: + kfree(xrcd); + return ERR_PTR(err); +} + +static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd) +{ + ib_destroy_cq(to_mxrcd(xrcd)->cq); + ib_dealloc_pd(to_mxrcd(xrcd)->pd); + mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn); + kfree(xrcd); + + return 0; +} + static int add_gid_entry(struct ib_qp *ibqp, union ib_gid *gid) { struct mlx4_ib_qp *mqp = to_mqp(ibqp); struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); - struct gid_entry *ge; + struct mlx4_ib_gid_entry *ge; ge = kzalloc(sizeof *ge, GFP_KERNEL); if (!ge) return -ENOMEM; ge->gid = *gid; if (mlx4_ib_add_mc(mdev, mqp, gid)) { ge->port = mqp->port; ge->added = 1; } mutex_lock(&mqp->mutex); list_add_tail(&ge->list, &mqp->gid_list); mutex_unlock(&mqp->mutex); return 0; } int mlx4_ib_add_mc(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, union ib_gid *gid) { u8 mac[6]; struct net_device *ndev; int ret = 0; if (!mqp->port) return 0; + spin_lock(&mdev->iboe.lock); ndev = mdev->iboe.netdevs[mqp->port - 1]; if (ndev) dev_hold(ndev); spin_unlock(&mdev->iboe.lock); + if (ndev) { rdma_get_mcast_mac((struct in6_addr *)gid, mac); rtnl_lock(); dev_mc_add(mdev->iboe.netdevs[mqp->port - 1], mac, 6, 0); ret = 1; rtnl_unlock(); dev_put(ndev); } return ret; } +struct mlx4_ib_steering { + struct list_head list; + u64 reg_id; + union ib_gid gid; +}; + static int mlx4_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) { int err; struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); struct mlx4_ib_qp *mqp = to_mqp(ibqp); + u64 reg_id; + struct mlx4_ib_steering *ib_steering = NULL; - err = mlx4_multicast_attach(mdev->dev, &mqp->mqp, gid->raw, !!(mqp->flags & - MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK), - (ibqp->qp_type == IB_QPT_RAW_ETH) ? - MLX4_MCAST_PROT_EN : MLX4_MCAST_PROT_IB); + if (mdev->dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) { + ib_steering = kmalloc(sizeof(*ib_steering), GFP_KERNEL); + if (!ib_steering) + return -ENOMEM; + } + + err = mlx4_multicast_attach(mdev->dev, &mqp->mqp, gid->raw, mqp->port, + !!(mqp->flags & + MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK), + MLX4_PROT_IB_IPV6, ®_id); if (err) - return err; + goto err_malloc; err = add_gid_entry(ibqp, gid); if (err) goto err_add; + if (ib_steering) { + memcpy(ib_steering->gid.raw, gid->raw, 16); + ib_steering->reg_id = reg_id; + mutex_lock(&mqp->mutex); + list_add(&ib_steering->list, &mqp->steering_rules); + mutex_unlock(&mqp->mutex); + } return 0; err_add: mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw, - (ibqp->qp_type == IB_QPT_RAW_ETH) ? - MLX4_MCAST_PROT_EN : MLX4_MCAST_PROT_IB); + MLX4_PROT_IB_IPV6, reg_id); +err_malloc: + kfree(ib_steering); + return err; } -static struct gid_entry *find_gid_entry(struct mlx4_ib_qp *qp, u8 *raw) +enum { + IBV_FLOW_L4_NONE = 0, + IBV_FLOW_L4_OTHER = 3, + IBV_FLOW_L4_UDP = 5, + IBV_FLOW_L4_TCP = 6 +}; + +struct mlx4_cm_steering { + struct list_head list; + u64 reg_id; + struct ib_flow_spec spec; +}; + +static int flow_spec_to_net_rule(struct ib_device *dev, struct ib_flow_spec *flow_spec, + struct list_head *rule_list_h) { - struct gid_entry *ge; - struct gid_entry *tmp; - struct gid_entry *ret = NULL; + struct mlx4_spec_list *spec_l2, *spec_l3, *spec_l4; + u64 mac_msk = cpu_to_be64(MLX4_MAC_MASK << 16); + spec_l2 = kzalloc(sizeof *spec_l2, GFP_KERNEL); + if (!spec_l2) + return -ENOMEM; + + switch (flow_spec->type) { + case IB_FLOW_ETH: + spec_l2->id = MLX4_NET_TRANS_RULE_ID_ETH; + memcpy(spec_l2->eth.dst_mac, flow_spec->l2_id.eth.mac, ETH_ALEN); + memcpy(spec_l2->eth.dst_mac_msk, &mac_msk, ETH_ALEN); + spec_l2->eth.ether_type = flow_spec->l2_id.eth.ethertype; + if (flow_spec->l2_id.eth.vlan_present) { + spec_l2->eth.vlan_id = flow_spec->l2_id.eth.vlan; + spec_l2->eth.vlan_id_msk = cpu_to_be16(0x0fff); + } + break; + case IB_FLOW_IB_UC: + spec_l2->id = MLX4_NET_TRANS_RULE_ID_IB; + if(flow_spec->l2_id.ib_uc.qpn) { + spec_l2->ib.l3_qpn = cpu_to_be32(flow_spec->l2_id.ib_uc.qpn); + spec_l2->ib.qpn_msk = cpu_to_be32(0xffffff); + } + break; + case IB_FLOW_IB_MC_IPV4: + case IB_FLOW_IB_MC_IPV6: + spec_l2->id = MLX4_NET_TRANS_RULE_ID_IB; + memcpy(spec_l2->ib.dst_gid, flow_spec->l2_id.ib_mc.mgid, 16); + memset(spec_l2->ib.dst_gid_msk, 0xff, 16); + break; + } + + + list_add_tail(&spec_l2->list, rule_list_h); + + if (flow_spec->l2_id.eth.ethertype == cpu_to_be16(ETH_P_IP) || + flow_spec->type != IB_FLOW_ETH) { + spec_l3 = kzalloc(sizeof *spec_l3, GFP_KERNEL); + if (!spec_l3) + return -ENOMEM; + + spec_l3->id = MLX4_NET_TRANS_RULE_ID_IPV4; + spec_l3->ipv4.src_ip = flow_spec->src_ip; + if (flow_spec->type != IB_FLOW_IB_MC_IPV4 && + flow_spec->type != IB_FLOW_IB_MC_IPV6) + spec_l3->ipv4.dst_ip = flow_spec->dst_ip; + + if (spec_l3->ipv4.src_ip) + spec_l3->ipv4.src_ip_msk = MLX4_BE_WORD_MASK; + if (spec_l3->ipv4.dst_ip) + spec_l3->ipv4.dst_ip_msk = MLX4_BE_WORD_MASK; + + list_add_tail(&spec_l3->list, rule_list_h); + } + + if (flow_spec->l4_protocol) { + spec_l4 = kzalloc(sizeof(*spec_l4), GFP_KERNEL); + if (!spec_l4) + return -ENOMEM; + + spec_l4->tcp_udp.src_port = flow_spec->src_port; + spec_l4->tcp_udp.dst_port = flow_spec->dst_port; + if (spec_l4->tcp_udp.src_port) + spec_l4->tcp_udp.src_port_msk = + MLX4_BE_SHORT_MASK; + if (spec_l4->tcp_udp.dst_port) + spec_l4->tcp_udp.dst_port_msk = + MLX4_BE_SHORT_MASK; + + switch (flow_spec->l4_protocol) { + case IBV_FLOW_L4_UDP: + spec_l4->id = MLX4_NET_TRANS_RULE_ID_UDP; + break; + case IBV_FLOW_L4_TCP: + spec_l4->id = MLX4_NET_TRANS_RULE_ID_TCP; + break; + default: + dev_err(dev->dma_device, + "Unsupported l4 protocol.\n"); + kfree(spec_l4); + return -EPROTONOSUPPORT; + } + list_add_tail(&spec_l4->list, rule_list_h); + } + return 0; +} + +static int __mlx4_ib_flow_attach(struct mlx4_ib_dev *mdev, + struct mlx4_ib_qp *mqp, + struct ib_flow_spec *flow_spec, + int priority, int lock_qp) +{ + u64 reg_id = 0; + int err = 0; + struct mlx4_cm_steering *cm_flow; + struct mlx4_spec_list *spec, *tmp_spec; + + struct mlx4_net_trans_rule rule = + { .queue_mode = MLX4_NET_TRANS_Q_FIFO, + .exclusive = 0, + }; + + rule.promisc_mode = flow_spec->rule_type; + rule.port = mqp->port; + rule.qpn = mqp->mqp.qpn; + INIT_LIST_HEAD(&rule.list); + + cm_flow = kmalloc(sizeof(*cm_flow), GFP_KERNEL); + if (!cm_flow) + return -ENOMEM; + + if (rule.promisc_mode == MLX4_FS_REGULAR) { + rule.allow_loopback = !flow_spec->block_mc_loopback; + rule.priority = MLX4_DOMAIN_UVERBS | priority; + err = flow_spec_to_net_rule(&mdev->ib_dev, flow_spec, + &rule.list); + if (err) + goto free_list; + } + + err = mlx4_flow_attach(mdev->dev, &rule, ®_id); + if (err) + goto free_list; + + memcpy(&cm_flow->spec, flow_spec, sizeof(*flow_spec)); + cm_flow->reg_id = reg_id; + + if (lock_qp) + mutex_lock(&mqp->mutex); + list_add(&cm_flow->list, &mqp->rules_list); + if (lock_qp) + mutex_unlock(&mqp->mutex); + +free_list: + list_for_each_entry_safe(spec, tmp_spec, &rule.list, list) { + list_del(&spec->list); + kfree(spec); + } + if (err) { + kfree(cm_flow); + dev_err(mdev->ib_dev.dma_device, + "Fail to attach flow steering rule\n"); + } + return err; +} + +static int __mlx4_ib_flow_detach(struct mlx4_ib_dev *mdev, + struct mlx4_ib_qp *mqp, + struct ib_flow_spec *spec, int priority, + int lock_qp) +{ + struct mlx4_cm_steering *cm_flow; + int ret; + + if (lock_qp) + mutex_lock(&mqp->mutex); + list_for_each_entry(cm_flow, &mqp->rules_list, list) { + if (!memcmp(&cm_flow->spec, spec, sizeof(*spec))) { + list_del(&cm_flow->list); + break; + } + } + if (lock_qp) + mutex_unlock(&mqp->mutex); + + if (&cm_flow->list == &mqp->rules_list) { + dev_err(mdev->ib_dev.dma_device, "Couldn't find reg_id for flow spec. " + "Steering rule is left attached\n"); + return -EINVAL; + } + + ret = mlx4_flow_detach(mdev->dev, cm_flow->reg_id); + + kfree(cm_flow); + return ret; +} + +static int mlx4_ib_flow_attach(struct ib_qp *qp, struct ib_flow_spec *flow_spec, + int priority) +{ + return __mlx4_ib_flow_attach(to_mdev(qp->device), to_mqp(qp), + flow_spec, priority, 1); +} + +static int mlx4_ib_flow_detach(struct ib_qp *qp, struct ib_flow_spec *spec, + int priority) +{ + return __mlx4_ib_flow_detach(to_mdev(qp->device), to_mqp(qp), + spec, priority, 1); +} + +static struct mlx4_ib_gid_entry *find_gid_entry(struct mlx4_ib_qp *qp, u8 *raw) +{ + struct mlx4_ib_gid_entry *ge; + struct mlx4_ib_gid_entry *tmp; + struct mlx4_ib_gid_entry *ret = NULL; + list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) { if (!memcmp(raw, ge->gid.raw, 16)) { ret = ge; break; } } return ret; } static int mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) { int err; struct mlx4_ib_dev *mdev = to_mdev(ibqp->device); struct mlx4_ib_qp *mqp = to_mqp(ibqp); u8 mac[6]; struct net_device *ndev; - struct gid_entry *ge; + struct mlx4_ib_gid_entry *ge; + u64 reg_id = 0; + if (mdev->dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) { + struct mlx4_ib_steering *ib_steering; + + mutex_lock(&mqp->mutex); + list_for_each_entry(ib_steering, &mqp->steering_rules, list) { + if (!memcmp(ib_steering->gid.raw, gid->raw, 16)) { + list_del(&ib_steering->list); + break; + } + } + mutex_unlock(&mqp->mutex); + if (&ib_steering->list == &mqp->steering_rules) { + pr_err("Couldn't find reg_id for mgid. Steering rule is left attached\n"); + return -EINVAL; + } + reg_id = ib_steering->reg_id; + kfree(ib_steering); + } + err = mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw, - (ibqp->qp_type == IB_QPT_RAW_ETH) ? - MLX4_MCAST_PROT_EN : MLX4_MCAST_PROT_IB); + MLX4_PROT_IB_IPV6, reg_id); if (err) return err; mutex_lock(&mqp->mutex); ge = find_gid_entry(mqp, gid->raw); if (ge) { spin_lock(&mdev->iboe.lock); ndev = ge->added ? mdev->iboe.netdevs[ge->port - 1] : NULL; if (ndev) dev_hold(ndev); spin_unlock(&mdev->iboe.lock); rdma_get_mcast_mac((struct in6_addr *)gid, mac); if (ndev) { rtnl_lock(); dev_mc_delete(mdev->iboe.netdevs[ge->port - 1], mac, 6, 0); rtnl_unlock(); dev_put(ndev); } list_del(&ge->list); kfree(ge); } else - printk(KERN_WARNING "could not find mgid entry\n"); + pr_warn("could not find mgid entry\n"); mutex_unlock(&mqp->mutex); return 0; } -static void mlx4_dummy_comp_handler(struct ib_cq *cq, void *cq_context) -{ -} - -static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev, - struct ib_ucontext *context, - struct ib_udata *udata) -{ - struct mlx4_ib_xrcd *xrcd; - struct mlx4_ib_dev *mdev = to_mdev(ibdev); - struct ib_pd *pd; - struct ib_cq *cq; - int err; - - if (!(mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) - return ERR_PTR(-ENOSYS); - - xrcd = kmalloc(sizeof *xrcd, GFP_KERNEL); - if (!xrcd) - return ERR_PTR(-ENOMEM); - - err = mlx4_xrcd_alloc(mdev->dev, &xrcd->xrcdn); - if (err) - goto err_xrcd; - - pd = mlx4_ib_alloc_pd(ibdev, NULL, NULL); - if (IS_ERR(pd)) { - err = PTR_ERR(pd); - goto err_pd; - } - pd->device = ibdev; - - cq = mlx4_ib_create_cq(ibdev, 1, 0, NULL, NULL); - if (IS_ERR(cq)) { - err = PTR_ERR(cq); - goto err_cq; - } - cq->device = ibdev; - cq->comp_handler = mlx4_dummy_comp_handler; - - if (context) - if (ib_copy_to_udata(udata, &xrcd->xrcdn, sizeof(__u32))) { - err = -EFAULT; - goto err_copy; - } - - xrcd->cq = cq; - xrcd->pd = pd; - return &xrcd->ibxrcd; - -err_copy: - mlx4_ib_destroy_cq(cq); -err_cq: - mlx4_ib_dealloc_pd(pd); -err_pd: - mlx4_xrcd_free(mdev->dev, xrcd->xrcdn); -err_xrcd: - kfree(xrcd); - return ERR_PTR(err); -} - -static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd) -{ - struct mlx4_ib_xrcd *mxrcd = to_mxrcd(xrcd); - - mlx4_ib_destroy_cq(mxrcd->cq); - mlx4_ib_dealloc_pd(mxrcd->pd); - mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn); - kfree(xrcd); - - return 0; -} - - static int init_node_data(struct mlx4_ib_dev *dev) { struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; + int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS; int err = -ENOMEM; in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); if (!in_mad || !out_mad) goto out; init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_NODE_DESC; + if (mlx4_is_master(dev->dev)) + mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW; - err = mlx4_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad); + err = mlx4_MAD_IFC(dev, mad_ifc_flags, 1, NULL, NULL, in_mad, out_mad); if (err) goto out; memcpy(dev->ib_dev.node_desc, out_mad->data, 64); in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; - err = mlx4_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad); + err = mlx4_MAD_IFC(dev, mad_ifc_flags, 1, NULL, NULL, in_mad, out_mad); if (err) goto out; dev->dev->rev_id = be32_to_cpup((__be32 *) (out_mad->data + 32)); memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8); out: kfree(in_mad); kfree(out_mad); return err; } static ssize_t show_hca(struct device *device, struct device_attribute *attr, char *buf) { struct mlx4_ib_dev *dev = container_of(device, struct mlx4_ib_dev, ib_dev.dev); return sprintf(buf, "MT%d\n", dev->dev->pdev->device); } static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, char *buf) { struct mlx4_ib_dev *dev = container_of(device, struct mlx4_ib_dev, ib_dev.dev); return sprintf(buf, "%d.%d.%d\n", (int) (dev->dev->caps.fw_ver >> 32), (int) (dev->dev->caps.fw_ver >> 16) & 0xffff, (int) dev->dev->caps.fw_ver & 0xffff); } static ssize_t show_rev(struct device *device, struct device_attribute *attr, char *buf) { struct mlx4_ib_dev *dev = container_of(device, struct mlx4_ib_dev, ib_dev.dev); return sprintf(buf, "%x\n", dev->dev->rev_id); } static ssize_t show_board(struct device *device, struct device_attribute *attr, char *buf) { struct mlx4_ib_dev *dev = container_of(device, struct mlx4_ib_dev, ib_dev.dev); return sprintf(buf, "%.*s\n", MLX4_BOARD_ID_LEN, dev->dev->board_id); } static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL); static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL); static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL); static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL); static struct device_attribute *mlx4_class_attributes[] = { &dev_attr_hw_rev, &dev_attr_fw_ver, &dev_attr_hca_type, &dev_attr_board_id }; -/* - * create show function and a device_attribute struct pointing to - * the function for _name - */ -#define DEVICE_DIAG_RPRT_ATTR(_name, _offset, _op_mod) \ -static ssize_t show_rprt_##_name(struct device *dev, \ - struct device_attribute *attr, \ - char *buf){ \ - return show_diag_rprt(dev, buf, _offset, _op_mod); \ -} \ -static DEVICE_ATTR(_name, S_IRUGO, show_rprt_##_name, NULL); - -#define MLX4_DIAG_RPRT_CLEAR_DIAGS 3 - -static size_t show_diag_rprt(struct device *device, char *buf, - u32 offset, u8 op_modifier) -{ - size_t ret; - u32 counter_offset = offset; - u32 diag_counter = 0; - struct mlx4_ib_dev *dev = container_of(device, struct mlx4_ib_dev, - ib_dev.dev); - - ret = mlx4_query_diag_counters(dev->dev, 1, op_modifier, - &counter_offset, &diag_counter); - if (ret) - return ret; - - return sprintf(buf,"%d\n", diag_counter); -} - -static ssize_t clear_diag_counters(struct device *device, - struct device_attribute *attr, - const char *buf, size_t length) -{ - size_t ret; - struct mlx4_ib_dev *dev = container_of(device, struct mlx4_ib_dev, - ib_dev.dev); - - ret = mlx4_query_diag_counters(dev->dev, 0, MLX4_DIAG_RPRT_CLEAR_DIAGS, - NULL, NULL); - if (ret) - return ret; - - return length; -} - -DEVICE_DIAG_RPRT_ATTR(rq_num_lle , 0x00, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_lle , 0x04, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_lqpoe , 0x08, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_lqpoe , 0x0C, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_leeoe , 0x10, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_leeoe , 0x14, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_lpe , 0x18, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_lpe , 0x1C, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_wrfe , 0x20, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_wrfe , 0x24, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_mwbe , 0x2C, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_bre , 0x34, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_lae , 0x38, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_rire , 0x44, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_rire , 0x48, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_rae , 0x4C, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_rae , 0x50, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_roe , 0x54, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_tree , 0x5C, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_rree , 0x64, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_rnr , 0x68, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_rnr , 0x6C, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_rabrte , 0x7C, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_ieecne , 0x84, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_ieecse , 0x8C, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_oos , 0x100, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_oos , 0x104, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_mce , 0x108, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_rsync , 0x110, 2); -DEVICE_DIAG_RPRT_ATTR(sq_num_rsync , 0x114, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_udsdprd , 0x118, 2); -DEVICE_DIAG_RPRT_ATTR(rq_num_ucsdprd , 0x120, 2); -DEVICE_DIAG_RPRT_ATTR(num_cqovf , 0x1A0, 2); -DEVICE_DIAG_RPRT_ATTR(num_eqovf , 0x1A4, 2); -DEVICE_DIAG_RPRT_ATTR(num_baddb , 0x1A8, 2); - -static DEVICE_ATTR(clear_diag, S_IWUGO, NULL, clear_diag_counters); - -static struct attribute *diag_rprt_attrs[] = { - &dev_attr_rq_num_lle.attr, - &dev_attr_sq_num_lle.attr, - &dev_attr_rq_num_lqpoe.attr, - &dev_attr_sq_num_lqpoe.attr, - &dev_attr_rq_num_leeoe.attr, - &dev_attr_sq_num_leeoe.attr, - &dev_attr_rq_num_lpe.attr, - &dev_attr_sq_num_lpe.attr, - &dev_attr_rq_num_wrfe.attr, - &dev_attr_sq_num_wrfe.attr, - &dev_attr_sq_num_mwbe.attr, - &dev_attr_sq_num_bre.attr, - &dev_attr_rq_num_lae.attr, - &dev_attr_sq_num_rire.attr, - &dev_attr_rq_num_rire.attr, - &dev_attr_sq_num_rae.attr, - &dev_attr_rq_num_rae.attr, - &dev_attr_sq_num_roe.attr, - &dev_attr_sq_num_tree.attr, - &dev_attr_sq_num_rree.attr, - &dev_attr_rq_num_rnr.attr, - &dev_attr_sq_num_rnr.attr, - &dev_attr_sq_num_rabrte.attr, - &dev_attr_sq_num_ieecne.attr, - &dev_attr_sq_num_ieecse.attr, - &dev_attr_rq_num_oos.attr, - &dev_attr_sq_num_oos.attr, - &dev_attr_rq_num_mce.attr, - &dev_attr_rq_num_rsync.attr, - &dev_attr_sq_num_rsync.attr, - &dev_attr_rq_num_udsdprd.attr, - &dev_attr_rq_num_ucsdprd.attr, - &dev_attr_num_cqovf.attr, - &dev_attr_num_eqovf.attr, - &dev_attr_num_baddb.attr, - &dev_attr_clear_diag.attr, - NULL -}; - -struct attribute_group diag_counters_group = { - .name = "diag_counters", - .attrs = diag_rprt_attrs -}; - static void mlx4_addrconf_ifid_eui48(u8 *eui, u16 vlan_id, struct net_device *dev) { #ifdef __linux__ memcpy(eui, dev->dev_addr, 3); memcpy(eui + 5, dev->dev_addr + 3, 3); #else - memcpy(eui, IF_LLADDR(dev), 3); - memcpy(eui + 5, IF_LLADDR(dev) + 3, 3); + memcpy(eui, IF_LLADDR(dev), 3); + memcpy(eui + 5, IF_LLADDR(dev) + 3, 3); #endif if (vlan_id < 0x1000) { eui[3] = vlan_id >> 8; eui[4] = vlan_id & 0xff; } else { eui[3] = 0xff; eui[4] = 0xfe; } eui[0] ^= 2; } static void update_gids_task(struct work_struct *work) { struct update_gid_work *gw = container_of(work, struct update_gid_work, work); struct mlx4_cmd_mailbox *mailbox; union ib_gid *gids; int err; struct mlx4_dev *dev = gw->dev->dev; - struct ib_event event; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) { - printk(KERN_WARNING "update gid table failed %ld\n", PTR_ERR(mailbox)); + pr_warn("update gid table failed %ld\n", PTR_ERR(mailbox)); return; } gids = mailbox->buf; memcpy(gids, gw->gids, sizeof gw->gids); err = mlx4_cmd(dev, mailbox->dma, MLX4_SET_PORT_GID_TABLE << 8 | gw->port, - 1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B); + 1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); if (err) - printk(KERN_WARNING "set port command failed\n"); + pr_warn("set port command failed\n"); else { memcpy(gw->dev->iboe.gid_table[gw->port - 1], gw->gids, sizeof gw->gids); - event.device = &gw->dev->ib_dev; - event.element.port_num = gw->port; - event.event = IB_EVENT_GID_CHANGE; - ib_dispatch_event(&event); + mlx4_ib_dispatch_event(gw->dev, gw->port, IB_EVENT_GID_CHANGE); } mlx4_free_cmd_mailbox(dev, mailbox); kfree(gw); } -enum { - MLX4_MAX_EFF_VLANS = 128 - MLX4_VLAN_REGULAR, -}; - static int update_ipv6_gids(struct mlx4_ib_dev *dev, int port, int clear) { struct net_device *ndev = dev->iboe.netdevs[port - 1]; struct update_gid_work *work; struct net_device *tmp; int i; u8 *hits; - int ret; union ib_gid gid; - int tofree; + int index_free; int found; int need_update = 0; + int max_gids; u16 vid; work = kzalloc(sizeof *work, GFP_ATOMIC); if (!work) return -ENOMEM; - hits = kzalloc(MLX4_MAX_EFF_VLANS + 1, GFP_ATOMIC); + hits = kzalloc(128, GFP_ATOMIC); if (!hits) { - ret = -ENOMEM; - goto out; + kfree(work); + return -ENOMEM; } + max_gids = dev->dev->caps.gid_table_len[port]; + #ifdef __linux__ - read_lock(&dev_base_lock); - for_each_netdev(&init_net, tmp) { + rcu_read_lock(); + for_each_netdev_rcu(&init_net, tmp) { #else - IFNET_RLOCK(); - TAILQ_FOREACH(tmp, &V_ifnet, if_link) { + IFNET_RLOCK(); + TAILQ_FOREACH(tmp, &V_ifnet, if_link) { #endif if (ndev && (tmp == ndev || rdma_vlan_dev_real_dev(tmp) == ndev)) { gid.global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL); vid = rdma_vlan_dev_vlan_id(tmp); mlx4_addrconf_ifid_eui48(&gid.raw[8], vid, ndev); found = 0; - tofree = -1; - for (i = 0; i < MLX4_MAX_EFF_VLANS + 1; ++i) { - if (tofree < 0 && + index_free = -1; + for (i = 0; i < max_gids; ++i) { + if (index_free < 0 && !memcmp(&dev->iboe.gid_table[port - 1][i], &zgid, sizeof zgid)) - tofree = i; + index_free = i; if (!memcmp(&dev->iboe.gid_table[port - 1][i], &gid, sizeof gid)) { hits[i] = 1; found = 1; break; } } if (!found) { - if (tmp == ndev && (memcmp(&dev->iboe.gid_table[port - 1][0], &gid, sizeof gid) || !memcmp(&dev->iboe.gid_table[port - 1][0], &zgid, sizeof gid))) { + if (tmp == ndev && + (memcmp(&dev->iboe.gid_table[port - 1][0], + &gid, sizeof gid) || + !memcmp(&dev->iboe.gid_table[port - 1][0], + &zgid, sizeof gid))) { dev->iboe.gid_table[port - 1][0] = gid; ++need_update; hits[0] = 1; - } else if (tofree >= 0) { - dev->iboe.gid_table[port - 1][tofree] = gid; - hits[tofree] = 1; + } else if (index_free >= 0) { + dev->iboe.gid_table[port - 1][index_free] = gid; + hits[index_free] = 1; ++need_update; } } } -#ifdef __linux__ - } - read_unlock(&dev_base_lock); +#ifdef __linux__ + } + rcu_read_unlock(); #else - } - IFNET_RUNLOCK(); + } + IFNET_RUNLOCK(); #endif - for (i = 0; i < MLX4_MAX_EFF_VLANS + 1; ++i) + for (i = 0; i < max_gids; ++i) if (!hits[i]) { if (memcmp(&dev->iboe.gid_table[port - 1][i], &zgid, sizeof zgid)) ++need_update; dev->iboe.gid_table[port - 1][i] = zgid; } - if (need_update) { memcpy(work->gids, dev->iboe.gid_table[port - 1], sizeof work->gids); INIT_WORK(&work->work, update_gids_task); work->port = port; work->dev = dev; queue_work(wq, &work->work); } else kfree(work); kfree(hits); return 0; - -out: - kfree(work); - return ret; } static void handle_en_event(struct mlx4_ib_dev *dev, int port, unsigned long event) { switch (event) { case NETDEV_UP: #ifdef __linux__ case NETDEV_CHANGEADDR: #endif update_ipv6_gids(dev, port, 0); break; case NETDEV_DOWN: update_ipv6_gids(dev, port, 1); dev->iboe.netdevs[port - 1] = NULL; } } static void netdev_added(struct mlx4_ib_dev *dev, int port) { update_ipv6_gids(dev, port, 0); } static void netdev_removed(struct mlx4_ib_dev *dev, int port) { update_ipv6_gids(dev, port, 1); } static int mlx4_ib_netdev_event(struct notifier_block *this, unsigned long event, void *ptr) { struct net_device *dev = ptr; struct mlx4_ib_dev *ibdev; struct net_device *oldnd; struct mlx4_ib_iboe *iboe; int port; #ifdef __linux__ if (!net_eq(dev_net(dev), &init_net)) return NOTIFY_DONE; #endif ibdev = container_of(this, struct mlx4_ib_dev, iboe.nb); iboe = &ibdev->iboe; spin_lock(&iboe->lock); mlx4_foreach_ib_transport_port(port, ibdev->dev) { oldnd = iboe->netdevs[port - 1]; - iboe->netdevs[port - 1] = mlx4_get_prot_dev(ibdev->dev, MLX4_PROT_EN, port); + iboe->netdevs[port - 1] = + mlx4_get_protocol_dev(ibdev->dev, MLX4_PROT_ETH, port); if (oldnd != iboe->netdevs[port - 1]) { if (iboe->netdevs[port - 1]) netdev_added(ibdev, port); else netdev_removed(ibdev, port); } } if (dev == iboe->netdevs[0] || (iboe->netdevs[0] && rdma_vlan_dev_real_dev(dev) == iboe->netdevs[0])) handle_en_event(ibdev, 1, event); else if (dev == iboe->netdevs[1] || (iboe->netdevs[1] && rdma_vlan_dev_real_dev(dev) == iboe->netdevs[1])) handle_en_event(ibdev, 2, event); spin_unlock(&iboe->lock); return NOTIFY_DONE; } +static void init_pkeys(struct mlx4_ib_dev *ibdev) +{ + int port; + int slave; + int i; + + if (mlx4_is_master(ibdev->dev)) { + for (slave = 0; slave <= ibdev->dev->num_vfs; ++slave) { + for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) { + for (i = 0; + i < ibdev->dev->phys_caps.pkey_phys_table_len[port]; + ++i) { + ibdev->pkeys.virt2phys_pkey[slave][port - 1][i] = + /* master has the identity virt2phys pkey mapping */ + (slave == mlx4_master_func_num(ibdev->dev) || !i) ? i : + ibdev->dev->phys_caps.pkey_phys_table_len[port] - 1; + mlx4_sync_pkey_table(ibdev->dev, slave, port, i, + ibdev->pkeys.virt2phys_pkey[slave][port - 1][i]); + } + } + } + /* initialize pkey cache */ + for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) { + for (i = 0; + i < ibdev->dev->phys_caps.pkey_phys_table_len[port]; + ++i) + ibdev->pkeys.phys_pkey_cache[port-1][i] = + (i) ? 0 : 0xFFFF; + } + } +} + +static void mlx4_ib_alloc_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev) +{ + char name[32]; + int eq_per_port = 0; + int added_eqs = 0; + int total_eqs = 0; + int i, j, eq; + + /* Legacy mode or comp_pool is not large enough */ + if (dev->caps.comp_pool == 0 || + dev->caps.num_ports > dev->caps.comp_pool) + return; + + eq_per_port = rounddown_pow_of_two(dev->caps.comp_pool/ + dev->caps.num_ports); + + /* Init eq table */ + added_eqs = 0; + mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB) + added_eqs += eq_per_port; + + total_eqs = dev->caps.num_comp_vectors + added_eqs; + + ibdev->eq_table = kzalloc(total_eqs * sizeof(int), GFP_KERNEL); + if (!ibdev->eq_table) + return; + + ibdev->eq_added = added_eqs; + + eq = 0; + mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB) { + for (j = 0; j < eq_per_port; j++) { + //sprintf(name, "mlx4-ib-%d-%d@%s", + // i, j, dev->pdev->bus->conf.pd_name); + /* Set IRQ for specific name (per ring) */ + if (mlx4_assign_eq(dev, name, + &ibdev->eq_table[eq])) { + /* Use legacy (same as mlx4_en driver) */ + pr_warn("Can't allocate EQ %d; reverting to legacy\n", eq); + ibdev->eq_table[eq] = + (eq % dev->caps.num_comp_vectors); + } + eq++; + } + } + + /* Fill the reset of the vector with legacy EQ */ + for (i = 0, eq = added_eqs; i < dev->caps.num_comp_vectors; i++) + ibdev->eq_table[eq++] = i; + + /* Advertise the new number of EQs to clients */ + ibdev->ib_dev.num_comp_vectors = total_eqs; +} + +static void mlx4_ib_free_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev) +{ + int i; + + /* no additional eqs were added */ + if (!ibdev->eq_table) + return; + + /* Reset the advertised EQ number */ + ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors; + + /* Free only the added eqs */ + for (i = 0; i < ibdev->eq_added; i++) { + /* Don't free legacy eqs if used */ + if (ibdev->eq_table[i] <= dev->caps.num_comp_vectors) + continue; + mlx4_release_eq(dev, ibdev->eq_table[i]); + } + + kfree(ibdev->eq_table); +} + +/* + * create show function and a device_attribute struct pointing to + * the function for _name + */ +#define DEVICE_DIAG_RPRT_ATTR(_name, _offset, _op_mod) \ +static ssize_t show_rprt_##_name(struct device *dev, \ + struct device_attribute *attr, \ + char *buf){ \ + return show_diag_rprt(dev, buf, _offset, _op_mod); \ +} \ +static DEVICE_ATTR(_name, S_IRUGO, show_rprt_##_name, NULL); + +#define MLX4_DIAG_RPRT_CLEAR_DIAGS 3 + +static size_t show_diag_rprt(struct device *device, char *buf, + u32 offset, u8 op_modifier) +{ + size_t ret; + u32 counter_offset = offset; + u32 diag_counter = 0; + struct mlx4_ib_dev *dev = container_of(device, struct mlx4_ib_dev, + ib_dev.dev); + + ret = mlx4_query_diag_counters(dev->dev, 1, op_modifier, + &counter_offset, &diag_counter); + if (ret) + return ret; + + return sprintf(buf, "%d\n", diag_counter); +} + +static ssize_t clear_diag_counters(struct device *device, + struct device_attribute *attr, + const char *buf, size_t length) +{ + size_t ret; + struct mlx4_ib_dev *dev = container_of(device, struct mlx4_ib_dev, + ib_dev.dev); + + ret = mlx4_query_diag_counters(dev->dev, 0, MLX4_DIAG_RPRT_CLEAR_DIAGS, + NULL, NULL); + if (ret) + return ret; + + return length; +} + +DEVICE_DIAG_RPRT_ATTR(rq_num_lle , 0x00, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_lle , 0x04, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_lqpoe , 0x08, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_lqpoe , 0x0C, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_lpe , 0x18, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_lpe , 0x1C, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_wrfe , 0x20, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_wrfe , 0x24, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_mwbe , 0x2C, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_bre , 0x34, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_lae , 0x38, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_rire , 0x44, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_rire , 0x48, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_rae , 0x4C, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_rae , 0x50, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_roe , 0x54, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_tree , 0x5C, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_rree , 0x64, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_rnr , 0x68, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_rnr , 0x6C, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_oos , 0x100, 2); +DEVICE_DIAG_RPRT_ATTR(sq_num_oos , 0x104, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_mce , 0x108, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_udsdprd , 0x118, 2); +DEVICE_DIAG_RPRT_ATTR(rq_num_ucsdprd , 0x120, 2); +DEVICE_DIAG_RPRT_ATTR(num_cqovf , 0x1A0, 2); +DEVICE_DIAG_RPRT_ATTR(num_eqovf , 0x1A4, 2); +DEVICE_DIAG_RPRT_ATTR(num_baddb , 0x1A8, 2); + +static DEVICE_ATTR(clear_diag, S_IWUSR, NULL, clear_diag_counters); + +static struct attribute *diag_rprt_attrs[] = { + &dev_attr_rq_num_lle.attr, + &dev_attr_sq_num_lle.attr, + &dev_attr_rq_num_lqpoe.attr, + &dev_attr_sq_num_lqpoe.attr, + &dev_attr_rq_num_lpe.attr, + &dev_attr_sq_num_lpe.attr, + &dev_attr_rq_num_wrfe.attr, + &dev_attr_sq_num_wrfe.attr, + &dev_attr_sq_num_mwbe.attr, + &dev_attr_sq_num_bre.attr, + &dev_attr_rq_num_lae.attr, + &dev_attr_sq_num_rire.attr, + &dev_attr_rq_num_rire.attr, + &dev_attr_sq_num_rae.attr, + &dev_attr_rq_num_rae.attr, + &dev_attr_sq_num_roe.attr, + &dev_attr_sq_num_tree.attr, + &dev_attr_sq_num_rree.attr, + &dev_attr_rq_num_rnr.attr, + &dev_attr_sq_num_rnr.attr, + &dev_attr_rq_num_oos.attr, + &dev_attr_sq_num_oos.attr, + &dev_attr_rq_num_mce.attr, + &dev_attr_rq_num_udsdprd.attr, + &dev_attr_rq_num_ucsdprd.attr, + &dev_attr_num_cqovf.attr, + &dev_attr_num_eqovf.attr, + &dev_attr_num_baddb.attr, + &dev_attr_clear_diag.attr, + NULL +}; + +static struct attribute_group diag_counters_group = { + .name = "diag_counters", + .attrs = diag_rprt_attrs +}; + +#ifdef __linux__ +static int mlx4_ib_proc_init(void) +{ + /* Creating procfs directories /proc/drivers/mlx4_ib/ && + /proc/drivers/mlx4_ib/mrs for further use by the driver. + */ + int err; + + mlx4_ib_driver_dir_entry = proc_mkdir(MLX4_IB_DRIVER_PROC_DIR_NAME, + NULL); + if (!mlx4_ib_driver_dir_entry) { + pr_err("mlx4_ib_proc_init has failed for %s\n", + MLX4_IB_DRIVER_PROC_DIR_NAME); + err = -ENODEV; + goto error; + } + + mlx4_mrs_dir_entry = proc_mkdir(MLX4_IB_MRS_PROC_DIR_NAME, + mlx4_ib_driver_dir_entry); + if (!mlx4_mrs_dir_entry) { + pr_err("mlx4_ib_proc_init has failed for %s\n", + MLX4_IB_MRS_PROC_DIR_NAME); + err = -ENODEV; + goto remove_entry; + } + + return 0; + +remove_entry: + remove_proc_entry(MLX4_IB_DRIVER_PROC_DIR_NAME, + NULL); +error: + return err; +} +#endif + +static void init_dev_assign(void) +{ + int bus, slot, fn, ib_idx; + char *p = dev_assign_str, *t; + char curr_val[32] = {0}; + int ret; + int j, i = 0; + + memset(dr, 0, sizeof dr); + + if (dev_assign_str[0] == 0) + return; + + while (strlen(p)) { + ret = sscanf(p, "%02x:%02x.%x-%x", &bus, &slot, &fn, &ib_idx); + if (ret != 4 || ib_idx < 0) + goto err; + + for (j = 0; j < i; j++) + if (dr[j].nr == ib_idx) + goto err; + + dr[i].bus = bus; + dr[i].dev = slot; + dr[i].func = fn; + dr[i].nr = ib_idx; + + t = strchr(p, ','); + sprintf(curr_val, "%02x:%02x.%x-%x", bus, slot, fn, ib_idx); + if ((!t) && strlen(p) == strlen(curr_val)) + return; + + if (!t || (t + 1) >= dev_assign_str + sizeof dev_assign_str) + goto err; + + ++i; + if (i >= MAX_DR) + goto err; + + p = t + 1; + } + + return; +err: + memset(dr, 0, sizeof dr); + printk(KERN_WARNING "mlx4_ib: The value of 'dev_assign_str' parameter " + "is incorrect. The parameter value is discarded!"); +} + static void *mlx4_ib_add(struct mlx4_dev *dev) { - static int mlx4_ib_version_printed; struct mlx4_ib_dev *ibdev; int num_ports = 0; - int i; + int i, j; int err; struct mlx4_ib_iboe *iboe; - int k; - if (!mlx4_ib_version_printed) { - printk(KERN_INFO "%s", mlx4_ib_version); - ++mlx4_ib_version_printed; - } + printk(KERN_INFO "%s", mlx4_ib_version); mlx4_foreach_ib_transport_port(i, dev) num_ports++; /* No point in registering a device with no ports... */ if (num_ports == 0) return NULL; ibdev = (struct mlx4_ib_dev *) ib_alloc_device(sizeof *ibdev); if (!ibdev) { dev_err(&dev->pdev->dev, "Device struct alloc failed\n"); return NULL; } iboe = &ibdev->iboe; if (mlx4_pd_alloc(dev, &ibdev->priv_pdn)) goto err_dealloc; if (mlx4_uar_alloc(dev, &ibdev->priv_uar)) goto err_pd; - ibdev->priv_uar.map = ioremap(ibdev->priv_uar.pfn << PAGE_SHIFT, PAGE_SIZE); + ibdev->priv_uar.map = ioremap(ibdev->priv_uar.pfn << PAGE_SHIFT, + PAGE_SIZE); + if (!ibdev->priv_uar.map) goto err_uar; + MLX4_INIT_DOORBELL_LOCK(&ibdev->uar_lock); ibdev->dev = dev; strlcpy(ibdev->ib_dev.name, "mlx4_%d", IB_DEVICE_NAME_MAX); ibdev->ib_dev.owner = THIS_MODULE; ibdev->ib_dev.node_type = RDMA_NODE_IB_CA; ibdev->ib_dev.local_dma_lkey = dev->caps.reserved_lkey; ibdev->num_ports = num_ports; ibdev->ib_dev.phys_port_cnt = ibdev->num_ports; ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors; ibdev->ib_dev.dma_device = &dev->pdev->dev; - ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION; + if (dev->caps.userspace_caps) + ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION; + else + ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION; + ibdev->ib_dev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | (1ull << IB_USER_VERBS_CMD_REG_MR) | (1ull << IB_USER_VERBS_CMD_DEREG_MR) | (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | (1ull << IB_USER_VERBS_CMD_CREATE_QP) | (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | (1ull << IB_USER_VERBS_CMD_QUERY_QP) | (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) | (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) | (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | + (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | + (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ) | + (1ull << IB_USER_VERBS_CMD_OPEN_QP) | + (1ull << IB_USER_VERBS_CMD_ATTACH_FLOW) | + (1ull << IB_USER_VERBS_CMD_DETACH_FLOW) | (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ); ibdev->ib_dev.query_device = mlx4_ib_query_device; ibdev->ib_dev.query_port = mlx4_ib_query_port; ibdev->ib_dev.get_link_layer = mlx4_ib_port_link_layer; ibdev->ib_dev.query_gid = mlx4_ib_query_gid; ibdev->ib_dev.query_pkey = mlx4_ib_query_pkey; ibdev->ib_dev.modify_device = mlx4_ib_modify_device; ibdev->ib_dev.modify_port = mlx4_ib_modify_port; ibdev->ib_dev.alloc_ucontext = mlx4_ib_alloc_ucontext; ibdev->ib_dev.dealloc_ucontext = mlx4_ib_dealloc_ucontext; ibdev->ib_dev.mmap = mlx4_ib_mmap; +#ifdef __linux__ + ibdev->ib_dev.get_unmapped_area = mlx4_ib_get_unmapped_area; +#endif ibdev->ib_dev.alloc_pd = mlx4_ib_alloc_pd; ibdev->ib_dev.dealloc_pd = mlx4_ib_dealloc_pd; ibdev->ib_dev.create_ah = mlx4_ib_create_ah; ibdev->ib_dev.query_ah = mlx4_ib_query_ah; ibdev->ib_dev.destroy_ah = mlx4_ib_destroy_ah; ibdev->ib_dev.create_srq = mlx4_ib_create_srq; ibdev->ib_dev.modify_srq = mlx4_ib_modify_srq; ibdev->ib_dev.query_srq = mlx4_ib_query_srq; ibdev->ib_dev.destroy_srq = mlx4_ib_destroy_srq; ibdev->ib_dev.post_srq_recv = mlx4_ib_post_srq_recv; ibdev->ib_dev.create_qp = mlx4_ib_create_qp; ibdev->ib_dev.modify_qp = mlx4_ib_modify_qp; ibdev->ib_dev.query_qp = mlx4_ib_query_qp; ibdev->ib_dev.destroy_qp = mlx4_ib_destroy_qp; ibdev->ib_dev.post_send = mlx4_ib_post_send; ibdev->ib_dev.post_recv = mlx4_ib_post_recv; ibdev->ib_dev.create_cq = mlx4_ib_create_cq; ibdev->ib_dev.modify_cq = mlx4_ib_modify_cq; ibdev->ib_dev.resize_cq = mlx4_ib_resize_cq; ibdev->ib_dev.destroy_cq = mlx4_ib_destroy_cq; ibdev->ib_dev.poll_cq = mlx4_ib_poll_cq; ibdev->ib_dev.req_notify_cq = mlx4_ib_arm_cq; ibdev->ib_dev.get_dma_mr = mlx4_ib_get_dma_mr; ibdev->ib_dev.reg_user_mr = mlx4_ib_reg_user_mr; ibdev->ib_dev.dereg_mr = mlx4_ib_dereg_mr; ibdev->ib_dev.alloc_fast_reg_mr = mlx4_ib_alloc_fast_reg_mr; ibdev->ib_dev.alloc_fast_reg_page_list = mlx4_ib_alloc_fast_reg_page_list; ibdev->ib_dev.free_fast_reg_page_list = mlx4_ib_free_fast_reg_page_list; ibdev->ib_dev.attach_mcast = mlx4_ib_mcg_attach; ibdev->ib_dev.detach_mcast = mlx4_ib_mcg_detach; + ibdev->ib_dev.attach_flow = mlx4_ib_flow_attach; + ibdev->ib_dev.detach_flow = mlx4_ib_flow_detach; ibdev->ib_dev.process_mad = mlx4_ib_process_mad; - ibdev->ib_dev.alloc_fmr = mlx4_ib_fmr_alloc; - ibdev->ib_dev.map_phys_fmr = mlx4_ib_map_phys_fmr; - ibdev->ib_dev.unmap_fmr = mlx4_ib_unmap_fmr; - ibdev->ib_dev.dealloc_fmr = mlx4_ib_fmr_dealloc; + if (!mlx4_is_slave(ibdev->dev)) { + ibdev->ib_dev.alloc_fmr = mlx4_ib_fmr_alloc; + ibdev->ib_dev.map_phys_fmr = mlx4_ib_map_phys_fmr; + ibdev->ib_dev.unmap_fmr = mlx4_ib_unmap_fmr; + ibdev->ib_dev.dealloc_fmr = mlx4_ib_fmr_dealloc; + } + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) { - ibdev->ib_dev.create_xrc_srq = mlx4_ib_create_xrc_srq; ibdev->ib_dev.alloc_xrcd = mlx4_ib_alloc_xrcd; ibdev->ib_dev.dealloc_xrcd = mlx4_ib_dealloc_xrcd; - ibdev->ib_dev.create_xrc_rcv_qp = mlx4_ib_create_xrc_rcv_qp; - ibdev->ib_dev.modify_xrc_rcv_qp = mlx4_ib_modify_xrc_rcv_qp; - ibdev->ib_dev.query_xrc_rcv_qp = mlx4_ib_query_xrc_rcv_qp; - ibdev->ib_dev.reg_xrc_rcv_qp = mlx4_ib_reg_xrc_rcv_qp; - ibdev->ib_dev.unreg_xrc_rcv_qp = mlx4_ib_unreg_xrc_rcv_qp; ibdev->ib_dev.uverbs_cmd_mask |= - (1ull << IB_USER_VERBS_CMD_CREATE_XRC_SRQ) | - (1ull << IB_USER_VERBS_CMD_OPEN_XRC_DOMAIN) | - (1ull << IB_USER_VERBS_CMD_CLOSE_XRC_DOMAIN) | - (1ull << IB_USER_VERBS_CMD_CREATE_XRC_RCV_QP) | - (1ull << IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP) | - (1ull << IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP) | - (1ull << IB_USER_VERBS_CMD_REG_XRC_RCV_QP) | - (1ull << IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP); + (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) | + (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD); } + mlx4_ib_alloc_eqs(dev, ibdev); spin_lock_init(&iboe->lock); + if (init_node_data(ibdev)) goto err_map; - for (k = 0; k < ibdev->num_ports; ++k) { - err = mlx4_counter_alloc(ibdev->dev, &ibdev->counters[k]); - if (err) - ibdev->counters[k] = -1; - else - mlx4_set_iboe_counter(dev, ibdev->counters[k], k + 1); + for (i = 0; i < ibdev->num_ports; ++i) { + if (mlx4_ib_port_link_layer(&ibdev->ib_dev, i + 1) == + IB_LINK_LAYER_ETHERNET) { + err = mlx4_counter_alloc(ibdev->dev, i + 1, &ibdev->counters[i]); + if (err) + ibdev->counters[i] = -1; + } else + ibdev->counters[i] = -1; } spin_lock_init(&ibdev->sm_lock); mutex_init(&ibdev->cap_mask_mutex); - mutex_init(&ibdev->xrc_reg_mutex); - if (ib_register_device(&ibdev->ib_dev)) - goto err_counter; + if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED && + !mlx4_is_slave(dev)) { + ibdev->steer_qpn_count = MLX4_IB_UC_MAX_NUM_QPS; + err = mlx4_qp_reserve_range(dev, ibdev->steer_qpn_count, + MLX4_IB_UC_STEER_QPN_ALIGN, &ibdev->steer_qpn_base, 0); + if (err) + goto err_counter; + ibdev->ib_uc_qpns_bitmap = + kmalloc(BITS_TO_LONGS(ibdev->steer_qpn_count) * + sizeof(long), + GFP_KERNEL); + if (!ibdev->ib_uc_qpns_bitmap) { + dev_err(&dev->pdev->dev, "bit map alloc failed\n"); + goto err_steer_qp_release; + } + + bitmap_zero(ibdev->ib_uc_qpns_bitmap, ibdev->steer_qpn_count); + + err = mlx4_FLOW_STEERING_IB_UC_QP_RANGE(dev, ibdev->steer_qpn_base, + ibdev->steer_qpn_base + ibdev->steer_qpn_count - 1); + if (err) + goto err_steer_free_bitmap; + } + + if (ib_register_device(&ibdev->ib_dev, NULL)) + goto err_steer_free_bitmap; + if (mlx4_ib_mad_init(ibdev)) goto err_reg; + + if (mlx4_ib_init_sriov(ibdev)) + goto err_mad; + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE && !iboe->nb.notifier_call) { iboe->nb.notifier_call = mlx4_ib_netdev_event; err = register_netdevice_notifier(&iboe->nb); if (err) - goto err_reg; + goto err_sriov; } - for (i = 0; i < ARRAY_SIZE(mlx4_class_attributes); ++i) { + + for (j = 0; j < ARRAY_SIZE(mlx4_class_attributes); ++j) { if (device_create_file(&ibdev->ib_dev.dev, - mlx4_class_attributes[i])) + mlx4_class_attributes[j])) goto err_notif; } - - if(sysfs_create_group(&ibdev->ib_dev.dev.kobj, &diag_counters_group)) + if (sysfs_create_group(&ibdev->ib_dev.dev.kobj, &diag_counters_group)) goto err_notif; - ibdev->ib_active = 1; + ibdev->ib_active = true; + if (mlx4_is_mfunc(ibdev->dev)) + init_pkeys(ibdev); + + /* create paravirt contexts for any VFs which are active */ + if (mlx4_is_master(ibdev->dev)) { + for (j = 0; j < MLX4_MFUNC_MAX; j++) { + if (j == mlx4_master_func_num(ibdev->dev)) + continue; + if (mlx4_is_slave_active(ibdev->dev, j)) + do_slave_init(ibdev, j, 1); + } + } return ibdev; err_notif: if (unregister_netdevice_notifier(&ibdev->iboe.nb)) - printk(KERN_WARNING "failure unregistering notifier\n"); + pr_warn("failure unregistering notifier\n"); flush_workqueue(wq); +err_sriov: + mlx4_ib_close_sriov(ibdev); + +err_mad: + mlx4_ib_mad_cleanup(ibdev); + err_reg: ib_unregister_device(&ibdev->ib_dev); +err_steer_free_bitmap: + kfree(ibdev->ib_uc_qpns_bitmap); + +err_steer_qp_release: + if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) + mlx4_qp_release_range(dev, ibdev->steer_qpn_base, + ibdev->steer_qpn_count); err_counter: - for (; k; --k) - mlx4_counter_free(ibdev->dev, ibdev->counters[k - 1]); + for (; i; --i) + if (ibdev->counters[i - 1] != -1) + mlx4_counter_free(ibdev->dev, i, ibdev->counters[i - 1]); err_map: iounmap(ibdev->priv_uar.map); + mlx4_ib_free_eqs(dev, ibdev); err_uar: mlx4_uar_free(dev, &ibdev->priv_uar); err_pd: mlx4_pd_free(dev, ibdev->priv_pdn); err_dealloc: ib_dealloc_device(&ibdev->ib_dev); return NULL; } +int mlx4_ib_steer_qp_alloc(struct mlx4_ib_dev *dev, int count, int *qpn) +{ + int offset; + + WARN_ON(!dev->ib_uc_qpns_bitmap); + + offset = bitmap_find_free_region(dev->ib_uc_qpns_bitmap, + dev->steer_qpn_count, + get_count_order(count)); + if (offset < 0) + return offset; + + *qpn = dev->steer_qpn_base + offset; + return 0; +} + +void mlx4_ib_steer_qp_free(struct mlx4_ib_dev *dev, u32 qpn, int count) +{ + if (!qpn || + dev->dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED) + return; + + BUG_ON(qpn < dev->steer_qpn_base); + + bitmap_release_region(dev->ib_uc_qpns_bitmap, + qpn - dev->steer_qpn_base, get_count_order(count)); +} + +int mlx4_ib_steer_qp_reg(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, + int is_attach) +{ + struct ib_flow_spec spec = { + .type = IB_FLOW_IB_UC, + .l2_id.ib_uc.qpn = mqp->ibqp.qp_num, + }; + + return is_attach ? + __mlx4_ib_flow_attach(mdev, mqp, &spec, MLX4_DOMAIN_NIC, 0) + : __mlx4_ib_flow_detach(mdev, mqp, &spec, MLX4_DOMAIN_NIC, 0); +} + static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr) { struct mlx4_ib_dev *ibdev = ibdev_ptr; - int p; - int k; + int p,j; + mlx4_ib_close_sriov(ibdev); sysfs_remove_group(&ibdev->ib_dev.dev.kobj, &diag_counters_group); - mlx4_ib_mad_cleanup(ibdev); + + for (j = 0; j < ARRAY_SIZE(mlx4_class_attributes); ++j) { + device_remove_file(&ibdev->ib_dev.dev, mlx4_class_attributes[j]); + } + ib_unregister_device(&ibdev->ib_dev); - for (k = 0; k < ibdev->num_ports; ++k) - mlx4_counter_free(ibdev->dev, ibdev->counters[k]); + if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) { + mlx4_qp_release_range(dev, ibdev->steer_qpn_base, + ibdev->steer_qpn_count); + kfree(ibdev->ib_uc_qpns_bitmap); + } + if (ibdev->iboe.nb.notifier_call) { - unregister_netdevice_notifier(&ibdev->iboe.nb); - flush_workqueue(wq); + if (unregister_netdevice_notifier(&ibdev->iboe.nb)) + pr_warn("failure unregistering notifier\n"); ibdev->iboe.nb.notifier_call = NULL; } iounmap(ibdev->priv_uar.map); - + for (p = 0; p < ibdev->num_ports; ++p) + if (ibdev->counters[p] != -1) + mlx4_counter_free(ibdev->dev, p + 1, ibdev->counters[p]); mlx4_foreach_port(p, dev, MLX4_PORT_TYPE_IB) mlx4_CLOSE_PORT(dev, p); + mlx4_ib_free_eqs(dev, ibdev); + mlx4_uar_free(dev, &ibdev->priv_uar); mlx4_pd_free(dev, ibdev->priv_pdn); ib_dealloc_device(&ibdev->ib_dev); } +static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init) +{ + struct mlx4_ib_demux_work **dm = NULL; + struct mlx4_dev *dev = ibdev->dev; + int i; + unsigned long flags; + + if (!mlx4_is_master(dev)) + return; + + dm = kcalloc(dev->caps.num_ports, sizeof *dm, GFP_ATOMIC); + if (!dm) { + pr_err("failed to allocate memory for tunneling qp update\n"); + goto out; + } + + for (i = 0; i < dev->caps.num_ports; i++) { + dm[i] = kmalloc(sizeof (struct mlx4_ib_demux_work), GFP_ATOMIC); + if (!dm[i]) { + pr_err("failed to allocate memory for tunneling qp update work struct\n"); + for (i = 0; i < dev->caps.num_ports; i++) { + if (dm[i]) + kfree(dm[i]); + } + goto out; + } + } + /* initialize or tear down tunnel QPs for the slave */ + for (i = 0; i < dev->caps.num_ports; i++) { + INIT_WORK(&dm[i]->work, mlx4_ib_tunnels_update_work); + dm[i]->port = i + 1; + dm[i]->slave = slave; + dm[i]->do_init = do_init; + dm[i]->dev = ibdev; + spin_lock_irqsave(&ibdev->sriov.going_down_lock, flags); + if (!ibdev->sriov.is_going_down) + queue_work(ibdev->sriov.demux[i].ud_wq, &dm[i]->work); + spin_unlock_irqrestore(&ibdev->sriov.going_down_lock, flags); + } +out: + if (dm) + kfree(dm); + return; +} + static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr, - enum mlx4_dev_event event, int port) + enum mlx4_dev_event event, unsigned long param) { struct ib_event ibev; struct mlx4_ib_dev *ibdev = to_mdev((struct ib_device *) ibdev_ptr); + struct mlx4_eqe *eqe = NULL; + struct ib_event_work *ew; + int p = 0; - if (port > ibdev->num_ports) - return; + if (event == MLX4_DEV_EVENT_PORT_MGMT_CHANGE) + eqe = (struct mlx4_eqe *)param; + else + p = (int) param; switch (event) { case MLX4_DEV_EVENT_PORT_UP: + if (p > ibdev->num_ports) + return; + if (mlx4_is_master(dev) && + rdma_port_get_link_layer(&ibdev->ib_dev, p) == + IB_LINK_LAYER_INFINIBAND) { + mlx4_ib_invalidate_all_guid_record(ibdev, p); + } + mlx4_ib_info((struct ib_device *) ibdev_ptr, + "Port %d logical link is up\n", p); ibev.event = IB_EVENT_PORT_ACTIVE; break; case MLX4_DEV_EVENT_PORT_DOWN: + if (p > ibdev->num_ports) + return; + mlx4_ib_info((struct ib_device *) ibdev_ptr, + "Port %d logical link is down\n", p); ibev.event = IB_EVENT_PORT_ERR; break; case MLX4_DEV_EVENT_CATASTROPHIC_ERROR: - ibdev->ib_active = 0; + ibdev->ib_active = false; ibev.event = IB_EVENT_DEVICE_FATAL; break; + case MLX4_DEV_EVENT_PORT_MGMT_CHANGE: + ew = kmalloc(sizeof *ew, GFP_ATOMIC); + if (!ew) { + pr_err("failed to allocate memory for events work\n"); + break; + } + + INIT_WORK(&ew->work, handle_port_mgmt_change_event); + memcpy(&ew->ib_eqe, eqe, sizeof *eqe); + ew->ib_dev = ibdev; + /* need to queue only for port owner, which uses GEN_EQE */ + if (mlx4_is_master(dev)) + queue_work(wq, &ew->work); + else + handle_port_mgmt_change_event(&ew->work); + return; + + case MLX4_DEV_EVENT_SLAVE_INIT: + /* here, p is the slave id */ + do_slave_init(ibdev, p, 1); + return; + + case MLX4_DEV_EVENT_SLAVE_SHUTDOWN: + /* here, p is the slave id */ + do_slave_init(ibdev, p, 0); + return; + default: return; } ibev.device = ibdev_ptr; - ibev.element.port_num = port; + ibev.element.port_num = (u8) p; ib_dispatch_event(&ibev); } static struct mlx4_interface mlx4_ib_interface = { - .add = mlx4_ib_add, - .remove = mlx4_ib_remove, - .event = mlx4_ib_event, - .get_prot_dev = get_ibdev, - .protocol = MLX4_PROT_IB, + .add = mlx4_ib_add, + .remove = mlx4_ib_remove, + .event = mlx4_ib_event, + .protocol = MLX4_PROT_IB_IPV6 }; static int __init mlx4_ib_init(void) { int err; wq = create_singlethread_workqueue("mlx4_ib"); if (!wq) return -ENOMEM; +#ifdef __linux__ + err = mlx4_ib_proc_init(); + if (err) + goto clean_wq; +#endif + + err = mlx4_ib_mcg_init(); + if (err) + goto clean_proc; + + init_dev_assign(); + err = mlx4_register_interface(&mlx4_ib_interface); - if (err) { - destroy_workqueue(wq); - return err; - } + if (err) + goto clean_mcg; return 0; + +clean_mcg: + mlx4_ib_mcg_destroy(); + +clean_proc: +#ifdef __linux__ + remove_proc_entry(MLX4_IB_MRS_PROC_DIR_NAME, + mlx4_ib_driver_dir_entry); + remove_proc_entry(MLX4_IB_DRIVER_PROC_DIR_NAME, NULL); + +clean_wq: +#endif + destroy_workqueue(wq); + return err; } static void __exit mlx4_ib_cleanup(void) { mlx4_unregister_interface(&mlx4_ib_interface); + mlx4_ib_mcg_destroy(); destroy_workqueue(wq); + + /* Remove proc entries */ +#ifdef __linux__ + remove_proc_entry(MLX4_IB_MRS_PROC_DIR_NAME, + mlx4_ib_driver_dir_entry); + remove_proc_entry(MLX4_IB_DRIVER_PROC_DIR_NAME, NULL); +#endif + } module_init_order(mlx4_ib_init, SI_ORDER_MIDDLE); module_exit(mlx4_ib_cleanup); #undef MODULE_VERSION #include static int mlx4ib_evhand(module_t mod, int event, void *arg) { return (0); } + static moduledata_t mlx4ib_mod = { .name = "mlx4ib", .evhand = mlx4ib_evhand, }; DECLARE_MODULE(mlx4ib, mlx4ib_mod, SI_SUB_OFED_PREINIT, SI_ORDER_ANY); MODULE_DEPEND(mlx4ib, mlx4, 1, 1, 1); +MODULE_DEPEND(mlx4ib, ibcore, 1, 1, 1); Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/mcg.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/mcg.c (nonexistent) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/mcg.c (revision 273864) @@ -0,0 +1,1254 @@ +/* + * Copyright (c) 2012 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include + +#include +#include + +#include "mlx4_ib.h" + +#define MAX_VFS 80 +#define MAX_PEND_REQS_PER_FUNC 4 +#define MAD_TIMEOUT_MS 2000 + +#define mcg_warn(fmt, arg...) pr_warn("MCG WARNING: " fmt, ##arg) +#define mcg_error(fmt, arg...) pr_err(fmt, ##arg) +#define mcg_warn_group(group, format, arg...) \ + pr_warn("%s-%d: %16s (port %d): WARNING: " format, __func__, __LINE__,\ + (group)->name, group->demux->port, ## arg) + +#define mcg_error_group(group, format, arg...) \ + pr_err(" %16s: " format, (group)->name, ## arg) + +static union ib_gid mgid0; + +static struct workqueue_struct *clean_wq; + +enum mcast_state { + MCAST_NOT_MEMBER = 0, + MCAST_MEMBER, +}; + +enum mcast_group_state { + MCAST_IDLE, + MCAST_JOIN_SENT, + MCAST_LEAVE_SENT, + MCAST_RESP_READY +}; + +struct mcast_member { + enum mcast_state state; + uint8_t join_state; + int num_pend_reqs; + struct list_head pending; +}; + +struct ib_sa_mcmember_data { + union ib_gid mgid; + union ib_gid port_gid; + __be32 qkey; + __be16 mlid; + u8 mtusel_mtu; + u8 tclass; + __be16 pkey; + u8 ratesel_rate; + u8 lifetmsel_lifetm; + __be32 sl_flowlabel_hoplimit; + u8 scope_join_state; + u8 proxy_join; + u8 reserved[2]; +}; + +struct mcast_group { + struct ib_sa_mcmember_data rec; + struct rb_node node; + struct list_head mgid0_list; + struct mlx4_ib_demux_ctx *demux; + struct mcast_member func[MAX_VFS]; + struct mutex lock; + struct work_struct work; + struct list_head pending_list; + int members[3]; + enum mcast_group_state state; + enum mcast_group_state prev_state; + struct ib_sa_mad response_sa_mad; + __be64 last_req_tid; + + char name[33]; /* MGID string */ + struct device_attribute dentry; + + /* refcount is the reference count for the following: + 1. Each queued request + 2. Each invocation of the worker thread + 3. Membership of the port at the SA + */ + atomic_t refcount; + + /* delayed work to clean pending SM request */ + struct delayed_work timeout_work; + struct list_head cleanup_list; +}; + +struct mcast_req { + int func; + struct ib_sa_mad sa_mad; + struct list_head group_list; + struct list_head func_list; + struct mcast_group *group; + int clean; +}; + + +#define safe_atomic_dec(ref) \ + do {\ + if (atomic_dec_and_test(ref)) \ + mcg_warn_group(group, "did not expect to reach zero\n"); \ + } while (0) + +static const char *get_state_string(enum mcast_group_state state) +{ + switch (state) { + case MCAST_IDLE: + return "MCAST_IDLE"; + case MCAST_JOIN_SENT: + return "MCAST_JOIN_SENT"; + case MCAST_LEAVE_SENT: + return "MCAST_LEAVE_SENT"; + case MCAST_RESP_READY: + return "MCAST_RESP_READY"; + } + return "Invalid State"; +} + +static struct mcast_group *mcast_find(struct mlx4_ib_demux_ctx *ctx, + union ib_gid *mgid) +{ + struct rb_node *node = ctx->mcg_table.rb_node; + struct mcast_group *group; + int ret; + + while (node) { + group = rb_entry(node, struct mcast_group, node); + ret = memcmp(mgid->raw, group->rec.mgid.raw, sizeof *mgid); + if (!ret) + return group; + + if (ret < 0) + node = node->rb_left; + else + node = node->rb_right; + } + return NULL; +} + +static struct mcast_group *mcast_insert(struct mlx4_ib_demux_ctx *ctx, + struct mcast_group *group) +{ + struct rb_node **link = &ctx->mcg_table.rb_node; + struct rb_node *parent = NULL; + struct mcast_group *cur_group; + int ret; + + while (*link) { + parent = *link; + cur_group = rb_entry(parent, struct mcast_group, node); + + ret = memcmp(group->rec.mgid.raw, cur_group->rec.mgid.raw, + sizeof group->rec.mgid); + if (ret < 0) + link = &(*link)->rb_left; + else if (ret > 0) + link = &(*link)->rb_right; + else + return cur_group; + } + rb_link_node(&group->node, parent, link); + rb_insert_color(&group->node, &ctx->mcg_table); + return NULL; +} + +static int send_mad_to_wire(struct mlx4_ib_demux_ctx *ctx, struct ib_mad *mad) +{ + struct mlx4_ib_dev *dev = ctx->dev; + struct ib_ah_attr ah_attr; + + spin_lock(&dev->sm_lock); + if (!dev->sm_ah[ctx->port - 1]) { + /* port is not yet Active, sm_ah not ready */ + spin_unlock(&dev->sm_lock); + return -EAGAIN; + } + mlx4_ib_query_ah(dev->sm_ah[ctx->port - 1], &ah_attr); + spin_unlock(&dev->sm_lock); + return mlx4_ib_send_to_wire(dev, mlx4_master_func_num(dev->dev), ctx->port, + IB_QPT_GSI, 0, 1, IB_QP1_QKEY, &ah_attr, mad); +} + +static int send_mad_to_slave(int slave, struct mlx4_ib_demux_ctx *ctx, + struct ib_mad *mad) +{ + struct mlx4_ib_dev *dev = ctx->dev; + struct ib_mad_agent *agent = dev->send_agent[ctx->port - 1][1]; + struct ib_wc wc; + struct ib_ah_attr ah_attr; + + /* Our agent might not yet be registered when mads start to arrive */ + if (!agent) + return -EAGAIN; + + ib_query_ah(dev->sm_ah[ctx->port - 1], &ah_attr); + + if (ib_find_cached_pkey(&dev->ib_dev, ctx->port, IB_DEFAULT_PKEY_FULL, &wc.pkey_index)) + return -EINVAL; + wc.sl = 0; + wc.dlid_path_bits = 0; + wc.port_num = ctx->port; + wc.slid = ah_attr.dlid; /* opensm lid */ + wc.src_qp = 1; + return mlx4_ib_send_to_slave(dev, slave, ctx->port, IB_QPT_GSI, &wc, NULL, mad); +} + +static int send_join_to_wire(struct mcast_group *group, struct ib_sa_mad *sa_mad) +{ + struct ib_sa_mad mad; + struct ib_sa_mcmember_data *sa_mad_data = (struct ib_sa_mcmember_data *)&mad.data; + int ret; + + /* we rely on a mad request as arrived from a VF */ + memcpy(&mad, sa_mad, sizeof mad); + + /* fix port GID to be the real one (slave 0) */ + sa_mad_data->port_gid.global.interface_id = group->demux->guid_cache[0]; + + /* assign our own TID */ + mad.mad_hdr.tid = mlx4_ib_get_new_demux_tid(group->demux); + group->last_req_tid = mad.mad_hdr.tid; /* keep it for later validation */ + + ret = send_mad_to_wire(group->demux, (struct ib_mad *)&mad); + /* set timeout handler */ + if (!ret) { + /* calls mlx4_ib_mcg_timeout_handler */ + queue_delayed_work(group->demux->mcg_wq, &group->timeout_work, + msecs_to_jiffies(MAD_TIMEOUT_MS)); + } + + return ret; +} + +static int send_leave_to_wire(struct mcast_group *group, u8 join_state) +{ + struct ib_sa_mad mad; + struct ib_sa_mcmember_data *sa_data = (struct ib_sa_mcmember_data *)&mad.data; + int ret; + + memset(&mad, 0, sizeof mad); + mad.mad_hdr.base_version = 1; + mad.mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; + mad.mad_hdr.class_version = 2; + mad.mad_hdr.method = IB_SA_METHOD_DELETE; + mad.mad_hdr.status = cpu_to_be16(0); + mad.mad_hdr.class_specific = cpu_to_be16(0); + mad.mad_hdr.tid = mlx4_ib_get_new_demux_tid(group->demux); + group->last_req_tid = mad.mad_hdr.tid; /* keep it for later validation */ + mad.mad_hdr.attr_id = cpu_to_be16(IB_SA_ATTR_MC_MEMBER_REC); + mad.mad_hdr.attr_mod = cpu_to_be32(0); + mad.sa_hdr.sm_key = 0x0; + mad.sa_hdr.attr_offset = cpu_to_be16(7); + mad.sa_hdr.comp_mask = IB_SA_MCMEMBER_REC_MGID | + IB_SA_MCMEMBER_REC_PORT_GID | IB_SA_MCMEMBER_REC_JOIN_STATE; + + *sa_data = group->rec; + sa_data->scope_join_state = join_state; + + ret = send_mad_to_wire(group->demux, (struct ib_mad *)&mad); + if (ret) + group->state = MCAST_IDLE; + + /* set timeout handler */ + if (!ret) { + /* calls mlx4_ib_mcg_timeout_handler */ + queue_delayed_work(group->demux->mcg_wq, &group->timeout_work, + msecs_to_jiffies(MAD_TIMEOUT_MS)); + } + + return ret; +} + +static int send_reply_to_slave(int slave, struct mcast_group *group, + struct ib_sa_mad *req_sa_mad, u16 status) +{ + struct ib_sa_mad mad; + struct ib_sa_mcmember_data *sa_data = (struct ib_sa_mcmember_data *)&mad.data; + struct ib_sa_mcmember_data *req_sa_data = (struct ib_sa_mcmember_data *)&req_sa_mad->data; + int ret; + + memset(&mad, 0, sizeof mad); + mad.mad_hdr.base_version = 1; + mad.mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; + mad.mad_hdr.class_version = 2; + mad.mad_hdr.method = IB_MGMT_METHOD_GET_RESP; + mad.mad_hdr.status = cpu_to_be16(status); + mad.mad_hdr.class_specific = cpu_to_be16(0); + mad.mad_hdr.tid = req_sa_mad->mad_hdr.tid; + *(u8 *)&mad.mad_hdr.tid = 0; /* resetting tid to 0 */ + mad.mad_hdr.attr_id = cpu_to_be16(IB_SA_ATTR_MC_MEMBER_REC); + mad.mad_hdr.attr_mod = cpu_to_be32(0); + mad.sa_hdr.sm_key = req_sa_mad->sa_hdr.sm_key; + mad.sa_hdr.attr_offset = cpu_to_be16(7); + mad.sa_hdr.comp_mask = 0; /* ignored on responses, see IBTA spec */ + + *sa_data = group->rec; + + /* reconstruct VF's requested join_state and port_gid */ + sa_data->scope_join_state &= 0xf0; + sa_data->scope_join_state |= (group->func[slave].join_state & 0x0f); + memcpy(&sa_data->port_gid, &req_sa_data->port_gid, sizeof req_sa_data->port_gid); + + ret = send_mad_to_slave(slave, group->demux, (struct ib_mad *)&mad); + return ret; +} + +static int check_selector(ib_sa_comp_mask comp_mask, + ib_sa_comp_mask selector_mask, + ib_sa_comp_mask value_mask, + u8 src_value, u8 dst_value) +{ + int err; + u8 selector = dst_value >> 6; + dst_value &= 0x3f; + src_value &= 0x3f; + + if (!(comp_mask & selector_mask) || !(comp_mask & value_mask)) + return 0; + + switch (selector) { + case IB_SA_GT: + err = (src_value <= dst_value); + break; + case IB_SA_LT: + err = (src_value >= dst_value); + break; + case IB_SA_EQ: + err = (src_value != dst_value); + break; + default: + err = 0; + break; + } + + return err; +} + +static u16 cmp_rec(struct ib_sa_mcmember_data *src, + struct ib_sa_mcmember_data *dst, ib_sa_comp_mask comp_mask) +{ + /* src is group record, dst is request record */ + /* MGID must already match */ + /* Port_GID we always replace to our Port_GID, so it is a match */ + +#define MAD_STATUS_REQ_INVALID 0x0200 + if (comp_mask & IB_SA_MCMEMBER_REC_QKEY && src->qkey != dst->qkey) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_MLID && src->mlid != dst->mlid) + return MAD_STATUS_REQ_INVALID; + if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_MTU_SELECTOR, + IB_SA_MCMEMBER_REC_MTU, + src->mtusel_mtu, dst->mtusel_mtu)) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_TRAFFIC_CLASS && + src->tclass != dst->tclass) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_PKEY && src->pkey != dst->pkey) + return MAD_STATUS_REQ_INVALID; + if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_RATE_SELECTOR, + IB_SA_MCMEMBER_REC_RATE, + src->ratesel_rate, dst->ratesel_rate)) + return MAD_STATUS_REQ_INVALID; + if (check_selector(comp_mask, + IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR, + IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME, + src->lifetmsel_lifetm, dst->lifetmsel_lifetm)) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_SL && + (be32_to_cpu(src->sl_flowlabel_hoplimit) & 0xf0000000) != + (be32_to_cpu(dst->sl_flowlabel_hoplimit) & 0xf0000000)) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_FLOW_LABEL && + (be32_to_cpu(src->sl_flowlabel_hoplimit) & 0x0fffff00) != + (be32_to_cpu(dst->sl_flowlabel_hoplimit) & 0x0fffff00)) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_HOP_LIMIT && + (be32_to_cpu(src->sl_flowlabel_hoplimit) & 0x000000ff) != + (be32_to_cpu(dst->sl_flowlabel_hoplimit) & 0x000000ff)) + return MAD_STATUS_REQ_INVALID; + if (comp_mask & IB_SA_MCMEMBER_REC_SCOPE && + (src->scope_join_state & 0xf0) != + (dst->scope_join_state & 0xf0)) + return MAD_STATUS_REQ_INVALID; + + /* join_state checked separately, proxy_join ignored */ + + return 0; +} + +/* release group, return 1 if this was last release and group is destroyed + * timout work is canceled sync */ +static int release_group(struct mcast_group *group, int from_timeout_handler) +{ + struct mlx4_ib_demux_ctx *ctx = group->demux; + int nzgroup; + + mutex_lock(&ctx->mcg_table_lock); + mutex_lock(&group->lock); + if (atomic_dec_and_test(&group->refcount)) { + if (!from_timeout_handler) { + if (group->state != MCAST_IDLE && + !cancel_delayed_work(&group->timeout_work)) { + atomic_inc(&group->refcount); + mutex_unlock(&group->lock); + mutex_unlock(&ctx->mcg_table_lock); + return 0; + } + } + + nzgroup = memcmp(&group->rec.mgid, &mgid0, sizeof mgid0); + if (nzgroup) + del_sysfs_port_mcg_attr(ctx->dev, ctx->port, &group->dentry.attr); + if (!list_empty(&group->pending_list)) + mcg_warn_group(group, "releasing a group with non empty pending list\n"); + if (nzgroup) + rb_erase(&group->node, &ctx->mcg_table); + list_del_init(&group->mgid0_list); + mutex_unlock(&group->lock); + mutex_unlock(&ctx->mcg_table_lock); + kfree(group); + return 1; + } else { + mutex_unlock(&group->lock); + mutex_unlock(&ctx->mcg_table_lock); + } + return 0; +} + +static void adjust_membership(struct mcast_group *group, u8 join_state, int inc) +{ + int i; + + for (i = 0; i < 3; i++, join_state >>= 1) + if (join_state & 0x1) + group->members[i] += inc; +} + +static u8 get_leave_state(struct mcast_group *group) +{ + u8 leave_state = 0; + int i; + + for (i = 0; i < 3; i++) + if (!group->members[i]) + leave_state |= (1 << i); + + return leave_state & (group->rec.scope_join_state & 7); +} + +static int join_group(struct mcast_group *group, int slave, u8 join_mask) +{ + int ret = 0; + u8 join_state; + + /* remove bits that slave is already member of, and adjust */ + join_state = join_mask & (~group->func[slave].join_state); + adjust_membership(group, join_state, 1); + group->func[slave].join_state |= join_state; + if (group->func[slave].state != MCAST_MEMBER && join_state) { + group->func[slave].state = MCAST_MEMBER; + ret = 1; + } + return ret; +} + +static int leave_group(struct mcast_group *group, int slave, u8 leave_state) +{ + int ret = 0; + + adjust_membership(group, leave_state, -1); + group->func[slave].join_state &= ~leave_state; + if (!group->func[slave].join_state) { + group->func[slave].state = MCAST_NOT_MEMBER; + ret = 1; + } + return ret; +} + +static int check_leave(struct mcast_group *group, int slave, u8 leave_mask) +{ + if (group->func[slave].state != MCAST_MEMBER) + return MAD_STATUS_REQ_INVALID; + + /* make sure we're not deleting unset bits */ + if (~group->func[slave].join_state & leave_mask) + return MAD_STATUS_REQ_INVALID; + + if (!leave_mask) + return MAD_STATUS_REQ_INVALID; + + return 0; +} + +static void mlx4_ib_mcg_timeout_handler(struct work_struct *work) +{ + struct delayed_work *delay = to_delayed_work(work); + struct mcast_group *group; + struct mcast_req *req = NULL; + + group = container_of(delay, typeof(*group), timeout_work); + + mutex_lock(&group->lock); + if (group->state == MCAST_JOIN_SENT) { + if (!list_empty(&group->pending_list)) { + req = list_first_entry(&group->pending_list, struct mcast_req, group_list); + list_del(&req->group_list); + list_del(&req->func_list); + --group->func[req->func].num_pend_reqs; + mutex_unlock(&group->lock); + kfree(req); + if (memcmp(&group->rec.mgid, &mgid0, sizeof mgid0)) { + if (release_group(group, 1)) + return; + } else { + kfree(group); + return; + } + mutex_lock(&group->lock); + } else + mcg_warn_group(group, "DRIVER BUG\n"); + } else if (group->state == MCAST_LEAVE_SENT) { + if (group->rec.scope_join_state & 7) + group->rec.scope_join_state &= 0xf8; + group->state = MCAST_IDLE; + mutex_unlock(&group->lock); + if (release_group(group, 1)) + return; + mutex_lock(&group->lock); + } else + mcg_warn_group(group, "invalid state %s\n", get_state_string(group->state)); + group->state = MCAST_IDLE; + atomic_inc(&group->refcount); + queue_work(group->demux->mcg_wq, &group->work); + safe_atomic_dec(&group->refcount); + + mutex_unlock(&group->lock); +} + +static int handle_leave_req(struct mcast_group *group, u8 leave_mask, + struct mcast_req *req) +{ + u16 status; + + if (req->clean) + leave_mask = group->func[req->func].join_state; + + status = check_leave(group, req->func, leave_mask); + if (!status) + leave_group(group, req->func, leave_mask); + + if (!req->clean) + send_reply_to_slave(req->func, group, &req->sa_mad, status); + --group->func[req->func].num_pend_reqs; + list_del(&req->group_list); + list_del(&req->func_list); + kfree(req); + return 1; +} + +static int handle_join_req(struct mcast_group *group, u8 join_mask, + struct mcast_req *req) +{ + u8 group_join_state = group->rec.scope_join_state & 7; + int ref = 0; + u16 status; + struct ib_sa_mcmember_data *sa_data = (struct ib_sa_mcmember_data *)req->sa_mad.data; + + if (join_mask == (group_join_state & join_mask)) { + /* port's membership need not change */ + status = cmp_rec(&group->rec, sa_data, req->sa_mad.sa_hdr.comp_mask); + if (!status) + join_group(group, req->func, join_mask); + + --group->func[req->func].num_pend_reqs; + send_reply_to_slave(req->func, group, &req->sa_mad, status); + list_del(&req->group_list); + list_del(&req->func_list); + kfree(req); + ++ref; + } else { + /* port's membership needs to be updated */ + group->prev_state = group->state; + if (send_join_to_wire(group, &req->sa_mad)) { + --group->func[req->func].num_pend_reqs; + list_del(&req->group_list); + list_del(&req->func_list); + kfree(req); + ref = 1; + group->state = group->prev_state; + } else + group->state = MCAST_JOIN_SENT; + } + + return ref; +} + +static void mlx4_ib_mcg_work_handler(struct work_struct *work) +{ + struct mcast_group *group; + struct mcast_req *req = NULL; + struct ib_sa_mcmember_data *sa_data; + u8 req_join_state; + int rc = 1; /* release_count - this is for the scheduled work */ + u16 status; + u8 method; + + group = container_of(work, typeof(*group), work); + + mutex_lock(&group->lock); + + /* First, let's see if a response from SM is waiting regarding this group. + * If so, we need to update the group's REC. If this is a bad response, we + * may need to send a bad response to a VF waiting for it. If VF is waiting + * and this is a good response, the VF will be answered later in this func. */ + if (group->state == MCAST_RESP_READY) { + /* cancels mlx4_ib_mcg_timeout_handler */ + cancel_delayed_work(&group->timeout_work); + status = be16_to_cpu(group->response_sa_mad.mad_hdr.status); + method = group->response_sa_mad.mad_hdr.method; + if (group->last_req_tid != group->response_sa_mad.mad_hdr.tid) { + mcg_warn_group(group, "Got MAD response to existing MGID but wrong TID, dropping. Resp TID=%llx, group TID=%llx\n", + (long long unsigned int)be64_to_cpu(group->response_sa_mad.mad_hdr.tid), + (long long unsigned int)be64_to_cpu(group->last_req_tid)); + group->state = group->prev_state; + goto process_requests; + } + if (status) { + if (!list_empty(&group->pending_list)) + req = list_first_entry(&group->pending_list, + struct mcast_req, group_list); + if (method == IB_MGMT_METHOD_GET_RESP) { + if (req) { + send_reply_to_slave(req->func, group, &req->sa_mad, status); + --group->func[req->func].num_pend_reqs; + list_del(&req->group_list); + list_del(&req->func_list); + kfree(req); + ++rc; + } else + mcg_warn_group(group, "no request for failed join\n"); + } else if (method == IB_SA_METHOD_DELETE_RESP && group->demux->flushing) + ++rc; + } else { + u8 resp_join_state; + u8 cur_join_state; + + resp_join_state = ((struct ib_sa_mcmember_data *) + group->response_sa_mad.data)->scope_join_state & 7; + cur_join_state = group->rec.scope_join_state & 7; + + if (method == IB_MGMT_METHOD_GET_RESP) { + /* successfull join */ + if (!cur_join_state && resp_join_state) + --rc; + } else if (!resp_join_state) + ++rc; + memcpy(&group->rec, group->response_sa_mad.data, sizeof group->rec); + } + group->state = MCAST_IDLE; + } + +process_requests: + /* We should now go over pending join/leave requests, as long as we are idle. */ + while (!list_empty(&group->pending_list) && group->state == MCAST_IDLE) { + req = list_first_entry(&group->pending_list, struct mcast_req, + group_list); + sa_data = (struct ib_sa_mcmember_data *)req->sa_mad.data; + req_join_state = sa_data->scope_join_state & 0x7; + + /* For a leave request, we will immediately answer the VF, and + * update our internal counters. The actual leave will be sent + * to SM later, if at all needed. We dequeue the request now. */ + if (req->sa_mad.mad_hdr.method == IB_SA_METHOD_DELETE) + rc += handle_leave_req(group, req_join_state, req); + else + rc += handle_join_req(group, req_join_state, req); + } + + /* Handle leaves */ + if (group->state == MCAST_IDLE) { + req_join_state = get_leave_state(group); + if (req_join_state) { + group->rec.scope_join_state &= ~req_join_state; + group->prev_state = group->state; + if (send_leave_to_wire(group, req_join_state)) { + group->state = group->prev_state; + ++rc; + } else + group->state = MCAST_LEAVE_SENT; + } + } + + if (!list_empty(&group->pending_list) && group->state == MCAST_IDLE) + goto process_requests; + mutex_unlock(&group->lock); + + while (rc--) + release_group(group, 0); +} + +static struct mcast_group *search_relocate_mgid0_group(struct mlx4_ib_demux_ctx *ctx, + __be64 tid, + union ib_gid *new_mgid) +{ + struct mcast_group *group = NULL, *cur_group; + struct mcast_req *req; + struct list_head *pos; + struct list_head *n; + + mutex_lock(&ctx->mcg_table_lock); + list_for_each_safe(pos, n, &ctx->mcg_mgid0_list) { + group = list_entry(pos, struct mcast_group, mgid0_list); + mutex_lock(&group->lock); + if (group->last_req_tid == tid) { + if (memcmp(new_mgid, &mgid0, sizeof mgid0)) { + group->rec.mgid = *new_mgid; + sprintf(group->name, "%016llx%016llx", + (long long unsigned int)be64_to_cpu(group->rec.mgid.global.subnet_prefix), + (long long unsigned int)be64_to_cpu(group->rec.mgid.global.interface_id)); + list_del_init(&group->mgid0_list); + cur_group = mcast_insert(ctx, group); + if (cur_group) { + /* A race between our code and SM. Silently cleaning the new one */ + req = list_first_entry(&group->pending_list, + struct mcast_req, group_list); + --group->func[req->func].num_pend_reqs; + list_del(&req->group_list); + list_del(&req->func_list); + kfree(req); + mutex_unlock(&group->lock); + mutex_unlock(&ctx->mcg_table_lock); + release_group(group, 0); + return NULL; + } + + atomic_inc(&group->refcount); + add_sysfs_port_mcg_attr(ctx->dev, ctx->port, &group->dentry.attr); + mutex_unlock(&group->lock); + mutex_unlock(&ctx->mcg_table_lock); + return group; + } else { + struct mcast_req *tmp1, *tmp2; + + list_del(&group->mgid0_list); + if (!list_empty(&group->pending_list) && group->state != MCAST_IDLE) + cancel_delayed_work_sync(&group->timeout_work); + + list_for_each_entry_safe(tmp1, tmp2, &group->pending_list, group_list) { + list_del(&tmp1->group_list); + kfree(tmp1); + } + mutex_unlock(&group->lock); + mutex_unlock(&ctx->mcg_table_lock); + kfree(group); + return NULL; + } + } + mutex_unlock(&group->lock); + } + mutex_unlock(&ctx->mcg_table_lock); + + return NULL; +} + +static ssize_t sysfs_show_group(struct device *dev, + struct device_attribute *attr, char *buf); + +static struct mcast_group *acquire_group(struct mlx4_ib_demux_ctx *ctx, + union ib_gid *mgid, int create, + gfp_t gfp_mask) +{ + struct mcast_group *group, *cur_group; + int is_mgid0; + int i; + + is_mgid0 = !memcmp(&mgid0, mgid, sizeof mgid0); + if (!is_mgid0) { + group = mcast_find(ctx, mgid); + if (group) + goto found; + } + + if (!create) + return ERR_PTR(-ENOENT); + + group = kzalloc(sizeof *group, gfp_mask); + if (!group) + return ERR_PTR(-ENOMEM); + + group->demux = ctx; + group->rec.mgid = *mgid; + INIT_LIST_HEAD(&group->pending_list); + INIT_LIST_HEAD(&group->mgid0_list); + for (i = 0; i < MAX_VFS; ++i) + INIT_LIST_HEAD(&group->func[i].pending); + INIT_WORK(&group->work, mlx4_ib_mcg_work_handler); + INIT_DELAYED_WORK(&group->timeout_work, mlx4_ib_mcg_timeout_handler); + mutex_init(&group->lock); + sprintf(group->name, "%016llx%016llx", + (long long unsigned int)be64_to_cpu(group->rec.mgid.global.subnet_prefix), + (long long unsigned int)be64_to_cpu(group->rec.mgid.global.interface_id)); + sysfs_attr_init(&group->dentry.attr); + group->dentry.show = sysfs_show_group; + group->dentry.store = NULL; + group->dentry.attr.name = group->name; + group->dentry.attr.mode = 0400; + group->state = MCAST_IDLE; + + if (is_mgid0) { + list_add(&group->mgid0_list, &ctx->mcg_mgid0_list); + goto found; + } + + cur_group = mcast_insert(ctx, group); + if (cur_group) { + mcg_warn("group just showed up %s - confused\n", cur_group->name); + kfree(group); + return ERR_PTR(-EINVAL); + } + + add_sysfs_port_mcg_attr(ctx->dev, ctx->port, &group->dentry.attr); + +found: + atomic_inc(&group->refcount); + return group; +} + +static void queue_req(struct mcast_req *req) +{ + struct mcast_group *group = req->group; + + atomic_inc(&group->refcount); /* for the request */ + atomic_inc(&group->refcount); /* for scheduling the work */ + list_add_tail(&req->group_list, &group->pending_list); + list_add_tail(&req->func_list, &group->func[req->func].pending); + /* calls mlx4_ib_mcg_work_handler */ + queue_work(group->demux->mcg_wq, &group->work); + safe_atomic_dec(&group->refcount); +} + +int mlx4_ib_mcg_demux_handler(struct ib_device *ibdev, int port, int slave, + struct ib_sa_mad *mad) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + struct ib_sa_mcmember_data *rec = (struct ib_sa_mcmember_data *)mad->data; + struct mlx4_ib_demux_ctx *ctx = &dev->sriov.demux[port - 1]; + struct mcast_group *group; + + switch (mad->mad_hdr.method) { + case IB_MGMT_METHOD_GET_RESP: + case IB_SA_METHOD_DELETE_RESP: + mutex_lock(&ctx->mcg_table_lock); + group = acquire_group(ctx, &rec->mgid, 0, GFP_KERNEL); + mutex_unlock(&ctx->mcg_table_lock); + if (IS_ERR(group)) { + if (mad->mad_hdr.method == IB_MGMT_METHOD_GET_RESP) { + __be64 tid = mad->mad_hdr.tid; + *(u8 *)(&tid) = (u8)slave; /* in group we kept the modified TID */ + group = search_relocate_mgid0_group(ctx, tid, &rec->mgid); + } else + group = NULL; + } + + if (!group) + return 1; + + mutex_lock(&group->lock); + group->response_sa_mad = *mad; + group->prev_state = group->state; + group->state = MCAST_RESP_READY; + /* calls mlx4_ib_mcg_work_handler */ + atomic_inc(&group->refcount); + queue_work(ctx->mcg_wq, &group->work); + safe_atomic_dec(&group->refcount); + mutex_unlock(&group->lock); + release_group(group, 0); + return 1; /* consumed */ + case IB_MGMT_METHOD_SET: + case IB_SA_METHOD_GET_TABLE: + case IB_SA_METHOD_GET_TABLE_RESP: + case IB_SA_METHOD_DELETE: + return 0; /* not consumed, pass-through to guest over tunnel */ + default: + mcg_warn("In demux, port %d: unexpected MCMember method: 0x%x, dropping\n", + port, mad->mad_hdr.method); + return 1; /* consumed */ + } +} + +int mlx4_ib_mcg_multiplex_handler(struct ib_device *ibdev, int port, + int slave, struct ib_sa_mad *sa_mad) +{ + struct mlx4_ib_dev *dev = to_mdev(ibdev); + struct ib_sa_mcmember_data *rec = (struct ib_sa_mcmember_data *)sa_mad->data; + struct mlx4_ib_demux_ctx *ctx = &dev->sriov.demux[port - 1]; + struct mcast_group *group; + struct mcast_req *req; + int may_create = 0; + + if (ctx->flushing) + return -EAGAIN; + + switch (sa_mad->mad_hdr.method) { + case IB_MGMT_METHOD_SET: + may_create = 1; + case IB_SA_METHOD_DELETE: + req = kzalloc(sizeof *req, GFP_KERNEL); + if (!req) + return -ENOMEM; + + req->func = slave; + req->sa_mad = *sa_mad; + + mutex_lock(&ctx->mcg_table_lock); + group = acquire_group(ctx, &rec->mgid, may_create, GFP_KERNEL); + mutex_unlock(&ctx->mcg_table_lock); + if (IS_ERR(group)) { + kfree(req); + return PTR_ERR(group); + } + mutex_lock(&group->lock); + if (group->func[slave].num_pend_reqs > MAX_PEND_REQS_PER_FUNC) { + mutex_unlock(&group->lock); + mcg_warn_group(group, "Port %d, Func %d has too many pending requests (%d), dropping\n", + port, slave, MAX_PEND_REQS_PER_FUNC); + release_group(group, 0); + kfree(req); + return -ENOMEM; + } + ++group->func[slave].num_pend_reqs; + req->group = group; + queue_req(req); + mutex_unlock(&group->lock); + release_group(group, 0); + return 1; /* consumed */ + case IB_SA_METHOD_GET_TABLE: + case IB_MGMT_METHOD_GET_RESP: + case IB_SA_METHOD_GET_TABLE_RESP: + case IB_SA_METHOD_DELETE_RESP: + return 0; /* not consumed, pass-through */ + default: + mcg_warn("In multiplex, port %d, func %d: unexpected MCMember method: 0x%x, dropping\n", + port, slave, sa_mad->mad_hdr.method); + return 1; /* consumed */ + } +} + +static ssize_t sysfs_show_group(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct mcast_group *group = + container_of(attr, struct mcast_group, dentry); + struct mcast_req *req = NULL; + char pending_str[40]; + char state_str[40]; + ssize_t len = 0; + int f; + + if (group->state == MCAST_IDLE) + sprintf(state_str, "%s", get_state_string(group->state)); + else + sprintf(state_str, "%s(TID=0x%llx)", + get_state_string(group->state), + (long long unsigned int)be64_to_cpu(group->last_req_tid)); + if (list_empty(&group->pending_list)) { + sprintf(pending_str, "No"); + } else { + req = list_first_entry(&group->pending_list, struct mcast_req, group_list); + sprintf(pending_str, "Yes(TID=0x%llx)", + (long long unsigned int)be64_to_cpu(req->sa_mad.mad_hdr.tid)); + } + len += sprintf(buf + len, "%1d [%02d,%02d,%02d] %4d %4s %5s ", + group->rec.scope_join_state & 0xf, + group->members[2], group->members[1], group->members[0], + atomic_read(&group->refcount), + pending_str, + state_str); + for (f = 0; f < MAX_VFS; ++f) + if (group->func[f].state == MCAST_MEMBER) + len += sprintf(buf + len, "%d[%1x] ", + f, group->func[f].join_state); + + len += sprintf(buf + len, "\t\t(%4hx %4x %2x %2x %2x %2x %2x " + "%4x %4x %2x %2x)\n", + be16_to_cpu(group->rec.pkey), + be32_to_cpu(group->rec.qkey), + (group->rec.mtusel_mtu & 0xc0) >> 6, + group->rec.mtusel_mtu & 0x3f, + group->rec.tclass, + (group->rec.ratesel_rate & 0xc0) >> 6, + group->rec.ratesel_rate & 0x3f, + (be32_to_cpu(group->rec.sl_flowlabel_hoplimit) & 0xf0000000) >> 28, + (be32_to_cpu(group->rec.sl_flowlabel_hoplimit) & 0x0fffff00) >> 8, + be32_to_cpu(group->rec.sl_flowlabel_hoplimit) & 0x000000ff, + group->rec.proxy_join); + + return len; +} + +int mlx4_ib_mcg_port_init(struct mlx4_ib_demux_ctx *ctx) +{ + char name[20]; + + atomic_set(&ctx->tid, 0); + sprintf(name, "mlx4_ib_mcg%d", ctx->port); + ctx->mcg_wq = create_singlethread_workqueue(name); + if (!ctx->mcg_wq) + return -ENOMEM; + + mutex_init(&ctx->mcg_table_lock); + ctx->mcg_table = RB_ROOT; + INIT_LIST_HEAD(&ctx->mcg_mgid0_list); + ctx->flushing = 0; + + return 0; +} + +static void force_clean_group(struct mcast_group *group) +{ + struct mcast_req *req, *tmp + ; + list_for_each_entry_safe(req, tmp, &group->pending_list, group_list) { + list_del(&req->group_list); + kfree(req); + } + del_sysfs_port_mcg_attr(group->demux->dev, group->demux->port, &group->dentry.attr); + rb_erase(&group->node, &group->demux->mcg_table); + kfree(group); +} + +static void _mlx4_ib_mcg_port_cleanup(struct mlx4_ib_demux_ctx *ctx, int destroy_wq) +{ + int i; + struct rb_node *p; + struct mcast_group *group; + unsigned long end; + int count; + + for (i = 0; i < MAX_VFS; ++i) + clean_vf_mcast(ctx, i); + + end = jiffies + msecs_to_jiffies(MAD_TIMEOUT_MS + 3000); + do { + count = 0; + mutex_lock(&ctx->mcg_table_lock); + for (p = rb_first(&ctx->mcg_table); p; p = rb_next(p)) + ++count; + mutex_unlock(&ctx->mcg_table_lock); + if (!count) + break; + + msleep(1); + } while (time_after(end, jiffies)); + + flush_workqueue(ctx->mcg_wq); + if (destroy_wq) + destroy_workqueue(ctx->mcg_wq); + + mutex_lock(&ctx->mcg_table_lock); + while ((p = rb_first(&ctx->mcg_table)) != NULL) { + group = rb_entry(p, struct mcast_group, node); + if (atomic_read(&group->refcount)) + mcg_warn_group(group, "group refcount %d!!! (pointer %p)\n", atomic_read(&group->refcount), group); + + force_clean_group(group); + } + mutex_unlock(&ctx->mcg_table_lock); +} + +struct clean_work { + struct work_struct work; + struct mlx4_ib_demux_ctx *ctx; + int destroy_wq; +}; + +static void mcg_clean_task(struct work_struct *work) +{ + struct clean_work *cw = container_of(work, struct clean_work, work); + + _mlx4_ib_mcg_port_cleanup(cw->ctx, cw->destroy_wq); + cw->ctx->flushing = 0; + kfree(cw); +} + +void mlx4_ib_mcg_port_cleanup(struct mlx4_ib_demux_ctx *ctx, int destroy_wq) +{ + struct clean_work *work; + + if (ctx->flushing) + return; + + ctx->flushing = 1; + + if (destroy_wq) { + _mlx4_ib_mcg_port_cleanup(ctx, destroy_wq); + ctx->flushing = 0; + return; + } + + work = kmalloc(sizeof *work, GFP_KERNEL); + if (!work) { + ctx->flushing = 0; + mcg_warn("failed allocating work for cleanup\n"); + return; + } + + work->ctx = ctx; + work->destroy_wq = destroy_wq; + INIT_WORK(&work->work, mcg_clean_task); + queue_work(clean_wq, &work->work); +} + +static void build_leave_mad(struct mcast_req *req) +{ + struct ib_sa_mad *mad = &req->sa_mad; + + mad->mad_hdr.method = IB_SA_METHOD_DELETE; +} + + +static void clear_pending_reqs(struct mcast_group *group, int vf) +{ + struct mcast_req *req, *tmp, *group_first = NULL; + int clear; + int pend = 0; + + if (!list_empty(&group->pending_list)) + group_first = list_first_entry(&group->pending_list, struct mcast_req, group_list); + + list_for_each_entry_safe(req, tmp, &group->func[vf].pending, func_list) { + clear = 1; + if (group_first == req && + (group->state == MCAST_JOIN_SENT || + group->state == MCAST_LEAVE_SENT)) { + clear = cancel_delayed_work(&group->timeout_work); + pend = !clear; + group->state = MCAST_IDLE; + } + if (clear) { + --group->func[vf].num_pend_reqs; + list_del(&req->group_list); + list_del(&req->func_list); + kfree(req); + atomic_dec(&group->refcount); + } + } + + if (!pend && (!list_empty(&group->func[vf].pending) || group->func[vf].num_pend_reqs)) { + mcg_warn_group(group, "DRIVER BUG: list_empty %d, num_pend_reqs %d\n", + list_empty(&group->func[vf].pending), group->func[vf].num_pend_reqs); + } +} + +static int push_deleteing_req(struct mcast_group *group, int slave) +{ + struct mcast_req *req; + struct mcast_req *pend_req; + + if (!group->func[slave].join_state) + return 0; + + req = kzalloc(sizeof *req, GFP_KERNEL); + if (!req) { + mcg_warn_group(group, "failed allocation - may leave stall groups\n"); + return -ENOMEM; + } + + if (!list_empty(&group->func[slave].pending)) { + pend_req = list_entry(group->func[slave].pending.prev, struct mcast_req, group_list); + if (pend_req->clean) { + kfree(req); + return 0; + } + } + + req->clean = 1; + req->func = slave; + req->group = group; + ++group->func[slave].num_pend_reqs; + build_leave_mad(req); + queue_req(req); + return 0; +} + +void clean_vf_mcast(struct mlx4_ib_demux_ctx *ctx, int slave) +{ + struct mcast_group *group; + struct rb_node *p; + + mutex_lock(&ctx->mcg_table_lock); + for (p = rb_first(&ctx->mcg_table); p; p = rb_next(p)) { + group = rb_entry(p, struct mcast_group, node); + mutex_lock(&group->lock); + if (atomic_read(&group->refcount)) { + /* clear pending requests of this VF */ + clear_pending_reqs(group, slave); + push_deleteing_req(group, slave); + } + mutex_unlock(&group->lock); + } + mutex_unlock(&ctx->mcg_table_lock); +} + + +int mlx4_ib_mcg_init(void) +{ + clean_wq = create_singlethread_workqueue("mlx4_ib_mcg"); + if (!clean_wq) + return -ENOMEM; + + return 0; +} + +void mlx4_ib_mcg_destroy(void) +{ + destroy_workqueue(clean_wq); +} Property changes on: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/mcg.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_ib.h =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_ib.h (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/mlx4_ib.h (revision 273864) @@ -1,409 +1,803 @@ /* * Copyright (c) 2006, 2007 Cisco Systems. All rights reserved. * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef MLX4_IB_H #define MLX4_IB_H #include #include #include +#include +#include #include #include +#include +#include #include #include +#include - #define MLX4_IB_DRV_NAME "mlx4_ib" -#ifdef CONFIG_MLX4_DEBUG -extern int mlx4_ib_debug_level; +#ifdef pr_fmt +#undef pr_fmt +#endif +#define pr_fmt(fmt) "<" MLX4_IB_DRV_NAME "> %s: " fmt, __func__ -#define mlx4_ib_dbg(format, arg...) \ - do { \ - if (mlx4_ib_debug_level) \ - printk(KERN_DEBUG "<" MLX4_IB_DRV_NAME "> %s: " format "\n",\ - __func__, ## arg); \ - } while (0) +#define mlx4_ib_warn(ibdev, format, arg...) \ + dev_warn((ibdev)->dma_device, MLX4_IB_DRV_NAME ": " format, ## arg) -#else /* CONFIG_MLX4_DEBUG */ +#define mlx4_ib_info(ibdev, format, arg...) \ + dev_info((ibdev)->dma_device, MLX4_IB_DRV_NAME ": " format, ## arg) -#define mlx4_ib_dbg(format, arg...) do {} while (0) - -#endif /* CONFIG_MLX4_DEBUG */ - enum { - MLX4_IB_SQ_MIN_WQE_SHIFT = 6 + MLX4_IB_SQ_MIN_WQE_SHIFT = 6, + MLX4_IB_MAX_HEADROOM = 2048 }; -#define MLX4_IB_SQ_HEADROOM(shift) ((2048 >> (shift)) + 1) -#define MLX4_IB_SQ_MAX_SPARE (MLX4_IB_SQ_HEADROOM(MLX4_IB_SQ_MIN_WQE_SHIFT)) +#define MLX4_IB_SQ_HEADROOM(shift) ((MLX4_IB_MAX_HEADROOM >> (shift)) + 1) +#define MLX4_IB_SQ_MAX_SPARE (MLX4_IB_SQ_HEADROOM(MLX4_IB_SQ_MIN_WQE_SHIFT)) +/*module param to indicate if SM assigns the alias_GUID*/ +extern int mlx4_ib_sm_guid_assign; +#ifdef __linux__ +extern struct proc_dir_entry *mlx4_mrs_dir_entry; +#endif + +#define MLX4_IB_UC_STEER_QPN_ALIGN 1 +#define MLX4_IB_UC_MAX_NUM_QPS (256 * 1024) + + +#define MLX4_IB_MMAP_CMD_MASK 0xFF +#define MLX4_IB_MMAP_CMD_BITS 8 + struct mlx4_ib_ucontext { struct ib_ucontext ibucontext; struct mlx4_uar uar; struct list_head db_page_list; struct mutex db_page_mutex; }; struct mlx4_ib_pd { struct ib_pd ibpd; u32 pdn; }; struct mlx4_ib_xrcd { - struct ib_xrcd ibxrcd; - u32 xrcdn; - struct ib_pd *pd; - struct ib_cq *cq; + struct ib_xrcd ibxrcd; + u32 xrcdn; + struct ib_pd *pd; + struct ib_cq *cq; }; struct mlx4_ib_cq_buf { struct mlx4_buf buf; struct mlx4_mtt mtt; + int entry_size; }; struct mlx4_ib_cq_resize { struct mlx4_ib_cq_buf buf; int cqe; }; +struct mlx4_shared_mr_info { + int mr_id; + struct ib_umem *umem; +}; + struct mlx4_ib_cq { struct ib_cq ibcq; struct mlx4_cq mcq; struct mlx4_ib_cq_buf buf; struct mlx4_ib_cq_resize *resize_buf; struct mlx4_db db; spinlock_t lock; struct mutex resize_mutex; struct ib_umem *umem; struct ib_umem *resize_umem; }; struct mlx4_ib_mr { struct ib_mr ibmr; struct mlx4_mr mmr; struct ib_umem *umem; + struct mlx4_shared_mr_info *smr_info; }; struct mlx4_ib_fast_reg_page_list { struct ib_fast_reg_page_list ibfrpl; __be64 *mapped_page_list; dma_addr_t map; }; struct mlx4_ib_fmr { struct ib_fmr ibfmr; struct mlx4_fmr mfmr; }; struct mlx4_ib_wq { u64 *wrid; spinlock_t lock; int wqe_cnt; int max_post; int max_gs; int offset; int wqe_shift; unsigned head; unsigned tail; }; enum mlx4_ib_qp_flags { - MLX4_IB_QP_LSO = 1 << 0, - MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK = 1 << 1, - MLX4_IB_XRC_RCV = 1 << 2, + MLX4_IB_QP_LSO = IB_QP_CREATE_IPOIB_UD_LSO, + MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK, + MLX4_IB_QP_NETIF = IB_QP_CREATE_NETIF_QP, + MLX4_IB_SRIOV_TUNNEL_QP = 1 << 30, + MLX4_IB_SRIOV_SQP = 1 << 31, }; -struct gid_entry { +struct mlx4_ib_gid_entry { struct list_head list; union ib_gid gid; int added; u8 port; }; +enum mlx4_ib_mmap_cmd { + MLX4_IB_MMAP_UAR_PAGE = 0, + MLX4_IB_MMAP_BLUE_FLAME_PAGE = 1, + MLX4_IB_MMAP_GET_CONTIGUOUS_PAGES = 2, +}; + +enum mlx4_ib_qp_type { + /* + * IB_QPT_SMI and IB_QPT_GSI have to be the first two entries + * here (and in that order) since the MAD layer uses them as + * indices into a 2-entry table. + */ + MLX4_IB_QPT_SMI = IB_QPT_SMI, + MLX4_IB_QPT_GSI = IB_QPT_GSI, + + MLX4_IB_QPT_RC = IB_QPT_RC, + MLX4_IB_QPT_UC = IB_QPT_UC, + MLX4_IB_QPT_UD = IB_QPT_UD, + MLX4_IB_QPT_RAW_IPV6 = IB_QPT_RAW_IPV6, + MLX4_IB_QPT_RAW_ETHERTYPE = IB_QPT_RAW_ETHERTYPE, + MLX4_IB_QPT_RAW_PACKET = IB_QPT_RAW_PACKET, + MLX4_IB_QPT_XRC_INI = IB_QPT_XRC_INI, + MLX4_IB_QPT_XRC_TGT = IB_QPT_XRC_TGT, + + MLX4_IB_QPT_PROXY_SMI_OWNER = 1 << 16, + MLX4_IB_QPT_PROXY_SMI = 1 << 17, + MLX4_IB_QPT_PROXY_GSI = 1 << 18, + MLX4_IB_QPT_TUN_SMI_OWNER = 1 << 19, + MLX4_IB_QPT_TUN_SMI = 1 << 20, + MLX4_IB_QPT_TUN_GSI = 1 << 21, +}; + +#define MLX4_IB_QPT_ANY_SRIOV (MLX4_IB_QPT_PROXY_SMI_OWNER | \ + MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER | \ + MLX4_IB_QPT_TUN_SMI | MLX4_IB_QPT_TUN_GSI) + +enum mlx4_ib_mad_ifc_flags { + MLX4_MAD_IFC_IGNORE_MKEY = 1, + MLX4_MAD_IFC_IGNORE_BKEY = 2, + MLX4_MAD_IFC_IGNORE_KEYS = (MLX4_MAD_IFC_IGNORE_MKEY | + MLX4_MAD_IFC_IGNORE_BKEY), + MLX4_MAD_IFC_NET_VIEW = 4, +}; + +enum { + MLX4_NUM_TUNNEL_BUFS = 256, +}; + +struct mlx4_ib_tunnel_header { + struct mlx4_av av; + __be32 remote_qpn; + __be32 qkey; + __be16 vlan; + u8 mac[6]; + __be16 pkey_index; + u8 reserved[6]; +}; + +struct mlx4_ib_buf { + void *addr; + dma_addr_t map; +}; + +struct mlx4_rcv_tunnel_hdr { + __be32 flags_src_qp; /* flags[6:5] is defined for VLANs: + * 0x0 - no vlan was in the packet + * 0x01 - C-VLAN was in the packet */ + u8 g_ml_path; /* gid bit stands for ipv6/4 header in RoCE */ + u8 reserved; + __be16 pkey_index; + __be16 sl_vid; + __be16 slid_mac_47_32; + __be32 mac_31_0; +}; + +struct mlx4_ib_proxy_sqp_hdr { + struct ib_grh grh; + struct mlx4_rcv_tunnel_hdr tun; +} __packed; + +struct mlx4_roce_smac_vlan_info { + u64 smac; + int smac_index; + int smac_port; + u64 candidate_smac; + int candidate_smac_index; + int candidate_smac_port; + u16 vid; + int vlan_index; + int vlan_port; + u16 candidate_vid; + int candidate_vlan_index; + int candidate_vlan_port; + int update_vid; +}; + +struct mlx4_ib_qpg_data { + unsigned long *tss_bitmap; + unsigned long *rss_bitmap; + struct mlx4_ib_qp *qpg_parent; + int tss_qpn_base; + int rss_qpn_base; + u32 tss_child_count; + u32 rss_child_count; + u32 qpg_tss_mask_sz; +}; + struct mlx4_ib_qp { struct ib_qp ibqp; struct mlx4_qp mqp; struct mlx4_buf buf; struct mlx4_db db; struct mlx4_ib_wq rq; u32 doorbell_qpn; __be32 sq_signal_bits; unsigned sq_next_wqe; int sq_max_wqes_per_wr; int sq_spare_wqes; struct mlx4_ib_wq sq; + enum mlx4_ib_qp_type mlx4_ib_qp_type; struct ib_umem *umem; struct mlx4_mtt mtt; int buf_size; struct mutex mutex; - u32 flags; - struct list_head xrc_reg_list; - spinlock_t xrc_reg_list_lock; u16 xrcdn; + u32 flags; u8 port; u8 alt_port; u8 atomic_rd_en; u8 resp_depth; u8 sq_no_prefetch; u8 state; int mlx_type; + enum ib_qpg_type qpg_type; + struct mlx4_ib_qpg_data *qpg_data; struct list_head gid_list; - int max_inline_data; - struct mlx4_bf bf; + struct list_head steering_rules; + struct mlx4_ib_buf *sqp_proxy_rcv; + struct mlx4_roce_smac_vlan_info pri; + struct mlx4_roce_smac_vlan_info alt; + struct list_head rules_list; + int max_inline_data; + struct mlx4_bf bf; }; struct mlx4_ib_srq { struct ib_srq ibsrq; struct mlx4_srq msrq; struct mlx4_buf buf; struct mlx4_db db; u64 *wrid; spinlock_t lock; int head; int tail; u16 wqe_ctr; struct ib_umem *umem; struct mlx4_mtt mtt; struct mutex mutex; }; struct mlx4_ib_ah { struct ib_ah ibah; union mlx4_ext_av av; }; +/****************************************/ +/* alias guid support */ +/****************************************/ +#define NUM_PORT_ALIAS_GUID 2 +#define NUM_ALIAS_GUID_IN_REC 8 +#define NUM_ALIAS_GUID_REC_IN_PORT 16 +#define GUID_REC_SIZE 8 +#define NUM_ALIAS_GUID_PER_PORT 128 +#define MLX4_NOT_SET_GUID (0x00LL) +#define MLX4_GUID_FOR_DELETE_VAL (~(0x00LL)) + +enum mlx4_guid_alias_rec_status { + MLX4_GUID_INFO_STATUS_IDLE, + MLX4_GUID_INFO_STATUS_SET, + MLX4_GUID_INFO_STATUS_PENDING, +}; + +enum mlx4_guid_alias_rec_ownership { + MLX4_GUID_DRIVER_ASSIGN, + MLX4_GUID_SYSADMIN_ASSIGN, + MLX4_GUID_NONE_ASSIGN, /*init state of each record*/ +}; + +enum mlx4_guid_alias_rec_method { + MLX4_GUID_INFO_RECORD_SET = IB_MGMT_METHOD_SET, + MLX4_GUID_INFO_RECORD_DELETE = IB_SA_METHOD_DELETE, +}; + +struct mlx4_sriov_alias_guid_info_rec_det { + u8 all_recs[GUID_REC_SIZE * NUM_ALIAS_GUID_IN_REC]; + ib_sa_comp_mask guid_indexes; /*indicates what from the 8 records are valid*/ + enum mlx4_guid_alias_rec_status status; /*indicates the administraively status of the record.*/ + u8 method; /*set or delete*/ + enum mlx4_guid_alias_rec_ownership ownership; /*indicates who assign that alias_guid record*/ +}; + +struct mlx4_sriov_alias_guid_port_rec_det { + struct mlx4_sriov_alias_guid_info_rec_det all_rec_per_port[NUM_ALIAS_GUID_REC_IN_PORT]; + struct workqueue_struct *wq; + struct delayed_work alias_guid_work; + u8 port; + struct mlx4_sriov_alias_guid *parent; + struct list_head cb_list; +}; + +struct mlx4_sriov_alias_guid { + struct mlx4_sriov_alias_guid_port_rec_det ports_guid[MLX4_MAX_PORTS]; + spinlock_t ag_work_lock; + struct ib_sa_client *sa_client; +}; + +struct mlx4_ib_demux_work { + struct work_struct work; + struct mlx4_ib_dev *dev; + int slave; + int do_init; + u8 port; + +}; + +struct mlx4_ib_tun_tx_buf { + struct mlx4_ib_buf buf; + struct ib_ah *ah; +}; + +struct mlx4_ib_demux_pv_qp { + struct ib_qp *qp; + enum ib_qp_type proxy_qpt; + struct mlx4_ib_buf *ring; + struct mlx4_ib_tun_tx_buf *tx_ring; + spinlock_t tx_lock; + unsigned tx_ix_head; + unsigned tx_ix_tail; +}; + +enum mlx4_ib_demux_pv_state { + DEMUX_PV_STATE_DOWN, + DEMUX_PV_STATE_STARTING, + DEMUX_PV_STATE_ACTIVE, + DEMUX_PV_STATE_DOWNING, +}; + +struct mlx4_ib_demux_pv_ctx { + int port; + int slave; + enum mlx4_ib_demux_pv_state state; + int has_smi; + struct ib_device *ib_dev; + struct ib_cq *cq; + struct ib_pd *pd; + struct ib_mr *mr; + struct work_struct work; + struct workqueue_struct *wq; + struct mlx4_ib_demux_pv_qp qp[2]; +}; + +struct mlx4_ib_demux_ctx { + struct ib_device *ib_dev; + int port; + struct workqueue_struct *wq; + struct workqueue_struct *ud_wq; + spinlock_t ud_lock; + __be64 subnet_prefix; + __be64 guid_cache[128]; + struct mlx4_ib_dev *dev; + /* the following lock protects both mcg_table and mcg_mgid0_list */ + struct mutex mcg_table_lock; + struct rb_root mcg_table; + struct list_head mcg_mgid0_list; + struct workqueue_struct *mcg_wq; + struct mlx4_ib_demux_pv_ctx **tun; + atomic_t tid; + int flushing; /* flushing the work queue */ +}; + +struct mlx4_ib_sriov { + struct mlx4_ib_demux_ctx demux[MLX4_MAX_PORTS]; + struct mlx4_ib_demux_pv_ctx *sqps[MLX4_MAX_PORTS]; + /* when using this spinlock you should use "irq" because + * it may be called from interrupt context.*/ + spinlock_t going_down_lock; + int is_going_down; + + struct mlx4_sriov_alias_guid alias_guid; + + /* CM paravirtualization fields */ + struct list_head cm_list; + spinlock_t id_map_lock; + struct rb_root sl_id_map; + struct idr pv_id_table; +}; + struct mlx4_ib_iboe { spinlock_t lock; struct net_device *netdevs[MLX4_MAX_PORTS]; struct notifier_block nb; union ib_gid gid_table[MLX4_MAX_PORTS][128]; }; +struct pkey_mgt { + u8 virt2phys_pkey[MLX4_MFUNC_MAX][MLX4_MAX_PORTS][MLX4_MAX_PORT_PKEYS]; + u16 phys_pkey_cache[MLX4_MAX_PORTS][MLX4_MAX_PORT_PKEYS]; + struct list_head pkey_port_list[MLX4_MFUNC_MAX]; + struct kobject *device_parent[MLX4_MFUNC_MAX]; +}; + +struct mlx4_ib_iov_sysfs_attr { + void *ctx; + struct kobject *kobj; + unsigned long data; + u32 entry_num; + char name[15]; + struct device_attribute dentry; + struct device *dev; +}; + +struct mlx4_ib_iov_sysfs_attr_ar { + struct mlx4_ib_iov_sysfs_attr dentries[3 * NUM_ALIAS_GUID_PER_PORT + 1]; +}; + +struct mlx4_ib_iov_port { + char name[100]; + u8 num; + struct mlx4_ib_dev *dev; + struct list_head list; + struct mlx4_ib_iov_sysfs_attr_ar *dentr_ar; + struct ib_port_attr attr; + struct kobject *cur_port; + struct kobject *admin_alias_parent; + struct kobject *gids_parent; + struct kobject *pkeys_parent; + struct kobject *mcgs_parent; + struct mlx4_ib_iov_sysfs_attr mcg_dentry; +}; + struct mlx4_ib_dev { struct ib_device ib_dev; struct mlx4_dev *dev; int num_ports; struct mlx4_uar priv_uar; u32 priv_pdn; MLX4_DECLARE_DOORBELL_LOCK(uar_lock); struct ib_mad_agent *send_agent[MLX4_MAX_PORTS][2]; struct ib_ah *sm_ah[MLX4_MAX_PORTS]; spinlock_t sm_lock; + struct mlx4_ib_sriov sriov; struct mutex cap_mask_mutex; - struct mutex xrc_reg_mutex; - int ib_active; + bool ib_active; struct mlx4_ib_iboe iboe; int counters[MLX4_MAX_PORTS]; + int *eq_table; + int eq_added; + struct kobject *iov_parent; + struct kobject *ports_parent; + struct kobject *dev_ports_parent[MLX4_MFUNC_MAX]; + struct mlx4_ib_iov_port iov_ports[MLX4_MAX_PORTS]; + struct pkey_mgt pkeys; + unsigned long *ib_uc_qpns_bitmap; + int steer_qpn_count; + int steer_qpn_base; }; +struct ib_event_work { + struct work_struct work; + struct mlx4_ib_dev *ib_dev; + struct mlx4_eqe ib_eqe; +}; + +struct mlx4_ib_qp_tunnel_init_attr { + struct ib_qp_init_attr init_attr; + int slave; + enum ib_qp_type proxy_qp_type; + u8 port; +}; + static inline struct mlx4_ib_dev *to_mdev(struct ib_device *ibdev) { return container_of(ibdev, struct mlx4_ib_dev, ib_dev); } static inline struct mlx4_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext) { return container_of(ibucontext, struct mlx4_ib_ucontext, ibucontext); } static inline struct mlx4_ib_pd *to_mpd(struct ib_pd *ibpd) { return container_of(ibpd, struct mlx4_ib_pd, ibpd); } static inline struct mlx4_ib_xrcd *to_mxrcd(struct ib_xrcd *ibxrcd) { return container_of(ibxrcd, struct mlx4_ib_xrcd, ibxrcd); } static inline struct mlx4_ib_cq *to_mcq(struct ib_cq *ibcq) { return container_of(ibcq, struct mlx4_ib_cq, ibcq); } static inline struct mlx4_ib_cq *to_mibcq(struct mlx4_cq *mcq) { return container_of(mcq, struct mlx4_ib_cq, mcq); } static inline struct mlx4_ib_mr *to_mmr(struct ib_mr *ibmr) { return container_of(ibmr, struct mlx4_ib_mr, ibmr); } static inline struct mlx4_ib_fast_reg_page_list *to_mfrpl(struct ib_fast_reg_page_list *ibfrpl) { return container_of(ibfrpl, struct mlx4_ib_fast_reg_page_list, ibfrpl); } static inline struct mlx4_ib_fmr *to_mfmr(struct ib_fmr *ibfmr) { return container_of(ibfmr, struct mlx4_ib_fmr, ibfmr); } static inline struct mlx4_ib_qp *to_mqp(struct ib_qp *ibqp) { return container_of(ibqp, struct mlx4_ib_qp, ibqp); } static inline struct mlx4_ib_qp *to_mibqp(struct mlx4_qp *mqp) { return container_of(mqp, struct mlx4_ib_qp, mqp); } static inline struct mlx4_ib_srq *to_msrq(struct ib_srq *ibsrq) { return container_of(ibsrq, struct mlx4_ib_srq, ibsrq); } static inline struct mlx4_ib_srq *to_mibsrq(struct mlx4_srq *msrq) { return container_of(msrq, struct mlx4_ib_srq, msrq); } static inline struct mlx4_ib_ah *to_mah(struct ib_ah *ibah) { return container_of(ibah, struct mlx4_ib_ah, ibah); } +int mlx4_ib_init_sriov(struct mlx4_ib_dev *dev); +void mlx4_ib_close_sriov(struct mlx4_ib_dev *dev); + int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt, struct mlx4_db *db); void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_db *db); struct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc); int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt, struct ib_umem *umem); +int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, + u64 start_va, + int *num_of_mtts); struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int access_flags, - struct ib_udata *udata); + struct ib_udata *udata, int mr_id); int mlx4_ib_dereg_mr(struct ib_mr *mr); struct ib_mr *mlx4_ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len); struct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device *ibdev, int page_list_len); void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list); int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period); int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata); +int mlx4_ib_ignore_overrun_cq(struct ib_cq *ibcq); struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector, struct ib_ucontext *context, struct ib_udata *udata); int mlx4_ib_destroy_cq(struct ib_cq *cq); int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq); void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq); struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr); int mlx4_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr); int mlx4_ib_destroy_ah(struct ib_ah *ah); struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *init_attr, struct ib_udata *udata); -struct ib_srq *mlx4_ib_create_xrc_srq(struct ib_pd *pd, - struct ib_cq *xrc_cq, - struct ib_xrcd *xrcd, - struct ib_srq_init_attr *init_attr, - struct ib_udata *udata); int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); int mlx4_ib_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); int mlx4_ib_destroy_srq(struct ib_srq *srq); void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index); int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, struct ib_recv_wr **bad_wr); struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata); int mlx4_ib_destroy_qp(struct ib_qp *qp); int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, struct ib_send_wr **bad_wr); int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, struct ib_recv_wr **bad_wr); -int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int ignore_mkey, int ignore_bkey, +int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int mad_ifc_flags, int port, struct ib_wc *in_wc, struct ib_grh *in_grh, void *in_mad, void *response_mad); int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, struct ib_wc *in_wc, struct ib_grh *in_grh, struct ib_mad *in_mad, struct ib_mad *out_mad); int mlx4_ib_mad_init(struct mlx4_ib_dev *dev); void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev); struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int mr_access_flags, struct ib_fmr_attr *fmr_attr); int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, int npages, u64 iova); int mlx4_ib_unmap_fmr(struct list_head *fmr_list); int mlx4_ib_fmr_dealloc(struct ib_fmr *fmr); -int mlx4_ib_create_xrc_rcv_qp(struct ib_qp_init_attr *init_attr, - u32 *qp_num); -int mlx4_ib_modify_xrc_rcv_qp(struct ib_xrcd *xrcd, u32 qp_num, - struct ib_qp_attr *attr, int attr_mask); -int mlx4_ib_query_xrc_rcv_qp(struct ib_xrcd *xrcd, u32 qp_num, - struct ib_qp_attr *attr, int attr_mask, - struct ib_qp_init_attr *init_attr); -int mlx4_ib_reg_xrc_rcv_qp(struct ib_xrcd *xrcd, void *context, u32 qp_num); -int mlx4_ib_unreg_xrc_rcv_qp(struct ib_xrcd *xrcd, void *context, u32 qp_num); +int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props, int netw_view); +int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, + u16 *pkey, int netw_view); +int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index, + union ib_gid *gid, int netw_view); int mlx4_ib_resolve_grh(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah_attr, u8 *mac, int *is_mcast, u8 port); +int mlx4_ib_query_if_stat(struct mlx4_ib_dev *dev, u32 counter_index, + union mlx4_counter *counter, u8 clear); + static inline int mlx4_ib_ah_grh_present(struct mlx4_ib_ah *ah) { u8 port = be32_to_cpu(ah->av.ib.port_pd) >> 24 & 3; if (rdma_port_get_link_layer(ah->ibah.device, port) == IB_LINK_LAYER_ETHERNET) return 1; return !!(ah->av.ib.g_slid & 0x80); } +int mlx4_ib_mcg_port_init(struct mlx4_ib_demux_ctx *ctx); +void mlx4_ib_mcg_port_cleanup(struct mlx4_ib_demux_ctx *ctx, int destroy_wq); +void clean_vf_mcast(struct mlx4_ib_demux_ctx *ctx, int slave); +int mlx4_ib_mcg_init(void); +void mlx4_ib_mcg_destroy(void); + +int mlx4_ib_find_real_gid(struct ib_device *ibdev, u8 port, __be64 guid); + +int mlx4_ib_mcg_multiplex_handler(struct ib_device *ibdev, int port, int slave, + struct ib_sa_mad *sa_mad); +int mlx4_ib_mcg_demux_handler(struct ib_device *ibdev, int port, int slave, + struct ib_sa_mad *mad); + int mlx4_ib_add_mc(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, union ib_gid *gid); + +void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u8 port_num, + enum ib_event_type type); + +void mlx4_ib_tunnels_update_work(struct work_struct *work); + +int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port, + enum ib_qp_type qpt, struct ib_wc *wc, + struct ib_grh *grh, struct ib_mad *mad); +int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port, + enum ib_qp_type dest_qpt, u16 pkey_index, u32 remote_qpn, + u32 qkey, struct ib_ah_attr *attr, struct ib_mad *mad); +__be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx); + +int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave, + struct ib_mad *mad, int is_eth); + +int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id, + struct ib_mad *mad); + +void mlx4_ib_cm_paravirt_init(struct mlx4_ib_dev *dev); +void mlx4_ib_cm_paravirt_clean(struct mlx4_ib_dev *dev, int slave_id); + +/* alias guid support */ +void mlx4_ib_init_alias_guid_work(struct mlx4_ib_dev *dev, int port); +int mlx4_ib_init_alias_guid_service(struct mlx4_ib_dev *dev); +void mlx4_ib_destroy_alias_guid_service(struct mlx4_ib_dev *dev); +void mlx4_ib_invalidate_all_guid_record(struct mlx4_ib_dev *dev, int port); + +void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev, + int block_num, + u8 port_num, u8 *p_data); + +void mlx4_ib_update_cache_on_guid_change(struct mlx4_ib_dev *dev, + int block_num, u8 port_num, + u8 *p_data); + +int add_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num, + struct attribute *attr); +void del_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num, + struct attribute *attr); +ib_sa_comp_mask mlx4_ib_get_aguid_comp_mask_from_ix(int index); + +int mlx4_ib_device_register_sysfs(struct mlx4_ib_dev *device) ; + +void mlx4_ib_device_unregister_sysfs(struct mlx4_ib_dev *device); + +__be64 mlx4_ib_gen_node_guid(void); + +int mlx4_ib_steer_qp_alloc(struct mlx4_ib_dev *dev, int count, int *qpn); +void mlx4_ib_steer_qp_free(struct mlx4_ib_dev *dev, u32 qpn, int count); +int mlx4_ib_steer_qp_reg(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp, + int is_attach); #endif /* MLX4_IB_H */ Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/mr.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/mr.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/mr.c (revision 273864) @@ -1,424 +1,753 @@ /* * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include +#include +#include + +#ifdef __linux__ +#include +#include +#endif + #include "mlx4_ib.h" static u32 convert_access(int acc) { return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX4_PERM_ATOMIC : 0) | (acc & IB_ACCESS_REMOTE_WRITE ? MLX4_PERM_REMOTE_WRITE : 0) | (acc & IB_ACCESS_REMOTE_READ ? MLX4_PERM_REMOTE_READ : 0) | (acc & IB_ACCESS_LOCAL_WRITE ? MLX4_PERM_LOCAL_WRITE : 0) | MLX4_PERM_LOCAL_READ; } +#ifdef __linux__ +static ssize_t shared_mr_proc_read(struct file *file, + char __user *buffer, + size_t len, + loff_t *offset) +{ + return -ENOSYS; + +} + +static ssize_t shared_mr_proc_write(struct file *file, + const char __user *buffer, + size_t len, + loff_t *offset) +{ + + return -ENOSYS; +} + +static int shared_mr_mmap(struct file *filep, struct vm_area_struct *vma) +{ + + struct proc_dir_entry *pde = PDE(filep->f_path.dentry->d_inode); + struct mlx4_shared_mr_info *smr_info = + (struct mlx4_shared_mr_info *)pde->data; + + /* Prevent any mapping not on start of area */ + if (vma->vm_pgoff != 0) + return -EINVAL; + + return ib_umem_map_to_vma(smr_info->umem, + vma); + +} + +static const struct file_operations shared_mr_proc_ops = { + .owner = THIS_MODULE, + .read = shared_mr_proc_read, + .write = shared_mr_proc_write, + .mmap = shared_mr_mmap +}; + +static mode_t convert_shared_access(int acc) +{ + + return (acc & IB_ACCESS_SHARED_MR_USER_READ ? S_IRUSR : 0) | + (acc & IB_ACCESS_SHARED_MR_USER_WRITE ? S_IWUSR : 0) | + (acc & IB_ACCESS_SHARED_MR_GROUP_READ ? S_IRGRP : 0) | + (acc & IB_ACCESS_SHARED_MR_GROUP_WRITE ? S_IWGRP : 0) | + (acc & IB_ACCESS_SHARED_MR_OTHER_READ ? S_IROTH : 0) | + (acc & IB_ACCESS_SHARED_MR_OTHER_WRITE ? S_IWOTH : 0); + +} +#endif struct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc) { struct mlx4_ib_mr *mr; int err; - mr = kmalloc(sizeof *mr, GFP_KERNEL); + mr = kzalloc(sizeof *mr, GFP_KERNEL); if (!mr) return ERR_PTR(-ENOMEM); err = mlx4_mr_alloc(to_mdev(pd->device)->dev, to_mpd(pd)->pdn, 0, ~0ull, convert_access(acc), 0, 0, &mr->mmr); if (err) goto err_free; err = mlx4_mr_enable(to_mdev(pd->device)->dev, &mr->mmr); if (err) goto err_mr; mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; mr->umem = NULL; return &mr->ibmr; err_mr: mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr); err_free: kfree(mr); return ERR_PTR(err); } +static int mlx4_ib_umem_write_mtt_block(struct mlx4_ib_dev *dev, + struct mlx4_mtt *mtt, + u64 mtt_size, + u64 mtt_shift, + u64 len, + u64 cur_start_addr, + u64 *pages, + int *start_index, + int *npages) +{ + int k; + int err = 0; + u64 mtt_entries; + u64 cur_end_addr = cur_start_addr + len; + u64 cur_end_addr_aligned = 0; + + len += (cur_start_addr & (mtt_size-1ULL)); + cur_end_addr_aligned = round_up(cur_end_addr, mtt_size); + len += (cur_end_addr_aligned - cur_end_addr); + if (len & (mtt_size-1ULL)) { + WARN(1 , + "write_block: len %llx is not aligned to mtt_size %llx\n", + (long long)len, (long long)mtt_size); + return -EINVAL; + } + + + mtt_entries = (len >> mtt_shift); + + /* Align the MTT start address to + the mtt_size. + Required to handle cases when the MR + starts in the middle of an MTT record. + Was not required in old code since + the physical addresses provided by + the dma subsystem were page aligned, + which was also the MTT size. + */ + cur_start_addr = round_down(cur_start_addr, mtt_size); + /* A new block is started ...*/ + for (k = 0; k < mtt_entries; ++k) { + pages[*npages] = cur_start_addr + (mtt_size * k); + (*npages)++; + /* + * Be friendly to mlx4_write_mtt() and + * pass it chunks of appropriate size. + */ + if (*npages == PAGE_SIZE / sizeof(u64)) { + err = mlx4_write_mtt(dev->dev, + mtt, *start_index, + *npages, pages); + if (err) + return err; + + (*start_index) += *npages; + *npages = 0; + } + } + + return 0; +} + int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt, struct ib_umem *umem) { u64 *pages; struct ib_umem_chunk *chunk; - int i, j, k; - int n; - int len; + int j; + u64 len = 0; int err = 0; + u64 mtt_size; + u64 cur_start_addr = 0; + u64 mtt_shift; + int start_index = 0; + int npages = 0; pages = (u64 *) __get_free_page(GFP_KERNEL); if (!pages) return -ENOMEM; - i = n = 0; + mtt_shift = mtt->page_shift; + mtt_size = 1ULL << mtt_shift; list_for_each_entry(chunk, &umem->chunk_list, list) for (j = 0; j < chunk->nmap; ++j) { - len = sg_dma_len(&chunk->page_list[j]) >> mtt->page_shift; - for (k = 0; k < len; ++k) { - pages[i++] = sg_dma_address(&chunk->page_list[j]) + - umem->page_size * k; - /* - * Be friendly to mlx4_write_mtt() and - * pass it chunks of appropriate size. - */ - if (i == PAGE_SIZE / sizeof (u64)) { - err = mlx4_write_mtt(dev->dev, mtt, n, - i, pages); - if (err) - goto out; - n += i; - i = 0; - } + if (cur_start_addr + len == + sg_dma_address(&chunk->page_list[j])) { + /* still the same block */ + len += sg_dma_len(&chunk->page_list[j]); + continue; } + /* A new block is started ...*/ + /* If len is malaligned, write an extra mtt entry to + cover the misaligned area (round up the division) + */ + err = mlx4_ib_umem_write_mtt_block(dev, + mtt, mtt_size, mtt_shift, + len, cur_start_addr, + pages, + &start_index, + &npages); + if (err) + goto out; + + cur_start_addr = + sg_dma_address(&chunk->page_list[j]); + len = sg_dma_len(&chunk->page_list[j]); } - if (i) - err = mlx4_write_mtt(dev->dev, mtt, n, i, pages); + /* Handle the last block */ + if (len > 0) { + /* If len is malaligned, write an extra mtt entry to cover + the misaligned area (round up the division) + */ + err = mlx4_ib_umem_write_mtt_block(dev, + mtt, mtt_size, mtt_shift, + len, cur_start_addr, + pages, + &start_index, + &npages); + if (err) + goto out; + } + + if (npages) + err = mlx4_write_mtt(dev->dev, mtt, start_index, npages, pages); + out: free_page((unsigned long) pages); return err; } -static int handle_hugetlb_user_mr(struct ib_pd *pd, struct mlx4_ib_mr *mr, - u64 start, u64 virt_addr, int access_flags) +static inline u64 alignment_of(u64 ptr) { -#if defined(CONFIG_HUGETLB_PAGE) && !defined(__powerpc__) && !defined(__ia64__) - struct mlx4_ib_dev *dev = to_mdev(pd->device); + return ilog2(ptr & (~(ptr-1))); +} + +static int mlx4_ib_umem_calc_block_mtt(u64 next_block_start, + u64 current_block_end, + u64 block_shift) +{ + /* Check whether the alignment of the new block + is aligned as well as the previous block. + Block address must start with zeros till size of entity_size. + */ + if ((next_block_start & ((1ULL << block_shift) - 1ULL)) != 0) + /* It is not as well aligned as the + previous block-reduce the mtt size + accordingly. + Here we take the last right bit + which is 1. + */ + block_shift = alignment_of(next_block_start); + + /* Check whether the alignment of the + end of previous block - is it aligned + as well as the start of the block + */ + if (((current_block_end) & ((1ULL << block_shift) - 1ULL)) != 0) + /* It is not as well aligned as + the start of the block - reduce the + mtt size accordingly. + */ + block_shift = alignment_of(current_block_end); + + return block_shift; +} + +/* Calculate optimal mtt size based on contiguous pages. +* Function will return also the number of pages that are not aligned to the + calculated mtt_size to be added to total number + of pages. For that we should check the first chunk length & last chunk + length and if not aligned to mtt_size we should increment + the non_aligned_pages number. + All chunks in the middle already handled as part of mtt shift calculation + for both their start & end addresses. +*/ +int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, + u64 start_va, + int *num_of_mtts) +{ struct ib_umem_chunk *chunk; - unsigned dsize; - dma_addr_t daddr; - unsigned cur_size = 0; - dma_addr_t uninitialized_var(cur_addr); - int n; - struct ib_umem *umem = mr->umem; - u64 *arr; - int err = 0; - int i; - int j = 0; - int off = start & (HPAGE_SIZE - 1); + int j; + u64 block_shift = MLX4_MAX_MTT_SHIFT; + u64 current_block_len = 0; + u64 current_block_start = 0; + u64 misalignment_bits; + u64 first_block_start = 0; + u64 last_block_end = 0; + u64 total_len = 0; + u64 last_block_aligned_end = 0; + u64 min_shift = ilog2(umem->page_size); - n = DIV_ROUND_UP(off + umem->length, HPAGE_SIZE); - arr = kmalloc(n * sizeof *arr, GFP_KERNEL); - if (!arr) - return -ENOMEM; + list_for_each_entry(chunk, &umem->chunk_list, list) { + /* Initialization - save the first chunk start as + the current_block_start - block means contiguous pages. + */ + if (current_block_len == 0 && current_block_start == 0) { + first_block_start = current_block_start = + sg_dma_address(&chunk->page_list[0]); + /* Find the bits that are different between + the physical address and the virtual + address for the start of the MR. + */ + /* umem_get aligned the start_va to a page + boundry. Therefore, we need to align the + start va to the same boundry */ + /* misalignment_bits is needed to handle the + case of a single memory region. In this + case, the rest of the logic will not reduce + the block size. If we use a block size + which is bigger than the alignment of the + misalignment bits, we might use the virtual + page number instead of the physical page + number, resulting in access to the wrong + data. */ + misalignment_bits = + (start_va & (~(((u64)(umem->page_size))-1ULL))) + ^ current_block_start; + block_shift = min(alignment_of(misalignment_bits) + , block_shift); + } - list_for_each_entry(chunk, &umem->chunk_list, list) - for (i = 0; i < chunk->nmap; ++i) { - daddr = sg_dma_address(&chunk->page_list[i]); - dsize = sg_dma_len(&chunk->page_list[i]); - if (!cur_size) { - cur_addr = daddr; - cur_size = dsize; - } else if (cur_addr + cur_size != daddr) { - err = -EINVAL; - goto out; - } else - cur_size += dsize; + /* Go over the scatter entries in the current chunk, check + if they continue the previous scatter entry. + */ + for (j = 0; j < chunk->nmap; ++j) { + u64 next_block_start = + sg_dma_address(&chunk->page_list[j]); + u64 current_block_end = current_block_start + + current_block_len; + /* If we have a split (non-contig.) between two block*/ + if (current_block_end != next_block_start) { + block_shift = mlx4_ib_umem_calc_block_mtt( + next_block_start, + current_block_end, + block_shift); - if (cur_size > HPAGE_SIZE) { - err = -EINVAL; - goto out; - } else if (cur_size == HPAGE_SIZE) { - cur_size = 0; - arr[j++] = cur_addr; + /* If we reached the minimum shift for 4k + page we stop the loop. + */ + if (block_shift <= min_shift) + goto end; + + /* If not saved yet we are in first block - + we save the length of first block to + calculate the non_aligned_pages number at + * the end. + */ + total_len += current_block_len; + + /* Start a new block */ + current_block_start = next_block_start; + current_block_len = + sg_dma_len(&chunk->page_list[j]); + continue; } + /* The scatter entry is another part of + the current block, increase the block size + * An entry in the scatter can be larger than + 4k (page) as of dma mapping + which merge some blocks together. + */ + current_block_len += + sg_dma_len(&chunk->page_list[j]); } + } - if (cur_size) { - arr[j++] = cur_addr; + /* Account for the last block in the total len */ + total_len += current_block_len; + /* Add to the first block the misalignment that it suffers from.*/ + total_len += (first_block_start & ((1ULL<> block_shift; +end: + if (block_shift < min_shift) { + /* If shift is less than the min we set a WARN and + return the min shift. + */ + WARN(1, + "mlx4_ib_umem_calc_optimal_mtt_size - unexpected shift %lld\n", + (long long)block_shift); + + block_shift = min_shift; } + return block_shift; +} - err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, virt_addr, umem->length, - convert_access(access_flags), n, HPAGE_SHIFT, &mr->mmr); - if (err) - goto out; +#ifdef __linux__ +static int prepare_shared_mr(struct mlx4_ib_mr *mr, int access_flags, int mr_id) +{ + struct proc_dir_entry *mr_proc_entry; + mode_t mode = S_IFREG; + char name_buff[16]; - err = mlx4_write_mtt(dev->dev, &mr->mmr.mtt, 0, n, arr); + mode |= convert_shared_access(access_flags); + sprintf(name_buff, "%X", mr_id); + mr->smr_info = kmalloc(sizeof(struct mlx4_shared_mr_info), GFP_KERNEL); + mr->smr_info->mr_id = mr_id; + mr->smr_info->umem = mr->umem; -out: - kfree(arr); - return err; -#else - return -ENOSYS; -#endif + mr_proc_entry = proc_create_data(name_buff, mode, + mlx4_mrs_dir_entry, + &shared_mr_proc_ops, + mr->smr_info); + + if (!mr_proc_entry) { + pr_err("prepare_shared_mr failed via proc\n"); + kfree(mr->smr_info); + return -ENODEV; + } + + current_uid_gid(&(mr_proc_entry->uid), &(mr_proc_entry->gid)); + mr_proc_entry->size = mr->umem->length; + return 0; + } +static int is_shared_mr(int access_flags) +{ + /* We should check whether IB_ACCESS_SHARED_MR_USER_READ or + other shared bits were turned on. + */ + return !!(access_flags & (IB_ACCESS_SHARED_MR_USER_READ | + IB_ACCESS_SHARED_MR_USER_WRITE | + IB_ACCESS_SHARED_MR_GROUP_READ | + IB_ACCESS_SHARED_MR_GROUP_WRITE | + IB_ACCESS_SHARED_MR_OTHER_READ | + IB_ACCESS_SHARED_MR_OTHER_WRITE)); +} +#endif + struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int access_flags, - struct ib_udata *udata) + struct ib_udata *udata, + int mr_id) { struct mlx4_ib_dev *dev = to_mdev(pd->device); struct mlx4_ib_mr *mr; int shift; int err; int n; - mr = kmalloc(sizeof *mr, GFP_KERNEL); + mr = kzalloc(sizeof *mr, GFP_KERNEL); if (!mr) return ERR_PTR(-ENOMEM); mr->umem = ib_umem_get(pd->uobject->context, start, length, - access_flags, 0); + access_flags, 0); if (IS_ERR(mr->umem)) { err = PTR_ERR(mr->umem); goto err_free; } - if (!mr->umem->hugetlb || - handle_hugetlb_user_mr(pd, mr, start, virt_addr, access_flags)) { - n = ib_umem_page_count(mr->umem); - shift = ilog2(mr->umem->page_size); + n = ib_umem_page_count(mr->umem); + shift = mlx4_ib_umem_calc_optimal_mtt_size(mr->umem, start, + &n); + err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, virt_addr, length, + convert_access(access_flags), n, shift, &mr->mmr); + if (err) + goto err_umem; - err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, virt_addr, length, - convert_access(access_flags), n, shift, &mr->mmr); - if (err) - goto err_umem; + err = mlx4_ib_umem_write_mtt(dev, &mr->mmr.mtt, mr->umem); + if (err) + goto err_mr; - err = mlx4_ib_umem_write_mtt(dev, &mr->mmr.mtt, mr->umem); - if (err) - goto err_mr; - } - err = mlx4_mr_enable(dev->dev, &mr->mmr); if (err) goto err_mr; mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; +#ifdef __linux__ + /* Check whether MR should be shared */ + if (is_shared_mr(access_flags)) { + /* start address and length must be aligned to page size in order + to map a full page and preventing leakage of data */ + if (mr->umem->offset || (length & ~PAGE_MASK)) { + err = -EINVAL; + goto err_mr; + } + err = prepare_shared_mr(mr, access_flags, mr_id); + if (err) + goto err_mr; + } +#endif return &mr->ibmr; err_mr: mlx4_mr_free(to_mdev(pd->device)->dev, &mr->mmr); err_umem: ib_umem_release(mr->umem); err_free: kfree(mr); return ERR_PTR(err); } + int mlx4_ib_dereg_mr(struct ib_mr *ibmr) { struct mlx4_ib_mr *mr = to_mmr(ibmr); mlx4_mr_free(to_mdev(ibmr->device)->dev, &mr->mmr); + if (mr->smr_info) { + /* When master/parent shared mr is dereged there is + no ability to share this mr any more - its mr_id will be + returned to the kernel as part of ib_uverbs_dereg_mr + and may be allocated again as part of other reg_mr. + */ + char name_buff[16]; + + sprintf(name_buff, "%X", mr->smr_info->mr_id); + /* Remove proc entry is checking internally that no operation + was strated on that proc fs file and if in the middle + current process will wait till end of operation. + That's why no sync mechanism is needed when we release + below the shared umem. + */ +#ifdef __linux__ + remove_proc_entry(name_buff, mlx4_mrs_dir_entry); + kfree(mr->smr_info); +#endif + } + if (mr->umem) ib_umem_release(mr->umem); + kfree(mr); return 0; } struct ib_mr *mlx4_ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len) { struct mlx4_ib_dev *dev = to_mdev(pd->device); struct mlx4_ib_mr *mr; int err; - mr = kmalloc(sizeof *mr, GFP_KERNEL); + mr = kzalloc(sizeof *mr, GFP_KERNEL); if (!mr) return ERR_PTR(-ENOMEM); err = mlx4_mr_alloc(dev->dev, to_mpd(pd)->pdn, 0, 0, 0, max_page_list_len, 0, &mr->mmr); if (err) goto err_free; err = mlx4_mr_enable(dev->dev, &mr->mmr); if (err) goto err_mr; mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; mr->umem = NULL; return &mr->ibmr; err_mr: mlx4_mr_free(dev->dev, &mr->mmr); err_free: kfree(mr); return ERR_PTR(err); } struct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device *ibdev, int page_list_len) { struct mlx4_ib_dev *dev = to_mdev(ibdev); struct mlx4_ib_fast_reg_page_list *mfrpl; int size = page_list_len * sizeof (u64); - if (page_list_len > MAX_FAST_REG_PAGES) + if (page_list_len > MLX4_MAX_FAST_REG_PAGES) return ERR_PTR(-EINVAL); mfrpl = kmalloc(sizeof *mfrpl, GFP_KERNEL); if (!mfrpl) return ERR_PTR(-ENOMEM); mfrpl->ibfrpl.page_list = kmalloc(size, GFP_KERNEL); if (!mfrpl->ibfrpl.page_list) goto err_free; mfrpl->mapped_page_list = dma_alloc_coherent(&dev->dev->pdev->dev, size, &mfrpl->map, GFP_KERNEL); if (!mfrpl->mapped_page_list) goto err_free; WARN_ON(mfrpl->map & 0x3f); return &mfrpl->ibfrpl; err_free: kfree(mfrpl->ibfrpl.page_list); kfree(mfrpl); return ERR_PTR(-ENOMEM); } void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list) { struct mlx4_ib_dev *dev = to_mdev(page_list->device); struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(page_list); int size = page_list->max_page_list_len * sizeof (u64); dma_free_coherent(&dev->dev->pdev->dev, size, mfrpl->mapped_page_list, mfrpl->map); kfree(mfrpl->ibfrpl.page_list); kfree(mfrpl); } struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int acc, struct ib_fmr_attr *fmr_attr) { struct mlx4_ib_dev *dev = to_mdev(pd->device); struct mlx4_ib_fmr *fmr; int err = -ENOMEM; fmr = kmalloc(sizeof *fmr, GFP_KERNEL); if (!fmr) return ERR_PTR(-ENOMEM); err = mlx4_fmr_alloc(dev->dev, to_mpd(pd)->pdn, convert_access(acc), fmr_attr->max_pages, fmr_attr->max_maps, fmr_attr->page_shift, &fmr->mfmr); if (err) goto err_free; err = mlx4_fmr_enable(to_mdev(pd->device)->dev, &fmr->mfmr); if (err) goto err_mr; fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mfmr.mr.key; return &fmr->ibfmr; err_mr: mlx4_mr_free(to_mdev(pd->device)->dev, &fmr->mfmr.mr); err_free: kfree(fmr); return ERR_PTR(err); } int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, int npages, u64 iova) { struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); struct mlx4_ib_dev *dev = to_mdev(ifmr->ibfmr.device); return mlx4_map_phys_fmr(dev->dev, &ifmr->mfmr, page_list, npages, iova, &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey); } int mlx4_ib_unmap_fmr(struct list_head *fmr_list) { struct ib_fmr *ibfmr; int err; struct mlx4_dev *mdev = NULL; list_for_each_entry(ibfmr, fmr_list, list) { if (mdev && to_mdev(ibfmr->device)->dev != mdev) return -EINVAL; mdev = to_mdev(ibfmr->device)->dev; } if (!mdev) return 0; list_for_each_entry(ibfmr, fmr_list, list) { struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); mlx4_fmr_unmap(mdev, &ifmr->mfmr, &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey); } /* * Make sure all MPT status updates are visible before issuing * SYNC_TPT firmware command. */ wmb(); err = mlx4_SYNC_TPT(mdev); if (err) - printk(KERN_WARNING "mlx4_ib: SYNC_TPT error %d when " + pr_warn("SYNC_TPT error %d when " "unmapping FMRs\n", err); return 0; } int mlx4_ib_fmr_dealloc(struct ib_fmr *ibfmr) { struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr); struct mlx4_ib_dev *dev = to_mdev(ibfmr->device); int err; err = mlx4_fmr_free(dev->dev, &ifmr->mfmr); if (!err) kfree(ifmr); return err; } Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/qp.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/qp.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/qp.c (revision 273864) @@ -1,2770 +1,3589 @@ /* * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include +#include #include +#include #include #include #include +#include #include +#include #include +#ifndef __linux__ +#define asm __asm +#endif + #include "mlx4_ib.h" #include "user.h" enum { MLX4_IB_ACK_REQ_FREQ = 8, }; enum { MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83, MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f, MLX4_IB_LINK_TYPE_IB = 0, - MLX4_IB_LINK_TYPE_ETH = 1, + MLX4_IB_LINK_TYPE_ETH = 1 }; enum { /* - * Largest possible UD header: send with GRH and immediate data. - * 4 bytes added to accommodate for eth header instead of lrh + * Largest possible UD header: send with GRH and immediate + * data plus 18 bytes for an Ethernet header with VLAN/802.1Q + * tag. (LRH would only use 8 bytes, so Ethernet is the + * biggest case) */ - MLX4_IB_UD_HEADER_SIZE = 76, - MLX4_IB_MAX_RAW_ETY_HDR_SIZE = 12 + MLX4_IB_UD_HEADER_SIZE = 82, + MLX4_IB_LSO_HEADER_SPARE = 128, }; enum { - MLX4_IBOE_ETHERTYPE = 0x8915 + MLX4_IB_IBOE_ETHERTYPE = 0x8915 }; -struct mlx4_ib_xrc_reg_entry { - struct list_head list; - void *context; -}; - struct mlx4_ib_sqp { struct mlx4_ib_qp qp; int pkey_index; u32 qkey; u32 send_psn; struct ib_ud_header ud_header; u8 header_buf[MLX4_IB_UD_HEADER_SIZE]; }; enum { - MLX4_IB_MIN_SQ_STRIDE = 6 + MLX4_IB_MIN_SQ_STRIDE = 6, + MLX4_IB_CACHE_LINE_SIZE = 64, }; +enum { + MLX4_RAW_QP_MTU = 7, + MLX4_RAW_QP_MSGMAX = 31, +}; + static const __be32 mlx4_ib_opcode[] = { [IB_WR_SEND] = cpu_to_be32(MLX4_OPCODE_SEND), [IB_WR_LSO] = cpu_to_be32(MLX4_OPCODE_LSO), [IB_WR_SEND_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_SEND_IMM), [IB_WR_RDMA_WRITE] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE), [IB_WR_RDMA_WRITE_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM), [IB_WR_RDMA_READ] = cpu_to_be32(MLX4_OPCODE_RDMA_READ), [IB_WR_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_ATOMIC_CS), [IB_WR_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_ATOMIC_FA), [IB_WR_SEND_WITH_INV] = cpu_to_be32(MLX4_OPCODE_SEND_INVAL), [IB_WR_LOCAL_INV] = cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL), [IB_WR_FAST_REG_MR] = cpu_to_be32(MLX4_OPCODE_FMR), [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS), [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA), }; #ifndef wc_wmb #if defined(__i386__) - #define wc_wmb() __asm volatile("lock; addl $0,0(%%esp) " ::: "memory") + #define wc_wmb() asm volatile("lock; addl $0,0(%%esp) " ::: "memory") #elif defined(__x86_64__) - #define wc_wmb() __asm volatile("sfence" ::: "memory") + #define wc_wmb() asm volatile("sfence" ::: "memory") #elif defined(__ia64__) - #define wc_wmb() __asm volatile("fwb" ::: "memory") + #define wc_wmb() asm volatile("fwb" ::: "memory") #else #define wc_wmb() wmb() #endif #endif - static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp) { return container_of(mqp, struct mlx4_ib_sqp, qp); } +static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) +{ + if (!mlx4_is_master(dev->dev)) + return 0; + + return qp->mqp.qpn >= dev->dev->phys_caps.base_tunnel_sqpn && + qp->mqp.qpn < dev->dev->phys_caps.base_tunnel_sqpn + + 8 * MLX4_MFUNC_MAX; +} + static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) { - return qp->mqp.qpn >= dev->dev->caps.sqp_start && - qp->mqp.qpn <= dev->dev->caps.sqp_start + 3; + int proxy_sqp = 0; + int real_sqp = 0; + int i; + /* PPF or Native -- real SQP */ + real_sqp = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) && + qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn && + qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 3); + if (real_sqp) + return 1; + /* VF or PF -- proxy SQP */ + if (mlx4_is_mfunc(dev->dev)) { + for (i = 0; i < dev->dev->caps.num_ports; i++) { + if (qp->mqp.qpn == dev->dev->caps.qp0_proxy[i] || + qp->mqp.qpn == dev->dev->caps.qp1_proxy[i]) { + proxy_sqp = 1; + break; + } + } + } + return proxy_sqp; } +/* used for INIT/CLOSE port logic */ static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) { - return qp->mqp.qpn >= dev->dev->caps.sqp_start && - qp->mqp.qpn <= dev->dev->caps.sqp_start + 1; + int proxy_qp0 = 0; + int real_qp0 = 0; + int i; + /* PPF or Native -- real QP0 */ + real_qp0 = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) && + qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn && + qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 1); + if (real_qp0) + return 1; + /* VF or PF -- proxy QP0 */ + if (mlx4_is_mfunc(dev->dev)) { + for (i = 0; i < dev->dev->caps.num_ports; i++) { + if (qp->mqp.qpn == dev->dev->caps.qp0_proxy[i]) { + proxy_qp0 = 1; + break; + } + } + } + return proxy_qp0; } static void *get_wqe(struct mlx4_ib_qp *qp, int offset) { return mlx4_buf_offset(&qp->buf, offset); } static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n) { return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift)); } static void *get_send_wqe(struct mlx4_ib_qp *qp, int n) { return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift)); } /* * Stamp a SQ WQE so that it is invalid if prefetched by marking the * first four bytes of every 64 byte chunk with * 0x7FFFFFF | (invalid_ownership_value << 31). * * When the max work request size is less than or equal to the WQE * basic block size, as an optimization, we can stamp all WQEs with * 0xffffffff, and skip the very first chunk of each WQE. */ static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size) { __be32 *wqe; int i; int s; int ind; void *buf; __be32 stamp; struct mlx4_wqe_ctrl_seg *ctrl; if (qp->sq_max_wqes_per_wr > 1) { s = roundup(size, 1U << qp->sq.wqe_shift); for (i = 0; i < s; i += 64) { ind = (i >> qp->sq.wqe_shift) + n; stamp = ind & qp->sq.wqe_cnt ? cpu_to_be32(0x7fffffff) : cpu_to_be32(0xffffffff); buf = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); wqe = buf + (i & ((1 << qp->sq.wqe_shift) - 1)); *wqe = stamp; } } else { ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1)); s = (ctrl->fence_size & 0x3f) << 4; for (i = 64; i < s; i += 64) { wqe = buf + i; *wqe = cpu_to_be32(0xffffffff); } } } static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size) { struct mlx4_wqe_ctrl_seg *ctrl; struct mlx4_wqe_inline_seg *inl; void *wqe; int s; ctrl = wqe = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1)); s = sizeof(struct mlx4_wqe_ctrl_seg); if (qp->ibqp.qp_type == IB_QPT_UD) { struct mlx4_wqe_datagram_seg *dgram = wqe + sizeof *ctrl; struct mlx4_av *av = (struct mlx4_av *)dgram->av; memset(dgram, 0, sizeof *dgram); av->port_pd = cpu_to_be32((qp->port << 24) | to_mpd(qp->ibqp.pd)->pdn); s += sizeof(struct mlx4_wqe_datagram_seg); } /* Pad the remainder of the WQE with an inline data segment. */ if (size > s) { inl = wqe + s; - inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl)); + inl->byte_count = cpu_to_be32(1U << 31 | (size - s - sizeof *inl)); } ctrl->srcrb_flags = 0; ctrl->fence_size = size / 16; /* * Make sure descriptor is fully written before setting ownership bit * (because HW can start executing as soon as we do). */ wmb(); ctrl->owner_opcode = cpu_to_be32(MLX4_OPCODE_NOP | MLX4_WQE_CTRL_NEC) | (n & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0); stamp_send_wqe(qp, n + qp->sq_spare_wqes, size); } /* Post NOP WQE to prevent wrap-around in the middle of WR */ static inline unsigned pad_wraparound(struct mlx4_ib_qp *qp, int ind) { unsigned s = qp->sq.wqe_cnt - (ind & (qp->sq.wqe_cnt - 1)); if (unlikely(s < qp->sq_max_wqes_per_wr)) { post_nop_wqe(qp, ind, s << qp->sq.wqe_shift); ind += s; } return ind; } static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type) { struct ib_event event; - struct mlx4_ib_qp *mqp = to_mibqp(qp); - struct ib_qp *ibqp = &mqp->ibqp; - struct mlx4_ib_xrc_reg_entry *ctx_entry; - unsigned long flags; + struct ib_qp *ibqp = &to_mibqp(qp)->ibqp; if (type == MLX4_EVENT_TYPE_PATH_MIG) to_mibqp(qp)->port = to_mibqp(qp)->alt_port; if (ibqp->event_handler) { event.device = ibqp->device; + event.element.qp = ibqp; switch (type) { case MLX4_EVENT_TYPE_PATH_MIG: event.event = IB_EVENT_PATH_MIG; break; case MLX4_EVENT_TYPE_COMM_EST: event.event = IB_EVENT_COMM_EST; break; case MLX4_EVENT_TYPE_SQ_DRAINED: event.event = IB_EVENT_SQ_DRAINED; break; case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE: event.event = IB_EVENT_QP_LAST_WQE_REACHED; break; case MLX4_EVENT_TYPE_WQ_CATAS_ERROR: event.event = IB_EVENT_QP_FATAL; break; case MLX4_EVENT_TYPE_PATH_MIG_FAILED: event.event = IB_EVENT_PATH_MIG_ERR; break; case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR: event.event = IB_EVENT_QP_REQ_ERR; break; case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR: event.event = IB_EVENT_QP_ACCESS_ERR; break; default: - printk(KERN_WARNING "mlx4_ib: Unexpected event type %d " + pr_warn("Unexpected event type %d " "on QP %06x\n", type, qp->qpn); return; } - if (unlikely(ibqp->qp_type == IB_QPT_XRC && - mqp->flags & MLX4_IB_XRC_RCV)) { - event.event |= IB_XRC_QP_EVENT_FLAG; - event.element.xrc_qp_num = ibqp->qp_num; - spin_lock_irqsave(&mqp->xrc_reg_list_lock, flags); - list_for_each_entry(ctx_entry, &mqp->xrc_reg_list, list) - ibqp->event_handler(&event, ctx_entry->context); - spin_unlock_irqrestore(&mqp->xrc_reg_list_lock, flags); - return; - } - event.element.qp = ibqp; ibqp->event_handler(&event, ibqp->qp_context); } } -static int send_wqe_overhead(enum ib_qp_type type, u32 flags) +static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags) { /* * UD WQEs must have a datagram segment. * RC and UC WQEs might have a remote address segment. * MLX WQEs need two extra inline data segments (for the UD * header and space for the ICRC). */ switch (type) { - case IB_QPT_UD: + case MLX4_IB_QPT_UD: return sizeof (struct mlx4_wqe_ctrl_seg) + sizeof (struct mlx4_wqe_datagram_seg) + - ((flags & MLX4_IB_QP_LSO) ? 128 : 0); - case IB_QPT_UC: + ((flags & MLX4_IB_QP_LSO) ? MLX4_IB_LSO_HEADER_SPARE : 0); + case MLX4_IB_QPT_PROXY_SMI_OWNER: + case MLX4_IB_QPT_PROXY_SMI: + case MLX4_IB_QPT_PROXY_GSI: return sizeof (struct mlx4_wqe_ctrl_seg) + + sizeof (struct mlx4_wqe_datagram_seg) + 64; + case MLX4_IB_QPT_TUN_SMI_OWNER: + case MLX4_IB_QPT_TUN_GSI: + return sizeof (struct mlx4_wqe_ctrl_seg) + + sizeof (struct mlx4_wqe_datagram_seg); + + case MLX4_IB_QPT_UC: + return sizeof (struct mlx4_wqe_ctrl_seg) + sizeof (struct mlx4_wqe_raddr_seg); - case IB_QPT_XRC: - case IB_QPT_RC: + case MLX4_IB_QPT_RC: return sizeof (struct mlx4_wqe_ctrl_seg) + - sizeof (struct mlx4_wqe_atomic_seg) + + sizeof (struct mlx4_wqe_masked_atomic_seg) + sizeof (struct mlx4_wqe_raddr_seg); - case IB_QPT_SMI: - case IB_QPT_GSI: + case MLX4_IB_QPT_SMI: + case MLX4_IB_QPT_GSI: return sizeof (struct mlx4_wqe_ctrl_seg) + ALIGN(MLX4_IB_UD_HEADER_SIZE + DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE, MLX4_INLINE_ALIGN) * sizeof (struct mlx4_wqe_inline_seg), sizeof (struct mlx4_wqe_data_seg)) + ALIGN(4 + sizeof (struct mlx4_wqe_inline_seg), sizeof (struct mlx4_wqe_data_seg)); - case IB_QPT_RAW_ETY: - return sizeof(struct mlx4_wqe_ctrl_seg) + - ALIGN(MLX4_IB_MAX_RAW_ETY_HDR_SIZE + - sizeof(struct mlx4_wqe_inline_seg), - sizeof(struct mlx4_wqe_data_seg)); - default: return sizeof (struct mlx4_wqe_ctrl_seg); } } static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap, - int is_user, int has_srq_or_is_xrc, struct mlx4_ib_qp *qp) + int is_user, int has_rq, struct mlx4_ib_qp *qp) { /* Sanity check RQ size before proceeding */ if (cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE || - cap->max_recv_sge > - min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg)) { - mlx4_ib_dbg("Requested RQ size (sge or wr) too large"); + cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg)) return -EINVAL; - } - if (has_srq_or_is_xrc) { - /* QPs attached to an SRQ should have no RQ */ - if (cap->max_recv_wr) { - mlx4_ib_dbg("non-zero RQ size for QP using SRQ"); + if (!has_rq) { + if (cap->max_recv_wr) return -EINVAL; - } qp->rq.wqe_cnt = qp->rq.max_gs = 0; } else { /* HW requires >= 1 RQ entry with >= 1 gather entry */ - if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) { - mlx4_ib_dbg("user QP RQ has 0 wr's or 0 sge's " - "(wr: 0x%x, sge: 0x%x)", cap->max_recv_wr, - cap->max_recv_sge); + if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) return -EINVAL; - } qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr)); qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge)); qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg)); } /* leave userspace return values as they were, so as not to break ABI */ if (is_user) { cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt; cap->max_recv_sge = qp->rq.max_gs; } else { cap->max_recv_wr = qp->rq.max_post = min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt); cap->max_recv_sge = min(qp->rq.max_gs, min(dev->dev->caps.max_sq_sg, - dev->dev->caps.max_rq_sg)); + dev->dev->caps.max_rq_sg)); } - /* We don't support inline sends for kernel QPs (yet) */ - return 0; } static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap, - enum ib_qp_type type, struct mlx4_ib_qp *qp) + enum mlx4_ib_qp_type type, struct mlx4_ib_qp *qp) { int s; /* Sanity check SQ size before proceeding */ - if (cap->max_send_wr > (dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE) || - cap->max_send_sge > - min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) || + if (cap->max_send_wr > (dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE) || + cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) || cap->max_inline_data + send_wqe_overhead(type, qp->flags) + - sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz) { - mlx4_ib_dbg("Requested SQ resources exceed device maxima"); + sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz) return -EINVAL; - } /* * For MLX transport we need 2 extra S/G entries: * one for the header and one for the checksum at the end */ - if ((type == IB_QPT_SMI || type == IB_QPT_GSI) && - cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg) { - mlx4_ib_dbg("No space for SQP hdr/csum sge's"); + if ((type == MLX4_IB_QPT_SMI || type == MLX4_IB_QPT_GSI || + type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) && + cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg) return -EINVAL; - } - if (type == IB_QPT_RAW_ETY && - cap->max_send_sge + 1 > dev->dev->caps.max_sq_sg) { - mlx4_ib_dbg("No space for RAW ETY hdr"); - return -EINVAL; - } - s = max(cap->max_send_sge * sizeof (struct mlx4_wqe_data_seg), cap->max_inline_data + sizeof (struct mlx4_wqe_inline_seg)) + send_wqe_overhead(type, qp->flags); if (s > dev->dev->caps.max_sq_desc_sz) return -EINVAL; /* * Hermon supports shrinking WQEs, such that a single work * request can include multiple units of 1 << wqe_shift. This * way, work requests can differ in size, and do not have to * be a power of 2 in size, saving memory and speeding up send * WR posting. Unfortunately, if we do this then the * wqe_index field in CQEs can't be used to look up the WR ID * anymore, so we do this only if selective signaling is off. * * Further, on 32-bit platforms, we can't use vmap() to make - * the QP buffer virtually contigious. Thus we have to use + * the QP buffer virtually contiguous. Thus we have to use * constant-sized WRs to make sure a WR is always fully within * a single page-sized chunk. * * Finally, we use NOP work requests to pad the end of the * work queue, to avoid wrap-around in the middle of WR. We * set NEC bit to avoid getting completions with error for * these NOP WRs, but since NEC is only supported starting * with firmware 2.2.232, we use constant-sized WRs for older * firmware. * * And, since MLX QPs only support SEND, we use constant-sized * WRs in this case. * * We look for the smallest value of wqe_shift such that the * resulting number of wqes does not exceed device * capabilities. * * We set WQE size to at least 64 bytes, this way stamping * invalidates each WQE. */ if (dev->dev->caps.fw_ver >= MLX4_FW_VER_WQE_CTRL_NEC && qp->sq_signal_bits && BITS_PER_LONG == 64 && - type != IB_QPT_SMI && type != IB_QPT_GSI && type != IB_QPT_RAW_ETY) + type != MLX4_IB_QPT_SMI && type != MLX4_IB_QPT_GSI && + !(type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_PROXY_SMI | + MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) qp->sq.wqe_shift = ilog2(64); else qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s)); for (;;) { qp->sq_max_wqes_per_wr = DIV_ROUND_UP(s, 1U << qp->sq.wqe_shift); /* * We need to leave 2 KB + 1 WR of headroom in the SQ to * allow HW to prefetch. */ qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + qp->sq_max_wqes_per_wr; qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr * qp->sq_max_wqes_per_wr + qp->sq_spare_wqes); if (qp->sq.wqe_cnt <= dev->dev->caps.max_wqes) break; if (qp->sq_max_wqes_per_wr <= 1) return -EINVAL; ++qp->sq.wqe_shift; } qp->sq.max_gs = (min(dev->dev->caps.max_sq_desc_sz, (qp->sq_max_wqes_per_wr << qp->sq.wqe_shift)) - send_wqe_overhead(type, qp->flags)) / sizeof (struct mlx4_wqe_data_seg); qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + (qp->sq.wqe_cnt << qp->sq.wqe_shift); if (qp->rq.wqe_shift > qp->sq.wqe_shift) { qp->rq.offset = 0; qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; } else { qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift; qp->sq.offset = 0; } cap->max_send_wr = qp->sq.max_post = (qp->sq.wqe_cnt - qp->sq_spare_wqes) / qp->sq_max_wqes_per_wr; cap->max_send_sge = min(qp->sq.max_gs, min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg)); qp->max_inline_data = cap->max_inline_data; return 0; } static int set_user_sq_size(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, struct mlx4_ib_create_qp *ucmd) { /* Sanity check SQ size before proceeding */ if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes || ucmd->log_sq_stride > ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) || - ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE) { - mlx4_ib_dbg("Requested max wqes or wqe stride exceeds max"); + ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE) return -EINVAL; - } qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count; qp->sq.wqe_shift = ucmd->log_sq_stride; qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + (qp->sq.wqe_cnt << qp->sq.wqe_shift); return 0; } +static int alloc_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp) +{ + int i; + + qp->sqp_proxy_rcv = + kmalloc(sizeof (struct mlx4_ib_buf) * qp->rq.wqe_cnt, + GFP_KERNEL); + if (!qp->sqp_proxy_rcv) + return -ENOMEM; + for (i = 0; i < qp->rq.wqe_cnt; i++) { + qp->sqp_proxy_rcv[i].addr = + kmalloc(sizeof (struct mlx4_ib_proxy_sqp_hdr), + GFP_KERNEL); + if (!qp->sqp_proxy_rcv[i].addr) + goto err; + qp->sqp_proxy_rcv[i].map = + ib_dma_map_single(dev, qp->sqp_proxy_rcv[i].addr, + sizeof (struct mlx4_ib_proxy_sqp_hdr), + DMA_FROM_DEVICE); + } + return 0; + +err: + while (i > 0) { + --i; + ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map, + sizeof (struct mlx4_ib_proxy_sqp_hdr), + DMA_FROM_DEVICE); + kfree(qp->sqp_proxy_rcv[i].addr); + } + kfree(qp->sqp_proxy_rcv); + qp->sqp_proxy_rcv = NULL; + return -ENOMEM; +} + +static void free_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp) +{ + int i; + + for (i = 0; i < qp->rq.wqe_cnt; i++) { + ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map, + sizeof (struct mlx4_ib_proxy_sqp_hdr), + DMA_FROM_DEVICE); + kfree(qp->sqp_proxy_rcv[i].addr); + } + kfree(qp->sqp_proxy_rcv); +} + +static int qp_has_rq(struct ib_qp_init_attr *attr) +{ + if (attr->qp_type == IB_QPT_XRC_INI || attr->qp_type == IB_QPT_XRC_TGT) + return 0; + + return !attr->srq; +} + +#ifdef __linux__ +static int init_qpg_parent(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *pqp, + struct ib_qp_init_attr *attr, int *qpn) +{ + struct mlx4_ib_qpg_data *qpg_data; + int tss_num, rss_num; + int tss_align_num, rss_align_num; + int tss_base, rss_base = 0; + int err; + + /* Parent is part of the TSS range (in SW TSS ARP is sent via parent) */ + tss_num = 1 + attr->parent_attrib.tss_child_count; + tss_align_num = roundup_pow_of_two(tss_num); + rss_num = attr->parent_attrib.rss_child_count; + rss_align_num = roundup_pow_of_two(rss_num); + + if (rss_num > 1) { + /* RSS is requested */ + if (!(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS)) + return -ENOSYS; + if (rss_align_num > dev->dev->caps.max_rss_tbl_sz) + return -EINVAL; + /* We must work with power of two */ + attr->parent_attrib.rss_child_count = rss_align_num; + } + + qpg_data = kzalloc(sizeof *qpg_data, GFP_KERNEL); + if (!qpg_data) + return -ENOMEM; + + if(pqp->flags & MLX4_IB_QP_NETIF) + err = mlx4_ib_steer_qp_alloc(dev, tss_align_num, &tss_base); + else + err = mlx4_qp_reserve_range(dev->dev, tss_align_num, + tss_align_num, &tss_base, 1); + if (err) + goto err1; + + if (tss_num > 1) { + u32 alloc = BITS_TO_LONGS(tss_align_num) * sizeof(long); + qpg_data->tss_bitmap = kzalloc(alloc, GFP_KERNEL); + if (qpg_data->tss_bitmap == NULL) { + err = -ENOMEM; + goto err2; + } + bitmap_fill(qpg_data->tss_bitmap, tss_num); + /* Note parent takes first index */ + clear_bit(0, qpg_data->tss_bitmap); + } + + if (rss_num > 1) { + u32 alloc = BITS_TO_LONGS(rss_align_num) * sizeof(long); + err = mlx4_qp_reserve_range(dev->dev, rss_align_num, + 1, &rss_base, 0); + if (err) + goto err3; + qpg_data->rss_bitmap = kzalloc(alloc, GFP_KERNEL); + if (qpg_data->rss_bitmap == NULL) { + err = -ENOMEM; + goto err4; + } + bitmap_fill(qpg_data->rss_bitmap, rss_align_num); + } + + qpg_data->tss_child_count = attr->parent_attrib.tss_child_count; + qpg_data->rss_child_count = attr->parent_attrib.rss_child_count; + qpg_data->qpg_parent = pqp; + qpg_data->qpg_tss_mask_sz = ilog2(tss_align_num); + qpg_data->tss_qpn_base = tss_base; + qpg_data->rss_qpn_base = rss_base; + + pqp->qpg_data = qpg_data; + *qpn = tss_base; + + return 0; + +err4: + mlx4_qp_release_range(dev->dev, rss_base, rss_align_num); + +err3: + if (tss_num > 1) + kfree(qpg_data->tss_bitmap); + +err2: + if(pqp->flags & MLX4_IB_QP_NETIF) + mlx4_ib_steer_qp_free(dev, tss_base, tss_align_num); + else + mlx4_qp_release_range(dev->dev, tss_base, tss_align_num); + +err1: + kfree(qpg_data); + return err; +} + +static void free_qpg_parent(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *pqp) +{ + struct mlx4_ib_qpg_data *qpg_data = pqp->qpg_data; + int align_num; + + if (qpg_data->tss_child_count > 1) + kfree(qpg_data->tss_bitmap); + + align_num = roundup_pow_of_two(1 + qpg_data->tss_child_count); + if(pqp->flags & MLX4_IB_QP_NETIF) + mlx4_ib_steer_qp_free(dev, qpg_data->tss_qpn_base, align_num); + else + mlx4_qp_release_range(dev->dev, qpg_data->tss_qpn_base, align_num); + + if (qpg_data->rss_child_count > 1) { + kfree(qpg_data->rss_bitmap); + align_num = roundup_pow_of_two(qpg_data->rss_child_count); + mlx4_qp_release_range(dev->dev, qpg_data->rss_qpn_base, + align_num); + } + + kfree(qpg_data); +} + +static int alloc_qpg_qpn(struct ib_qp_init_attr *init_attr, + struct mlx4_ib_qp *pqp, int *qpn) +{ + struct mlx4_ib_qp *mqp = to_mqp(init_attr->qpg_parent); + struct mlx4_ib_qpg_data *qpg_data = mqp->qpg_data; + u32 idx, old; + + switch (init_attr->qpg_type) { + case IB_QPG_CHILD_TX: + if (qpg_data->tss_child_count == 0) + return -EINVAL; + do { + /* Parent took index 0 */ + idx = find_first_bit(qpg_data->tss_bitmap, + qpg_data->tss_child_count + 1); + if (idx >= qpg_data->tss_child_count + 1) + return -ENOMEM; + old = test_and_clear_bit(idx, qpg_data->tss_bitmap); + } while (old == 0); + idx += qpg_data->tss_qpn_base; + break; + case IB_QPG_CHILD_RX: + if (qpg_data->rss_child_count == 0) + return -EINVAL; + do { + idx = find_first_bit(qpg_data->rss_bitmap, + qpg_data->rss_child_count); + if (idx >= qpg_data->rss_child_count) + return -ENOMEM; + old = test_and_clear_bit(idx, qpg_data->rss_bitmap); + } while (old == 0); + idx += qpg_data->rss_qpn_base; + break; + default: + return -EINVAL; + } + + pqp->qpg_data = qpg_data; + *qpn = idx; + + return 0; +} + +static void free_qpg_qpn(struct mlx4_ib_qp *mqp, int qpn) +{ + struct mlx4_ib_qpg_data *qpg_data = mqp->qpg_data; + + switch (mqp->qpg_type) { + case IB_QPG_CHILD_TX: + /* Do range check */ + qpn -= qpg_data->tss_qpn_base; + set_bit(qpn, qpg_data->tss_bitmap); + break; + case IB_QPG_CHILD_RX: + qpn -= qpg_data->rss_qpn_base; + set_bit(qpn, qpg_data->rss_bitmap); + break; + default: + /* error */ + pr_warn("wrong qpg type (%d)\n", mqp->qpg_type); + break; + } +} +#endif + +static int alloc_qpn_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, + struct ib_qp_init_attr *attr, int *qpn) +{ + int err = 0; + + switch (attr->qpg_type) { + case IB_QPG_NONE: + /* Raw packet QPNs must be aligned to 8 bits. If not, the WQE + * BlueFlame setup flow wrongly causes VLAN insertion. */ + if (attr->qp_type == IB_QPT_RAW_PACKET) { + err = mlx4_qp_reserve_range(dev->dev, 1, 1, qpn, 1); + } else { + if(qp->flags & MLX4_IB_QP_NETIF) + err = mlx4_ib_steer_qp_alloc(dev, 1, qpn); + else + err = mlx4_qp_reserve_range(dev->dev, 1, 1, qpn, 0); + } + break; + case IB_QPG_PARENT: +#ifdef __linux__ + err = init_qpg_parent(dev, qp, attr, qpn); +#endif + break; + case IB_QPG_CHILD_TX: + case IB_QPG_CHILD_RX: +#ifdef __linux__ + err = alloc_qpg_qpn(attr, qp, qpn); +#endif + break; + default: + qp->qpg_type = IB_QPG_NONE; + err = -EINVAL; + break; + } + if (err) + return err; + qp->qpg_type = attr->qpg_type; + return 0; +} + +static void free_qpn_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, + enum ib_qpg_type qpg_type, int qpn) +{ + switch (qpg_type) { + case IB_QPG_NONE: + if (qp->flags & MLX4_IB_QP_NETIF) + mlx4_ib_steer_qp_free(dev, qpn, 1); + else + mlx4_qp_release_range(dev->dev, qpn, 1); + break; + case IB_QPG_PARENT: +#ifdef __linux__ + free_qpg_parent(dev, qp); +#endif + break; + case IB_QPG_CHILD_TX: + case IB_QPG_CHILD_RX: +#ifdef __linux__ + free_qpg_qpn(qp, qpn); +#endif + break; + default: + break; + } +} + +/* Revert allocation on create_qp_common */ +static void unalloc_qpn_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, + struct ib_qp_init_attr *attr, int qpn) +{ + free_qpn_common(dev, qp, attr->qpg_type, qpn); +} + +static void release_qpn_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) +{ + free_qpn_common(dev, qp, qp->qpg_type, qp->mqp.qpn); +} + static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd, struct ib_qp_init_attr *init_attr, - struct ib_udata *udata, int sqpn, struct mlx4_ib_qp *qp) + struct ib_udata *udata, int sqpn, struct mlx4_ib_qp **caller_qp) { int qpn; int err; + struct mlx4_ib_sqp *sqp; + struct mlx4_ib_qp *qp; + enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type; +#ifndef __linux__ + init_attr->qpg_type = IB_QPG_NONE; +#endif + + /* When tunneling special qps, we use a plain UD qp */ + if (sqpn) { + if (mlx4_is_mfunc(dev->dev) && + (!mlx4_is_master(dev->dev) || + !(init_attr->create_flags & MLX4_IB_SRIOV_SQP))) { + if (init_attr->qp_type == IB_QPT_GSI) + qp_type = MLX4_IB_QPT_PROXY_GSI; + else if (mlx4_is_master(dev->dev)) + qp_type = MLX4_IB_QPT_PROXY_SMI_OWNER; + else + qp_type = MLX4_IB_QPT_PROXY_SMI; + } + qpn = sqpn; + /* add extra sg entry for tunneling */ + init_attr->cap.max_recv_sge++; + } else if (init_attr->create_flags & MLX4_IB_SRIOV_TUNNEL_QP) { + struct mlx4_ib_qp_tunnel_init_attr *tnl_init = + container_of(init_attr, + struct mlx4_ib_qp_tunnel_init_attr, init_attr); + if ((tnl_init->proxy_qp_type != IB_QPT_SMI && + tnl_init->proxy_qp_type != IB_QPT_GSI) || + !mlx4_is_master(dev->dev)) + return -EINVAL; + if (tnl_init->proxy_qp_type == IB_QPT_GSI) + qp_type = MLX4_IB_QPT_TUN_GSI; + else if (tnl_init->slave == mlx4_master_func_num(dev->dev)) + qp_type = MLX4_IB_QPT_TUN_SMI_OWNER; + else + qp_type = MLX4_IB_QPT_TUN_SMI; + /* we are definitely in the PPF here, since we are creating + * tunnel QPs. base_tunnel_sqpn is therefore valid. */ + qpn = dev->dev->phys_caps.base_tunnel_sqpn + 8 * tnl_init->slave + + tnl_init->proxy_qp_type * 2 + tnl_init->port - 1; + sqpn = qpn; + } + + if (!*caller_qp) { + if (qp_type == MLX4_IB_QPT_SMI || qp_type == MLX4_IB_QPT_GSI || + (qp_type & (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER | + MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) { + sqp = kzalloc(sizeof (struct mlx4_ib_sqp), GFP_KERNEL); + if (!sqp) + return -ENOMEM; + qp = &sqp->qp; + qp->pri.vid = qp->alt.vid = 0xFFFF; + } else { + qp = kzalloc(sizeof (struct mlx4_ib_qp), GFP_KERNEL); + if (!qp) + return -ENOMEM; + qp->pri.vid = qp->alt.vid = 0xFFFF; + } + } else + qp = *caller_qp; + + qp->mlx4_ib_qp_type = qp_type; + mutex_init(&qp->mutex); spin_lock_init(&qp->sq.lock); spin_lock_init(&qp->rq.lock); - spin_lock_init(&qp->xrc_reg_list_lock); INIT_LIST_HEAD(&qp->gid_list); + INIT_LIST_HEAD(&qp->steering_rules); + INIT_LIST_HEAD(&qp->rules_list); qp->state = IB_QPS_RESET; if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); - err = set_rq_size(dev, &init_attr->cap, !!pd->uobject, - !!init_attr->srq || !!init_attr->xrc_domain , qp); + err = set_rq_size(dev, &init_attr->cap, !!pd->uobject, qp_has_rq(init_attr), qp); if (err) goto err; if (pd->uobject) { struct mlx4_ib_create_qp ucmd; + int shift; + int n; if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { err = -EFAULT; goto err; } qp->sq_no_prefetch = ucmd.sq_no_prefetch; err = set_user_sq_size(dev, qp, &ucmd); if (err) goto err; qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, qp->buf_size, 0, 0); if (IS_ERR(qp->umem)) { err = PTR_ERR(qp->umem); - mlx4_ib_dbg("ib_umem_get error (%d)", err); goto err; } - err = mlx4_mtt_init(dev->dev, ib_umem_page_count(qp->umem), - ilog2(qp->umem->page_size), &qp->mtt); - if (err) { - mlx4_ib_dbg("mlx4_mtt_init error (%d)", err); + n = ib_umem_page_count(qp->umem); + shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n); + err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt); + + if (err) goto err_buf; - } err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem); - if (err) { - mlx4_ib_dbg("mlx4_ib_umem_write_mtt error (%d)", err); + if (err) goto err_mtt; - } - if (!init_attr->srq && init_attr->qp_type != IB_QPT_XRC) { + if (qp_has_rq(init_attr)) { err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context), ucmd.db_addr, &qp->db); - if (err) { - mlx4_ib_dbg("mlx4_ib_db_map_user error (%d)", err); + if (err) goto err_mtt; - } } } else { qp->sq_no_prefetch = 0; if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK; if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) qp->flags |= MLX4_IB_QP_LSO; - err = set_kernel_sq_size(dev, &init_attr->cap, init_attr->qp_type, qp); + if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP && + dev->dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED && + !mlx4_is_mfunc(dev->dev)) + qp->flags |= MLX4_IB_QP_NETIF; + + err = set_kernel_sq_size(dev, &init_attr->cap, qp_type, qp); if (err) goto err; - if (!init_attr->srq && init_attr->qp_type != IB_QPT_XRC) { + if (qp_has_rq(init_attr)) { err = mlx4_db_alloc(dev->dev, &qp->db, 0); if (err) goto err; *qp->db.db = 0; } if (qp->max_inline_data) { - err = mlx4_bf_alloc(dev->dev, &qp->bf); + err = mlx4_bf_alloc(dev->dev, &qp->bf, 0); if (err) { - mlx4_ib_dbg("failed to allocate blue flame register (%d)", err); + pr_debug("failed to allocate blue flame" + " register (%d)", err); qp->bf.uar = &dev->priv_uar; } } else qp->bf.uar = &dev->priv_uar; if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf)) { err = -ENOMEM; goto err_db; } err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift, &qp->mtt); - if (err) { - mlx4_ib_dbg("kernel qp mlx4_mtt_init error (%d)", err); + if (err) goto err_buf; - } err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf); - if (err) { - mlx4_ib_dbg("mlx4_buf_write_mtt error (%d)", err); + if (err) goto err_mtt; - } qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof (u64), GFP_KERNEL); qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof (u64), GFP_KERNEL); if (!qp->sq.wrid || !qp->rq.wrid) { err = -ENOMEM; goto err_wrid; } } if (sqpn) { - qpn = sqpn; + if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER | + MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) { + if (alloc_proxy_bufs(pd->device, qp)) { + err = -ENOMEM; + goto err_wrid; + } + } } else { - err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn); + err = alloc_qpn_common(dev, qp, init_attr, &qpn); if (err) - goto err_wrid; + goto err_proxy; } err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp); if (err) goto err_qpn; - if (init_attr->qp_type == IB_QPT_XRC) + if (init_attr->qp_type == IB_QPT_XRC_TGT) qp->mqp.qpn |= (1 << 23); /* * Hardware wants QPN written in big-endian order (after * shifting) for send doorbell. Precompute this value to save * a little bit when posting sends. */ qp->doorbell_qpn = swab32(qp->mqp.qpn << 8); qp->mqp.event = mlx4_ib_qp_event; - + if (!*caller_qp) + *caller_qp = qp; return 0; err_qpn: - if (!sqpn) - mlx4_qp_release_range(dev->dev, qpn, 1); + unalloc_qpn_common(dev, qp, init_attr, qpn); +err_proxy: + if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI) + free_proxy_bufs(pd->device, qp); err_wrid: if (pd->uobject) { - if (!init_attr->srq && init_attr->qp_type != IB_QPT_XRC) - mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), - &qp->db); + if (qp_has_rq(init_attr)) + mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db); } else { kfree(qp->sq.wrid); kfree(qp->rq.wrid); } err_mtt: mlx4_mtt_cleanup(dev->dev, &qp->mtt); err_buf: if (pd->uobject) ib_umem_release(qp->umem); else mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); err_db: - if (!pd->uobject && !init_attr->srq && init_attr->qp_type != IB_QPT_XRC) + if (!pd->uobject && qp_has_rq(init_attr)) mlx4_db_free(dev->dev, &qp->db); if (qp->max_inline_data) mlx4_bf_free(dev->dev, &qp->bf); err: + if (!*caller_qp) + kfree(qp); return err; } static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state) { switch (state) { case IB_QPS_RESET: return MLX4_QP_STATE_RST; case IB_QPS_INIT: return MLX4_QP_STATE_INIT; case IB_QPS_RTR: return MLX4_QP_STATE_RTR; case IB_QPS_RTS: return MLX4_QP_STATE_RTS; case IB_QPS_SQD: return MLX4_QP_STATE_SQD; case IB_QPS_SQE: return MLX4_QP_STATE_SQER; case IB_QPS_ERR: return MLX4_QP_STATE_ERR; default: return -1; } } static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq) + __acquires(&send_cq->lock) __acquires(&recv_cq->lock) { - if (send_cq == recv_cq) + if (send_cq == recv_cq) { spin_lock_irq(&send_cq->lock); - else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { + (void) __acquire(&recv_cq->lock); + } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { spin_lock_irq(&send_cq->lock); spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); } else { spin_lock_irq(&recv_cq->lock); spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); } } static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq) + __releases(&send_cq->lock) __releases(&recv_cq->lock) { - if (send_cq == recv_cq) + if (send_cq == recv_cq) { + (void) __release(&recv_cq->lock); spin_unlock_irq(&send_cq->lock); - else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { + } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { spin_unlock(&recv_cq->lock); spin_unlock_irq(&send_cq->lock); } else { spin_unlock(&send_cq->lock); spin_unlock_irq(&recv_cq->lock); } } static void del_gid_entries(struct mlx4_ib_qp *qp) { - struct gid_entry *ge, *tmp; + struct mlx4_ib_gid_entry *ge, *tmp; list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) { list_del(&ge->list); kfree(ge); } } +static struct mlx4_ib_pd *get_pd(struct mlx4_ib_qp *qp) +{ + if (qp->ibqp.qp_type == IB_QPT_XRC_TGT) + return to_mpd(to_mxrcd(qp->ibqp.xrcd)->pd); + else + return to_mpd(qp->ibqp.pd); +} + +static void get_cqs(struct mlx4_ib_qp *qp, + struct mlx4_ib_cq **send_cq, struct mlx4_ib_cq **recv_cq) +{ + switch (qp->ibqp.qp_type) { + case IB_QPT_XRC_TGT: + *send_cq = to_mcq(to_mxrcd(qp->ibqp.xrcd)->cq); + *recv_cq = *send_cq; + break; + case IB_QPT_XRC_INI: + *send_cq = to_mcq(qp->ibqp.send_cq); + *recv_cq = *send_cq; + break; + default: + *send_cq = to_mcq(qp->ibqp.send_cq); + *recv_cq = to_mcq(qp->ibqp.recv_cq); + break; + } +} + static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, int is_user) { struct mlx4_ib_cq *send_cq, *recv_cq; - if (qp->state != IB_QPS_RESET) + if (qp->state != IB_QPS_RESET) { if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state), MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp)) - printk(KERN_WARNING "mlx4_ib: modify QP %06x to RESET failed.\n", + pr_warn("modify QP %06x to RESET failed.\n", qp->mqp.qpn); + if (qp->pri.smac) { + mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac); + qp->pri.smac = 0; + } + if (qp->alt.smac) { + mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac); + qp->alt.smac = 0; + } + if (qp->pri.vid < 0x1000) { + mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid); + qp->pri.vid = 0xFFFF; + qp->pri.candidate_vid = 0xFFFF; + qp->pri.update_vid = 0; + } + if (qp->alt.vid < 0x1000) { + mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid); + qp->alt.vid = 0xFFFF; + qp->alt.candidate_vid = 0xFFFF; + qp->alt.update_vid = 0; + } + } - send_cq = to_mcq(qp->ibqp.send_cq); - recv_cq = to_mcq(qp->ibqp.recv_cq); + get_cqs(qp, &send_cq, &recv_cq); mlx4_ib_lock_cqs(send_cq, recv_cq); if (!is_user) { __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn, qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL); if (send_cq != recv_cq) __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL); } mlx4_qp_remove(dev->dev, &qp->mqp); mlx4_ib_unlock_cqs(send_cq, recv_cq); mlx4_qp_free(dev->dev, &qp->mqp); - if (!is_sqp(dev, qp)) - mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1); + if (!is_sqp(dev, qp) && !is_tunnel_qp(dev, qp)) + release_qpn_common(dev, qp); mlx4_mtt_cleanup(dev->dev, &qp->mtt); if (is_user) { - if (!qp->ibqp.srq && qp->ibqp.qp_type != IB_QPT_XRC) + if (qp->rq.wqe_cnt) mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.uobject->context), &qp->db); ib_umem_release(qp->umem); } else { kfree(qp->sq.wrid); kfree(qp->rq.wrid); + if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER | + MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) + free_proxy_bufs(&dev->ib_dev, qp); mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); if (qp->max_inline_data) mlx4_bf_free(dev->dev, &qp->bf); - if (!qp->ibqp.srq && qp->ibqp.qp_type != IB_QPT_XRC) + + if (qp->rq.wqe_cnt) mlx4_db_free(dev->dev, &qp->db); } del_gid_entries(qp); } +static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr) +{ + /* Native or PPF */ + if (!mlx4_is_mfunc(dev->dev) || + (mlx4_is_master(dev->dev) && + attr->create_flags & MLX4_IB_SRIOV_SQP)) { + return dev->dev->phys_caps.base_sqpn + + (attr->qp_type == IB_QPT_SMI ? 0 : 2) + + attr->port_num - 1; + } + /* PF or VF -- creating proxies */ + if (attr->qp_type == IB_QPT_SMI) + return dev->dev->caps.qp0_proxy[attr->port_num - 1]; + else + return dev->dev->caps.qp1_proxy[attr->port_num - 1]; +} + +#ifdef __linux__ +static int check_qpg_attr(struct mlx4_ib_dev *dev, + struct ib_qp_init_attr *attr) +{ + if (attr->qpg_type == IB_QPG_NONE) + return 0; + + if (attr->qp_type != IB_QPT_UD) + return -EINVAL; + + if (attr->qpg_type == IB_QPG_PARENT) { + if (attr->parent_attrib.tss_child_count == 1) + return -EINVAL; /* Doesn't make sense */ + if (attr->parent_attrib.rss_child_count == 1) + return -EINVAL; /* Doesn't make sense */ + if ((attr->parent_attrib.tss_child_count == 0) && + (attr->parent_attrib.rss_child_count == 0)) + /* Should be called with IP_QPG_NONE */ + return -EINVAL; + if (attr->parent_attrib.rss_child_count > 1) { + int rss_align_num; + if (!(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS)) + return -ENOSYS; + rss_align_num = roundup_pow_of_two( + attr->parent_attrib.rss_child_count); + if (rss_align_num > dev->dev->caps.max_rss_tbl_sz) + return -EINVAL; + } + } else { + struct mlx4_ib_qpg_data *qpg_data; + if (attr->qpg_parent == NULL) + return -EINVAL; + if (IS_ERR(attr->qpg_parent)) + return -EINVAL; + qpg_data = to_mqp(attr->qpg_parent)->qpg_data; + if (qpg_data == NULL) + return -EINVAL; + if (attr->qpg_type == IB_QPG_CHILD_TX && + !qpg_data->tss_child_count) + return -EINVAL; + if (attr->qpg_type == IB_QPG_CHILD_RX && + !qpg_data->rss_child_count) + return -EINVAL; + } + return 0; +} +#endif + +#define RESERVED_FLAGS_MASK ((((unsigned int)IB_QP_CREATE_RESERVED_END - 1) | IB_QP_CREATE_RESERVED_END) \ + & ~(IB_QP_CREATE_RESERVED_START - 1)) + +static enum mlx4_ib_qp_flags to_mlx4_ib_qp_flags(enum ib_qp_create_flags ib_qp_flags) +{ + enum mlx4_ib_qp_flags mlx4_ib_qp_flags = 0; + + if (ib_qp_flags & IB_QP_CREATE_IPOIB_UD_LSO) + mlx4_ib_qp_flags |= MLX4_IB_QP_LSO; + + if (ib_qp_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) + mlx4_ib_qp_flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK; + + if (ib_qp_flags & IB_QP_CREATE_NETIF_QP) + mlx4_ib_qp_flags |= MLX4_IB_QP_NETIF; + + /* reserved flags */ + mlx4_ib_qp_flags |= (ib_qp_flags & RESERVED_FLAGS_MASK); + + return mlx4_ib_qp_flags; +} + struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { - struct mlx4_ib_dev *dev = to_mdev(pd->device); - struct mlx4_ib_sqp *sqp; - struct mlx4_ib_qp *qp; + struct mlx4_ib_qp *qp = NULL; int err; + u16 xrcdn = 0; + enum mlx4_ib_qp_flags mlx4_qp_flags = to_mlx4_ib_qp_flags(init_attr->create_flags); + struct ib_device *device; + /* see ib_core::ib_create_qp same handling */ + device = pd ? pd->device : init_attr->xrcd->device; /* - * We only support LSO and multicast loopback blocking, and - * only for kernel UD QPs. + * We only support LSO, vendor flag1, and multicast loopback blocking, + * and only for kernel UD QPs. */ - if (init_attr->create_flags & ~(IB_QP_CREATE_IPOIB_UD_LSO | - IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)) + if (mlx4_qp_flags & ~(MLX4_IB_QP_LSO | + MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK | + MLX4_IB_SRIOV_TUNNEL_QP | MLX4_IB_SRIOV_SQP | + MLX4_IB_QP_NETIF)) return ERR_PTR(-EINVAL); + if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) { + if (init_attr->qp_type != IB_QPT_UD) + return ERR_PTR(-EINVAL); + } + if (init_attr->create_flags && - (pd->uobject || init_attr->qp_type != IB_QPT_UD)) + (udata || + ((mlx4_qp_flags & ~MLX4_IB_SRIOV_SQP) && + init_attr->qp_type != IB_QPT_UD) || + ((mlx4_qp_flags & MLX4_IB_SRIOV_SQP) && + init_attr->qp_type > IB_QPT_GSI))) return ERR_PTR(-EINVAL); +#ifdef __linux__ + err = check_qpg_attr(to_mdev(device), init_attr); + if (err) + return ERR_PTR(err); +#endif + switch (init_attr->qp_type) { - case IB_QPT_XRC: - if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) + case IB_QPT_XRC_TGT: + pd = to_mxrcd(init_attr->xrcd)->pd; + xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn; + init_attr->send_cq = to_mxrcd(init_attr->xrcd)->cq; + /* fall through */ + case IB_QPT_XRC_INI: + if (!(to_mdev(device)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) return ERR_PTR(-ENOSYS); + init_attr->recv_cq = init_attr->send_cq; + /* fall through */ case IB_QPT_RC: case IB_QPT_UC: - case IB_QPT_UD: - case IB_QPT_RAW_ETH: - { + case IB_QPT_RAW_PACKET: qp = kzalloc(sizeof *qp, GFP_KERNEL); if (!qp) return ERR_PTR(-ENOMEM); - - err = create_qp_common(dev, pd, init_attr, udata, 0, qp); + qp->pri.vid = qp->alt.vid = 0xFFFF; + /* fall through */ + case IB_QPT_UD: + { + err = create_qp_common(to_mdev(device), pd, init_attr, udata, 0, &qp); if (err) { kfree(qp); return ERR_PTR(err); } - if (init_attr->qp_type == IB_QPT_XRC) - qp->xrcdn = to_mxrcd(init_attr->xrc_domain)->xrcdn; - else - qp->xrcdn = 0; - qp->ibqp.qp_num = qp->mqp.qpn; + qp->xrcdn = xrcdn; break; } - case IB_QPT_RAW_ETY: - if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_RAW_ETY)) - return ERR_PTR(-ENOSYS); case IB_QPT_SMI: case IB_QPT_GSI: { /* Userspace is not allowed to create special QPs: */ - if (pd->uobject) { - mlx4_ib_dbg("Userspace is not allowed to create special QPs"); + if (udata) return ERR_PTR(-EINVAL); - } - sqp = kzalloc(sizeof *sqp, GFP_KERNEL); - if (!sqp) - return ERR_PTR(-ENOMEM); - - qp = &sqp->qp; - - err = create_qp_common(dev, pd, init_attr, udata, - dev->dev->caps.sqp_start + - (init_attr->qp_type == IB_QPT_RAW_ETY ? 4 : - (init_attr->qp_type == IB_QPT_SMI ? 0 : 2)) + - init_attr->port_num - 1, - qp); - if (err) { - kfree(sqp); + err = create_qp_common(to_mdev(device), pd, init_attr, udata, + get_sqp_num(to_mdev(device), init_attr), + &qp); + if (err) return ERR_PTR(err); - } qp->port = init_attr->port_num; qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1; break; } default: - mlx4_ib_dbg("Invalid QP type requested for create_qp (%d)", - init_attr->qp_type); + /* Don't support raw QPs */ return ERR_PTR(-EINVAL); } return &qp->ibqp; } int mlx4_ib_destroy_qp(struct ib_qp *qp) { struct mlx4_ib_dev *dev = to_mdev(qp->device); struct mlx4_ib_qp *mqp = to_mqp(qp); + struct mlx4_ib_pd *pd; if (is_qp0(dev, mqp)) mlx4_CLOSE_PORT(dev->dev, mqp->port); - destroy_qp_common(dev, mqp, !!qp->pd->uobject); + pd = get_pd(mqp); + destroy_qp_common(dev, mqp, !!pd->ibpd.uobject); if (is_sqp(dev, mqp)) kfree(to_msqp(mqp)); else kfree(mqp); return 0; } -static int to_mlx4_st(enum ib_qp_type type) +static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type) { switch (type) { - case IB_QPT_RC: return MLX4_QP_ST_RC; - case IB_QPT_UC: return MLX4_QP_ST_UC; - case IB_QPT_UD: return MLX4_QP_ST_UD; - case IB_QPT_XRC: return MLX4_QP_ST_XRC; - case IB_QPT_RAW_ETY: - case IB_QPT_SMI: - case IB_QPT_GSI: - case IB_QPT_RAW_ETH: return MLX4_QP_ST_MLX; - default: return -1; + case MLX4_IB_QPT_RC: return MLX4_QP_ST_RC; + case MLX4_IB_QPT_UC: return MLX4_QP_ST_UC; + case MLX4_IB_QPT_UD: return MLX4_QP_ST_UD; + case MLX4_IB_QPT_XRC_INI: + case MLX4_IB_QPT_XRC_TGT: return MLX4_QP_ST_XRC; + case MLX4_IB_QPT_SMI: + case MLX4_IB_QPT_GSI: + case MLX4_IB_QPT_RAW_PACKET: return MLX4_QP_ST_MLX; + + case MLX4_IB_QPT_PROXY_SMI_OWNER: + case MLX4_IB_QPT_TUN_SMI_OWNER: return (mlx4_is_mfunc(dev->dev) ? + MLX4_QP_ST_MLX : -1); + case MLX4_IB_QPT_PROXY_SMI: + case MLX4_IB_QPT_TUN_SMI: + case MLX4_IB_QPT_PROXY_GSI: + case MLX4_IB_QPT_TUN_GSI: return (mlx4_is_mfunc(dev->dev) ? + MLX4_QP_ST_UD : -1); + default: return -1; } } static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr, int attr_mask) { u8 dest_rd_atomic; u32 access_flags; u32 hw_access_flags = 0; if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) dest_rd_atomic = attr->max_dest_rd_atomic; else dest_rd_atomic = qp->resp_depth; if (attr_mask & IB_QP_ACCESS_FLAGS) access_flags = attr->qp_access_flags; else access_flags = qp->atomic_rd_en; if (!dest_rd_atomic) access_flags &= IB_ACCESS_REMOTE_WRITE; if (access_flags & IB_ACCESS_REMOTE_READ) hw_access_flags |= MLX4_QP_BIT_RRE; if (access_flags & IB_ACCESS_REMOTE_ATOMIC) hw_access_flags |= MLX4_QP_BIT_RAE; if (access_flags & IB_ACCESS_REMOTE_WRITE) hw_access_flags |= MLX4_QP_BIT_RWE; return cpu_to_be32(hw_access_flags); } static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr, int attr_mask) { if (attr_mask & IB_QP_PKEY_INDEX) sqp->pkey_index = attr->pkey_index; if (attr_mask & IB_QP_QKEY) sqp->qkey = attr->qkey; if (attr_mask & IB_QP_SQ_PSN) sqp->send_psn = attr->sq_psn; } static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port) { path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6); } static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah, - struct mlx4_qp_path *path, u8 port) + struct mlx4_ib_qp *qp, struct mlx4_qp_path *path, + u8 port, int is_primary) { + struct net_device *ndev; int err; int is_eth = rdma_port_get_link_layer(&dev->ib_dev, port) == IB_LINK_LAYER_ETHERNET; u8 mac[6]; int is_mcast; u16 vlan_tag; int vidx; + int smac_index; + u64 u64_mac; + u8 *smac; + struct mlx4_roce_smac_vlan_info *smac_info; path->grh_mylmc = ah->src_path_bits & 0x7f; path->rlid = cpu_to_be16(ah->dlid); if (ah->static_rate) { path->static_rate = ah->static_rate + MLX4_STAT_RATE_OFFSET; while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET && !(1 << path->static_rate & dev->dev->caps.stat_rate_support)) --path->static_rate; } else path->static_rate = 0; if (ah->ah_flags & IB_AH_GRH) { if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) { - printk(KERN_ERR "sgid_index (%u) too large. max is %d\n", + pr_err("sgid_index (%u) too large. max is %d\n", ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1); return -1; } path->grh_mylmc |= 1 << 7; path->mgid_index = ah->grh.sgid_index; path->hop_limit = ah->grh.hop_limit; path->tclass_flowlabel = cpu_to_be32((ah->grh.traffic_class << 20) | (ah->grh.flow_label)); memcpy(path->rgid, ah->grh.dgid.raw, 16); } if (is_eth) { - path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | - ((port - 1) << 6) | ((ah->sl & 0x7) << 3) | ((ah->sl & 8) >> 1); - if (!(ah->ah_flags & IB_AH_GRH)) return -1; + path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | + ((port - 1) << 6) | ((ah->sl & 7) << 3); + + if (is_primary) + smac_info = &qp->pri; + else + smac_info = &qp->alt; + + vlan_tag = rdma_get_vlan_id(&dev->iboe.gid_table[port - 1][ah->grh.sgid_index]); + if (vlan_tag < 0x1000) { + if (smac_info->vid < 0x1000) { + /* both valid vlan ids */ + if (smac_info->vid != vlan_tag) { + /* different VIDs. unreg old and reg new */ + err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx); + if (err) + return err; + smac_info->candidate_vid = vlan_tag; + smac_info->candidate_vlan_index = vidx; + smac_info->candidate_vlan_port = port; + smac_info->update_vid = 1; + path->vlan_index = vidx; + path->fl = 1 << 6; + } else { + path->vlan_index = smac_info->vlan_index; + path->fl = 1 << 6; + } + } else { + /* no current vlan tag in qp */ + err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx); + if (err) + return err; + smac_info->candidate_vid = vlan_tag; + smac_info->candidate_vlan_index = vidx; + smac_info->candidate_vlan_port = port; + smac_info->update_vid = 1; + path->vlan_index = vidx; + path->fl = 1 << 6; + } + } else { + /* have current vlan tag. unregister it at modify-qp success */ + if (smac_info->vid < 0x1000) { + smac_info->candidate_vid = 0xFFFF; + smac_info->update_vid = 1; + } + } + err = mlx4_ib_resolve_grh(dev, ah, mac, &is_mcast, port); if (err) return err; + /* get smac_index for RoCE use. + * If no smac was yet assigned, register one. + * If one was already assigned, but the new mac differs, + * unregister the old one and register the new one. + */ + spin_lock(&dev->iboe.lock); + ndev = dev->iboe.netdevs[port - 1]; + if (ndev) { +#ifdef __linux__ + smac = ndev->dev_addr; /* fixme: cache this value */ +#else + smac = IF_LLADDR(ndev); /* fixme: cache this value */ +#endif + + u64_mac = mlx4_mac_to_u64(smac); + } else + u64_mac = dev->dev->caps.def_mac[port]; + spin_unlock(&dev->iboe.lock); + + if (!smac_info->smac || smac_info->smac != u64_mac) { + /* register candidate now, unreg if needed, after success */ + smac_index = mlx4_register_mac(dev->dev, port, u64_mac); + if (smac_index >= 0) { + smac_info->candidate_smac_index = smac_index; + smac_info->candidate_smac = u64_mac; + smac_info->candidate_smac_port = port; + } else + return -EINVAL; + } else + smac_index = smac_info->smac_index; + memcpy(path->dmac, mac, 6); path->ackto = MLX4_IB_LINK_TYPE_ETH; - /* use index 0 into MAC table for IBoE */ - path->grh_mylmc &= 0x80; + /* put MAC table smac index for IBoE */ + path->grh_mylmc = (u8) (smac_index) | 0x80 ; - vlan_tag = rdma_get_vlan_id(&dev->iboe.gid_table[port - 1][ah->grh.sgid_index]); - if (vlan_tag < 0x1000) { - if (mlx4_find_cached_vlan(dev->dev, port, vlan_tag, &vidx)) - return -ENOENT; - - path->vlan_index = vidx; - path->fl = 1 << 6; - } } else path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | ((port - 1) << 6) | ((ah->sl & 0xf) << 2); return 0; } static void update_mcg_macs(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) { - struct gid_entry *ge, *tmp; + struct mlx4_ib_gid_entry *ge, *tmp; list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) { if (!ge->added && mlx4_ib_add_mc(dev, qp, &ge->gid)) { ge->added = 1; ge->port = qp->port; } } } +static int handle_eth_ud_smac_index(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, + struct mlx4_qp_context *context) +{ + struct net_device *ndev; + u64 u64_mac; + u8 *smac; + int smac_index; + + ndev = dev->iboe.netdevs[qp->port - 1]; + if (ndev) { +#ifdef __linux__ + smac = ndev->dev_addr; /* fixme: cache this value */ +#else + smac = IF_LLADDR(ndev); /* fixme: cache this value */ +#endif + u64_mac = mlx4_mac_to_u64(smac); + } else + u64_mac = dev->dev->caps.def_mac[qp->port]; + + context->pri_path.sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | ((qp->port - 1) << 6); + if (!qp->pri.smac) { + smac_index = mlx4_register_mac(dev->dev, qp->port, u64_mac); + if (smac_index >= 0) { + qp->pri.candidate_smac_index = smac_index; + qp->pri.candidate_smac = u64_mac; + qp->pri.candidate_smac_port = qp->port; + context->pri_path.grh_mylmc = 0x80 | (u8) smac_index; + } else + return -ENOENT; + } + return 0; +} static int __mlx4_ib_modify_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, enum ib_qp_state cur_state, enum ib_qp_state new_state) { struct mlx4_ib_dev *dev = to_mdev(ibqp->device); struct mlx4_ib_qp *qp = to_mqp(ibqp); + struct mlx4_ib_pd *pd; + struct mlx4_ib_cq *send_cq, *recv_cq; struct mlx4_qp_context *context; enum mlx4_qp_optpar optpar = 0; int sqd_event; + int steer_qp = 0; int err = -EINVAL; + int is_eth = -1; context = kzalloc(sizeof *context, GFP_KERNEL); if (!context) return -ENOMEM; context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) | - (to_mlx4_st(ibqp->qp_type) << 16)); + (to_mlx4_st(dev, qp->mlx4_ib_qp_type) << 16)); if (!(attr_mask & IB_QP_PATH_MIG_STATE)) context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11); else { optpar |= MLX4_QP_OPTPAR_PM_STATE; switch (attr->path_mig_state) { case IB_MIG_MIGRATED: context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11); break; case IB_MIG_REARM: context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11); break; case IB_MIG_ARMED: context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11); break; } } - if (ibqp->qp_type == IB_QPT_RAW_ETH) - context->mtu_msgmax = 0xff; - else if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI || - ibqp->qp_type == IB_QPT_RAW_ETY) + + if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI) context->mtu_msgmax = (IB_MTU_4096 << 5) | 11; + else if (ibqp->qp_type == IB_QPT_RAW_PACKET) + context->mtu_msgmax = (MLX4_RAW_QP_MTU << 5) | MLX4_RAW_QP_MSGMAX; else if (ibqp->qp_type == IB_QPT_UD) { if (qp->flags & MLX4_IB_QP_LSO) context->mtu_msgmax = (IB_MTU_4096 << 5) | ilog2(dev->dev->caps.max_gso_sz); else context->mtu_msgmax = (IB_MTU_4096 << 5) | 12; } else if (attr_mask & IB_QP_PATH_MTU) { if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) { - printk(KERN_ERR "path MTU (%u) is invalid\n", + pr_err("path MTU (%u) is invalid\n", attr->path_mtu); goto out; } context->mtu_msgmax = (attr->path_mtu << 5) | ilog2(dev->dev->caps.max_msg_sz); } if (qp->rq.wqe_cnt) context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3; context->rq_size_stride |= qp->rq.wqe_shift - 4; if (qp->sq.wqe_cnt) context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3; context->sq_size_stride |= qp->sq.wqe_shift - 4; if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { context->sq_size_stride |= !!qp->sq_no_prefetch << 7; - if (ibqp->qp_type == IB_QPT_XRC) - context->xrcd = cpu_to_be32((u32) qp->xrcdn); + context->xrcd = cpu_to_be32((u32) qp->xrcdn); + context->param3 |= cpu_to_be32(1 << 30); } if (qp->ibqp.uobject) context->usr_page = cpu_to_be32(to_mucontext(ibqp->uobject->context)->uar.index); else context->usr_page = cpu_to_be32(qp->bf.uar->index); if (attr_mask & IB_QP_DEST_QPN) context->remote_qpn = cpu_to_be32(attr->dest_qp_num); if (attr_mask & IB_QP_PORT) { if (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD && !(attr_mask & IB_QP_AV)) { mlx4_set_sched(&context->pri_path, attr->port_num); optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE; } } - if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR && - dev->counters[qp->port - 1] != -1) { - context->pri_path.counter_index = dev->counters[qp->port - 1]; - optpar |= MLX4_QP_OPTPAR_COUNTER_INDEX; + if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { + if (dev->counters[qp->port - 1] != -1) { + context->pri_path.counter_index = + dev->counters[qp->port - 1]; + optpar |= MLX4_QP_OPTPAR_COUNTER_INDEX; + } else + context->pri_path.counter_index = 0xff; + + if (qp->flags & MLX4_IB_QP_NETIF && + (qp->qpg_type == IB_QPG_NONE || qp->qpg_type == IB_QPG_PARENT)) { + mlx4_ib_steer_qp_reg(dev, qp, 1); + steer_qp = 1; + } } if (attr_mask & IB_QP_PKEY_INDEX) { + if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) + context->pri_path.disable_pkey_check = 0x40; context->pri_path.pkey_index = attr->pkey_index; optpar |= MLX4_QP_OPTPAR_PKEY_INDEX; } if (attr_mask & IB_QP_AV) { - if (mlx4_set_path(dev, &attr->ah_attr, &context->pri_path, - attr_mask & IB_QP_PORT ? attr->port_num : qp->port)) { - mlx4_ib_dbg("qpn 0x%x: could not set pri path params", - ibqp->qp_num); + if (mlx4_set_path(dev, &attr->ah_attr, qp, &context->pri_path, + attr_mask & IB_QP_PORT ? + attr->port_num : qp->port, 1)) goto out; - } optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH | MLX4_QP_OPTPAR_SCHED_QUEUE); } if (attr_mask & IB_QP_TIMEOUT) { - context->pri_path.ackto |= (attr->timeout << 3); + context->pri_path.ackto |= attr->timeout << 3; optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT; } if (attr_mask & IB_QP_ALT_PATH) { if (attr->alt_port_num == 0 || - attr->alt_port_num > dev->num_ports) { - mlx4_ib_dbg("qpn 0x%x: invalid alternate port num (%d)", - ibqp->qp_num, attr->alt_port_num); + attr->alt_port_num > dev->dev->caps.num_ports) goto out; - } if (attr->alt_pkey_index >= - dev->dev->caps.pkey_table_len[attr->alt_port_num]) { - mlx4_ib_dbg("qpn 0x%x: invalid alt pkey index (0x%x)", - ibqp->qp_num, attr->alt_pkey_index); + dev->dev->caps.pkey_table_len[attr->alt_port_num]) goto out; - } - if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path, - attr->alt_port_num)) { - mlx4_ib_dbg("qpn 0x%x: could not set alt path params", - ibqp->qp_num); + if (mlx4_set_path(dev, &attr->alt_ah_attr, qp, &context->alt_path, + attr->alt_port_num, 0)) goto out; - } context->alt_path.pkey_index = attr->alt_pkey_index; context->alt_path.ackto = attr->alt_timeout << 3; optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH; } - context->pd = cpu_to_be32(to_mpd(ibqp->pd)->pdn); - context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28); + pd = get_pd(qp); + get_cqs(qp, &send_cq, &recv_cq); + context->pd = cpu_to_be32(pd->pdn); + context->cqn_send = cpu_to_be32(send_cq->mcq.cqn); + context->cqn_recv = cpu_to_be32(recv_cq->mcq.cqn); + context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28); /* Set "fast registration enabled" for all kernel QPs */ if (!qp->ibqp.uobject) context->params1 |= cpu_to_be32(1 << 11); if (attr_mask & IB_QP_RNR_RETRY) { context->params1 |= cpu_to_be32(attr->rnr_retry << 13); optpar |= MLX4_QP_OPTPAR_RNR_RETRY; } if (attr_mask & IB_QP_RETRY_CNT) { context->params1 |= cpu_to_be32(attr->retry_cnt << 16); optpar |= MLX4_QP_OPTPAR_RETRY_COUNT; } if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { if (attr->max_rd_atomic) context->params1 |= cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21); optpar |= MLX4_QP_OPTPAR_SRA_MAX; } if (attr_mask & IB_QP_SQ_PSN) context->next_send_psn = cpu_to_be32(attr->sq_psn); - context->cqn_send = cpu_to_be32(to_mcq(ibqp->send_cq)->mcq.cqn); - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { if (attr->max_dest_rd_atomic) context->params2 |= cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21); optpar |= MLX4_QP_OPTPAR_RRA_MAX; } if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) { context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask); optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE; } + if (attr_mask & IB_M_EXT_CLASS_1) + context->params2 |= cpu_to_be32(MLX4_QP_BIT_COLL_MASTER); + + /* for now we enable also sqe on send */ + if (attr_mask & IB_M_EXT_CLASS_2) { + context->params2 |= cpu_to_be32(MLX4_QP_BIT_COLL_SYNC_SQ); + context->params2 |= cpu_to_be32(MLX4_QP_BIT_COLL_MASTER); + } + + if (attr_mask & IB_M_EXT_CLASS_3) + context->params2 |= cpu_to_be32(MLX4_QP_BIT_COLL_SYNC_RQ); + if (ibqp->srq) context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC); if (attr_mask & IB_QP_MIN_RNR_TIMER) { context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24); optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT; } if (attr_mask & IB_QP_RQ_PSN) context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn); - context->cqn_recv = cpu_to_be32(to_mcq(ibqp->recv_cq)->mcq.cqn); - + /* proxy and tunnel qp qkeys will be changed in modify-qp wrappers */ if (attr_mask & IB_QP_QKEY) { - context->qkey = cpu_to_be32(attr->qkey); + if (qp->mlx4_ib_qp_type & + (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) + context->qkey = cpu_to_be32(IB_QP_SET_QKEY); + else { + if (mlx4_is_mfunc(dev->dev) && + !(qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) && + (attr->qkey & MLX4_RESERVED_QKEY_MASK) == + MLX4_RESERVED_QKEY_BASE) { + pr_err("Cannot use reserved QKEY" + " 0x%x (range 0xffff0000..0xffffffff" + " is reserved)\n", attr->qkey); + err = -EINVAL; + goto out; + } + context->qkey = cpu_to_be32(attr->qkey); + } optpar |= MLX4_QP_OPTPAR_Q_KEY; } if (ibqp->srq) context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn); - if (!ibqp->srq && ibqp->qp_type != IB_QPT_XRC && - cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) + if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) context->db_rec_addr = cpu_to_be64(qp->db.dma); if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR && (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI || - ibqp->qp_type == IB_QPT_UD || ibqp->qp_type == IB_QPT_RAW_ETY || - ibqp->qp_type == IB_QPT_RAW_ETH)) { + ibqp->qp_type == IB_QPT_UD || + ibqp->qp_type == IB_QPT_RAW_PACKET)) { context->pri_path.sched_queue = (qp->port - 1) << 6; - if (is_qp0(dev, qp)) + if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI || + qp->mlx4_ib_qp_type & + (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) { context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE; - else + if (qp->mlx4_ib_qp_type != MLX4_IB_QPT_SMI) + context->pri_path.fl = 0x80; + } else { + if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) + context->pri_path.fl = 0x80; context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE; + } + is_eth = rdma_port_get_link_layer(&dev->ib_dev, qp->port) == + IB_LINK_LAYER_ETHERNET; + if (is_eth) { + if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI || + qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) + context->pri_path.feup = 1 << 7; /* don't fsm */ + /* handle smac_index */ + if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_UD || + qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI || + qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI) { + err = handle_eth_ud_smac_index(dev, qp, context); + if (err) + return -EINVAL; + } + } } if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD && attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify) sqd_event = 1; else sqd_event = 0; if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) context->rlkey |= (1 << 4); + if ((attr_mask & IB_QP_GROUP_RSS) && + (qp->qpg_data->rss_child_count > 1)) { + struct mlx4_ib_qpg_data *qpg_data = qp->qpg_data; + void *rss_context_base = &context->pri_path; + struct mlx4_rss_context *rss_context = + (struct mlx4_rss_context *) (rss_context_base + + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH); + + context->flags |= cpu_to_be32(1 << MLX4_RSS_QPC_FLAG_OFFSET); + + /* This should be tbl_sz_base_qpn */ + rss_context->base_qpn = cpu_to_be32(qpg_data->rss_qpn_base | + (ilog2(qpg_data->rss_child_count) << 24)); + rss_context->default_qpn = cpu_to_be32(qpg_data->rss_qpn_base); + /* This should be flags_hash_fn */ + rss_context->flags = MLX4_RSS_TCP_IPV6 | + MLX4_RSS_TCP_IPV4; + if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UDP_RSS) { + rss_context->base_qpn_udp = rss_context->default_qpn; + rss_context->flags |= MLX4_RSS_IPV6 | + MLX4_RSS_IPV4 | + MLX4_RSS_UDP_IPV6 | + MLX4_RSS_UDP_IPV4; + } + if (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP) { + static const u32 rsskey[10] = { 0xD181C62C, 0xF7F4DB5B, + 0x1983A2FC, 0x943E1ADB, 0xD9389E6B, 0xD1039C2C, + 0xA74499AD, 0x593D56D9, 0xF3253C06, 0x2ADC1FFC}; + rss_context->hash_fn = MLX4_RSS_HASH_TOP; + memcpy(rss_context->rss_key, rsskey, + sizeof(rss_context->rss_key)); + } else { + rss_context->hash_fn = MLX4_RSS_HASH_XOR; + memset(rss_context->rss_key, 0, + sizeof(rss_context->rss_key)); + } + } /* * Before passing a kernel QP to the HW, make sure that the * ownership bits of the send queue are set and the SQ * headroom is stamped so that the hardware doesn't start * processing stale work requests. */ if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { struct mlx4_wqe_ctrl_seg *ctrl; int i; for (i = 0; i < qp->sq.wqe_cnt; ++i) { ctrl = get_send_wqe(qp, i); ctrl->owner_opcode = cpu_to_be32(1 << 31); if (qp->sq_max_wqes_per_wr == 1) ctrl->fence_size = 1 << (qp->sq.wqe_shift - 4); stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift); } } err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state), to_mlx4_state(new_state), context, optpar, sqd_event, &qp->mqp); if (err) goto out; qp->state = new_state; if (attr_mask & IB_QP_ACCESS_FLAGS) qp->atomic_rd_en = attr->qp_access_flags; if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) qp->resp_depth = attr->max_dest_rd_atomic; if (attr_mask & IB_QP_PORT) { qp->port = attr->port_num; update_mcg_macs(dev, qp); } if (attr_mask & IB_QP_ALT_PATH) qp->alt_port = attr->alt_port_num; if (is_sqp(dev, qp)) store_sqp_attrs(to_msqp(qp), attr, attr_mask); + /* Set 'ignore_cq_overrun' bits for collectives offload */ + if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { + if (attr_mask & (IB_M_EXT_CLASS_2 | IB_M_EXT_CLASS_3)) { + err = mlx4_ib_ignore_overrun_cq(ibqp->send_cq); + if (err) { + pr_err("Failed to set ignore CQ " + "overrun for QP 0x%x's send CQ\n", + ibqp->qp_num); + goto out; + } + + if (ibqp->recv_cq != ibqp->send_cq) { + err = mlx4_ib_ignore_overrun_cq(ibqp->recv_cq); + if (err) { + pr_err("Failed to set ignore " + "CQ overrun for QP 0x%x's recv " + "CQ\n", ibqp->qp_num); + goto out; + } + } + } + } + /* * If we moved QP0 to RTR, bring the IB link up; if we moved * QP0 to RESET or ERROR, bring the link back down. */ if (is_qp0(dev, qp)) { if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR) if (mlx4_INIT_PORT(dev->dev, qp->port)) - printk(KERN_WARNING "INIT_PORT failed for port %d\n", + pr_warn("INIT_PORT failed for port %d\n", qp->port); if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR && (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR)) mlx4_CLOSE_PORT(dev->dev, qp->port); } /* * If we moved a kernel QP to RESET, clean up all old CQ * entries and reinitialize the QP. */ - if (new_state == IB_QPS_RESET && !ibqp->uobject) { - mlx4_ib_cq_clean(to_mcq(ibqp->recv_cq), qp->mqp.qpn, - ibqp->srq ? to_msrq(ibqp->srq): NULL); - if (ibqp->send_cq != ibqp->recv_cq) - mlx4_ib_cq_clean(to_mcq(ibqp->send_cq), qp->mqp.qpn, NULL); + if (new_state == IB_QPS_RESET) { + if (!ibqp->uobject) { + mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn, + ibqp->srq ? to_msrq(ibqp->srq) : NULL); + if (send_cq != recv_cq) + mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL); - qp->rq.head = 0; - qp->rq.tail = 0; - qp->sq.head = 0; - qp->sq.tail = 0; - qp->sq_next_wqe = 0; - if (!ibqp->srq && ibqp->qp_type != IB_QPT_XRC) - *qp->db.db = 0; + qp->rq.head = 0; + qp->rq.tail = 0; + qp->sq.head = 0; + qp->sq.tail = 0; + qp->sq_next_wqe = 0; + if (qp->rq.wqe_cnt) + *qp->db.db = 0; + + if (qp->flags & MLX4_IB_QP_NETIF && + (qp->qpg_type == IB_QPG_NONE || + qp->qpg_type == IB_QPG_PARENT)) + mlx4_ib_steer_qp_reg(dev, qp, 0); + } + if (qp->pri.smac) { + mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac); + qp->pri.smac = 0; + } + if (qp->alt.smac) { + mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac); + qp->alt.smac = 0; + } + if (qp->pri.vid < 0x1000) { + mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid); + qp->pri.vid = 0xFFFF; + qp->pri.candidate_vid = 0xFFFF; + qp->pri.update_vid = 0; + } + + if (qp->alt.vid < 0x1000) { + mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid); + qp->alt.vid = 0xFFFF; + qp->alt.candidate_vid = 0xFFFF; + qp->alt.update_vid = 0; + } } out: + if (err && steer_qp) + mlx4_ib_steer_qp_reg(dev, qp, 0); kfree(context); + if (qp->pri.candidate_smac) { + if (err) + mlx4_unregister_mac(dev->dev, qp->pri.candidate_smac_port, qp->pri.candidate_smac); + else { + if (qp->pri.smac) { + mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac); + } + qp->pri.smac = qp->pri.candidate_smac; + qp->pri.smac_index = qp->pri.candidate_smac_index; + qp->pri.smac_port = qp->pri.candidate_smac_port; + + } + qp->pri.candidate_smac = 0; + qp->pri.candidate_smac_index = 0; + qp->pri.candidate_smac_port = 0; + } + if (qp->alt.candidate_smac) { + if (err) + mlx4_unregister_mac(dev->dev, qp->alt.candidate_smac_port, qp->pri.candidate_smac); + else { + if (qp->pri.smac) { + mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac); + } + qp->alt.smac = qp->alt.candidate_smac; + qp->alt.smac_index = qp->alt.candidate_smac_index; + qp->alt.smac_port = qp->alt.candidate_smac_port; + + } + qp->pri.candidate_smac = 0; + qp->pri.candidate_smac_index = 0; + qp->pri.candidate_smac_port = 0; + } + + if (qp->pri.update_vid) { + if (err) { + if (qp->pri.candidate_vid < 0x1000) + mlx4_unregister_vlan(dev->dev, qp->pri.candidate_vlan_port, + qp->pri.candidate_vid); + } else { + if (qp->pri.vid < 0x1000) + mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, + qp->pri.vid); + qp->pri.vid = qp->pri.candidate_vid; + qp->pri.vlan_port = qp->pri.candidate_vlan_port; + qp->pri.vlan_index = qp->pri.candidate_vlan_index; + } + qp->pri.candidate_vid = 0xFFFF; + qp->pri.update_vid = 0; + } + + if (qp->alt.update_vid) { + if (err) { + if (qp->alt.candidate_vid < 0x1000) + mlx4_unregister_vlan(dev->dev, qp->alt.candidate_vlan_port, + qp->alt.candidate_vid); + } else { + if (qp->alt.vid < 0x1000) + mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, + qp->alt.vid); + qp->alt.vid = qp->alt.candidate_vid; + qp->alt.vlan_port = qp->alt.candidate_vlan_port; + qp->alt.vlan_index = qp->alt.candidate_vlan_index; + } + qp->alt.candidate_vid = 0xFFFF; + qp->alt.update_vid = 0; + } + return err; } int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(ibqp->device); struct mlx4_ib_qp *qp = to_mqp(ibqp); enum ib_qp_state cur_state, new_state; int err = -EINVAL; mutex_lock(&qp->mutex); cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state; new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; - if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask)) { - mlx4_ib_dbg("qpn 0x%x: invalid attribute mask specified " - "for transition %d to %d. qp_type %d, attr_mask 0x%x", - ibqp->qp_num, cur_state, new_state, - ibqp->qp_type, attr_mask); + if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, + attr_mask & ~IB_M_QP_MOD_VEND_MASK)) { + pr_debug("qpn 0x%x: invalid attribute mask specified " + "for transition %d to %d. qp_type %d," + " attr_mask 0x%x\n", + ibqp->qp_num, cur_state, new_state, + ibqp->qp_type, attr_mask); goto out; } - if ((attr_mask & IB_QP_PORT) && (ibqp->qp_type != IB_QPT_RAW_ETH) && - (attr->port_num == 0 || attr->port_num > dev->num_ports)) { - mlx4_ib_dbg("qpn 0x%x: invalid port number (%d) specified " - "for transition %d to %d. qp_type %d", - ibqp->qp_num, attr->port_num, cur_state, - new_state, ibqp->qp_type); + if ((attr_mask & IB_M_QP_MOD_VEND_MASK) && !dev->dev->caps.sync_qp) { + pr_err("extended verbs are not supported by %s\n", + dev->ib_dev.name); goto out; } - if ((attr_mask & IB_QP_PORT) && (ibqp->qp_type == IB_QPT_RAW_ETH) && - (rdma_port_get_link_layer(&dev->ib_dev, attr->port_num) - != IB_LINK_LAYER_ETHERNET)) { - mlx4_ib_dbg("qpn 0x%x: invalid port (%d) specified (not RDMAoE)" - "for transition %d to %d. qp_type %d", - ibqp->qp_num, attr->port_num, cur_state, - new_state, ibqp->qp_type); + if ((attr_mask & IB_QP_PORT) && + (attr->port_num == 0 || attr->port_num > dev->num_ports)) { + pr_debug("qpn 0x%x: invalid port number (%d) specified " + "for transition %d to %d. qp_type %d\n", + ibqp->qp_num, attr->port_num, cur_state, + new_state, ibqp->qp_type); goto out; } + if ((attr_mask & IB_QP_PORT) && (ibqp->qp_type == IB_QPT_RAW_PACKET) && + (rdma_port_get_link_layer(&dev->ib_dev, attr->port_num) != + IB_LINK_LAYER_ETHERNET)) + goto out; + if (attr_mask & IB_QP_PKEY_INDEX) { int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p]) { - mlx4_ib_dbg("qpn 0x%x: invalid pkey index (%d) specified " - "for transition %d to %d. qp_type %d", - ibqp->qp_num, attr->pkey_index, cur_state, - new_state, ibqp->qp_type); + pr_debug("qpn 0x%x: invalid pkey index (%d) specified " + "for transition %d to %d. qp_type %d\n", + ibqp->qp_num, attr->pkey_index, cur_state, + new_state, ibqp->qp_type); goto out; } } if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) { - mlx4_ib_dbg("qpn 0x%x: max_rd_atomic (%d) too large. " - "Transition %d to %d. qp_type %d", - ibqp->qp_num, attr->max_rd_atomic, cur_state, - new_state, ibqp->qp_type); + pr_debug("qpn 0x%x: max_rd_atomic (%d) too large. " + "Transition %d to %d. qp_type %d\n", + ibqp->qp_num, attr->max_rd_atomic, cur_state, + new_state, ibqp->qp_type); goto out; } if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) { - mlx4_ib_dbg("qpn 0x%x: max_dest_rd_atomic (%d) too large. " - "Transition %d to %d. qp_type %d", - ibqp->qp_num, attr->max_dest_rd_atomic, cur_state, - new_state, ibqp->qp_type); + pr_debug("qpn 0x%x: max_dest_rd_atomic (%d) too large. " + "Transition %d to %d. qp_type %d\n", + ibqp->qp_num, attr->max_dest_rd_atomic, cur_state, + new_state, ibqp->qp_type); goto out; } if (cur_state == new_state && cur_state == IB_QPS_RESET) { err = 0; goto out; } err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state); out: mutex_unlock(&qp->mutex); return err; } -static int build_raw_ety_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr, - void *wqe, unsigned *mlx_seg_len) +static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp, + struct ib_send_wr *wr, + void *wqe, unsigned *mlx_seg_len) { - int payload = 0; - int header_size, packet_length; + struct mlx4_ib_dev *mdev = to_mdev(sqp->qp.ibqp.device); + struct ib_device *ib_dev = &mdev->ib_dev; struct mlx4_wqe_mlx_seg *mlx = wqe; struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx; - u32 *lrh = wqe + sizeof *mlx + sizeof *inl; + struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah); + u16 pkey; + u32 qkey; + int send_size; + int header_size; + int spc; int i; - /* Only IB_WR_SEND is supported */ if (wr->opcode != IB_WR_SEND) return -EINVAL; + send_size = 0; + for (i = 0; i < wr->num_sge; ++i) - payload += wr->sg_list[i].length; + send_size += wr->sg_list[i].length; - header_size = IB_LRH_BYTES + 4; /* LRH + RAW_HEADER (32 bits) */ + /* for proxy-qp0 sends, need to add in size of tunnel header */ + /* for tunnel-qp0 sends, tunnel header is already in s/g list */ + if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) + send_size += sizeof (struct mlx4_ib_tunnel_header); - /* headers + payload and round up */ - packet_length = (header_size + payload + 3) / 4; + ib_ud_header_init(send_size, 1, 0, 0, 0, 0, &sqp->ud_header); + if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) { + sqp->ud_header.lrh.service_level = + be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28; + sqp->ud_header.lrh.destination_lid = + cpu_to_be16(ah->av.ib.g_slid & 0x7f); + sqp->ud_header.lrh.source_lid = + cpu_to_be16(ah->av.ib.g_slid & 0x7f); + } + mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); - mlx->flags |= cpu_to_be32(MLX4_WQE_MLX_ICRC | - (wr->wr.raw_ety.lrh->service_level << 8)); + /* force loopback */ + mlx->flags |= cpu_to_be32(MLX4_WQE_MLX_VL15 | 0x1 | MLX4_WQE_MLX_SLR); + mlx->rlid = sqp->ud_header.lrh.destination_lid; - mlx->rlid = wr->wr.raw_ety.lrh->destination_lid; + sqp->ud_header.lrh.virtual_lane = 0; + sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED); + ib_get_cached_pkey(ib_dev, sqp->qp.port, 0, &pkey); + sqp->ud_header.bth.pkey = cpu_to_be16(pkey); + if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER) + sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn); + else + sqp->ud_header.bth.destination_qpn = + cpu_to_be32(mdev->dev->caps.qp0_tunnel[sqp->qp.port - 1]); - wr->wr.raw_ety.lrh->packet_length = cpu_to_be16(packet_length); + sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); + if (mlx4_get_parav_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey)) + return -EINVAL; + sqp->ud_header.deth.qkey = cpu_to_be32(qkey); + sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.mqp.qpn); - ib_lrh_header_pack(wr->wr.raw_ety.lrh, lrh); - lrh += IB_LRH_BYTES / 4; /* LRH size is a dword multiple */ - *lrh = cpu_to_be32(wr->wr.raw_ety.eth_type); + sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY; + sqp->ud_header.immediate_present = 0; - inl->byte_count = cpu_to_be32(1 << 31 | header_size); + header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf); - *mlx_seg_len = - ALIGN(sizeof(struct mlx4_wqe_inline_seg) + header_size, 16); + /* + * Inline data segments may not cross a 64 byte boundary. If + * our UD header is bigger than the space available up to the + * next 64 byte boundary in the WQE, use two inline data + * segments to hold the UD header. + */ + spc = MLX4_INLINE_ALIGN - + ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1)); + if (header_size <= spc) { + inl->byte_count = cpu_to_be32(1U << 31 | header_size); + memcpy(inl + 1, sqp->header_buf, header_size); + i = 1; + } else { + inl->byte_count = cpu_to_be32(1U << 31 | spc); + memcpy(inl + 1, sqp->header_buf, spc); + inl = (void *) (inl + 1) + spc; + memcpy(inl + 1, sqp->header_buf + spc, header_size - spc); + /* + * Need a barrier here to make sure all the data is + * visible before the byte_count field is set. + * Otherwise the HCA prefetcher could grab the 64-byte + * chunk with this inline segment and get a valid (!= + * 0xffffffff) byte count but stale data, and end up + * generating a packet with bad headers. + * + * The first inline segment's byte_count field doesn't + * need a barrier, because it comes after a + * control/MLX segment and therefore is at an offset + * of 16 mod 64. + */ + wmb(); + inl->byte_count = cpu_to_be32(1U << 31 | (header_size - spc)); + i = 2; + } + + *mlx_seg_len = + ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16); return 0; } static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr, void *wqe, unsigned *mlx_seg_len) { - struct ib_device *ib_dev = &to_mdev(sqp->qp.ibqp.device)->ib_dev; + struct ib_device *ib_dev = sqp->qp.ibqp.device; struct mlx4_wqe_mlx_seg *mlx = wqe; + struct mlx4_wqe_ctrl_seg *ctrl = wqe; struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx; struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah); + union ib_gid sgid; u16 pkey; int send_size; int header_size; int spc; int i; - union ib_gid sgid; int is_eth; - int is_grh; int is_vlan = 0; - int err; - u16 vlan; + int is_grh; + u16 vlan = 0; + int err = 0; - vlan = 0; send_size = 0; for (i = 0; i < wr->num_sge; ++i) send_size += wr->sg_list[i].length; is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET; is_grh = mlx4_ib_ah_grh_present(ah); - err = ib_get_cached_gid(ib_dev, be32_to_cpu(ah->av.ib.port_pd) >> 24, - ah->av.ib.gid_index, &sgid); - if (err) - return err; if (is_eth) { - is_vlan = rdma_get_vlan_id(&sgid) < 0x1000; + if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) { + /* When multi-function is enabled, the ib_core gid + * indexes don't necessarily match the hw ones, so + * we must use our own cache */ + err = mlx4_get_roce_gid_from_slave(to_mdev(ib_dev)->dev, + be32_to_cpu(ah->av.ib.port_pd) >> 24, + ah->av.ib.gid_index, &sgid.raw[0]); + if (err) + return err; + } else { + err = ib_get_cached_gid(ib_dev, + be32_to_cpu(ah->av.ib.port_pd) >> 24, + ah->av.ib.gid_index, &sgid); + if (err) + return err; + } + vlan = rdma_get_vlan_id(&sgid); + is_vlan = vlan < 0x1000; } - ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh, 0, &sqp->ud_header); + if (!is_eth) { sqp->ud_header.lrh.service_level = be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28; sqp->ud_header.lrh.destination_lid = ah->av.ib.dlid; sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.ib.g_slid & 0x7f); } if (is_grh) { sqp->ud_header.grh.traffic_class = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff; sqp->ud_header.grh.flow_label = ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff); sqp->ud_header.grh.hop_limit = ah->av.ib.hop_limit; - ib_get_cached_gid(ib_dev, be32_to_cpu(ah->av.ib.port_pd) >> 24, - ah->av.ib.gid_index, &sqp->ud_header.grh.source_gid); + if (is_eth) + memcpy(sqp->ud_header.grh.source_gid.raw, sgid.raw, 16); + else { + if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) { + /* When multi-function is enabled, the ib_core gid + * indexes don't necessarily match the hw ones, so + * we must use our own cache */ + sqp->ud_header.grh.source_gid.global.subnet_prefix = + to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1]. + subnet_prefix; + sqp->ud_header.grh.source_gid.global.interface_id = + to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1]. + guid_cache[ah->av.ib.gid_index]; + } else + ib_get_cached_gid(ib_dev, + be32_to_cpu(ah->av.ib.port_pd) >> 24, + ah->av.ib.gid_index, + &sqp->ud_header.grh.source_gid); + } memcpy(sqp->ud_header.grh.destination_gid.raw, ah->av.ib.dgid, 16); } mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); if (!is_eth) { mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) | (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) | (sqp->ud_header.lrh.service_level << 8)); + if (ah->av.ib.port_pd & cpu_to_be32(0x80000000)) + mlx->flags |= cpu_to_be32(0x1); /* force loopback */ mlx->rlid = sqp->ud_header.lrh.destination_lid; } switch (wr->opcode) { case IB_WR_SEND: - sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY; + sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY; sqp->ud_header.immediate_present = 0; break; case IB_WR_SEND_WITH_IMM: - sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; + sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; sqp->ud_header.immediate_present = 1; sqp->ud_header.immediate_data = wr->ex.imm_data; break; default: return -EINVAL; } if (is_eth) { - u8 *smac; + u8 smac[6]; + struct in6_addr in6; + u16 pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 29) << 13; + + mlx->sched_prio = cpu_to_be16(pcp); + memcpy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac, 6); -#ifdef __linux__ - smac = to_mdev(sqp->qp.ibqp.device)->iboe.netdevs[sqp->qp.port - 1]->dev_addr; /* fixme: cache this value */ -#else - smac = IF_LLADDR(to_mdev(sqp->qp.ibqp.device)->iboe.netdevs[sqp->qp.port - 1]); /* fixme: cache this value */ -#endif + /* FIXME: cache smac value? */ + memcpy(&ctrl->srcrb_flags16[0], ah->av.eth.mac, 2); + memcpy(&ctrl->imm, ah->av.eth.mac + 2, 4); + memcpy(&in6, sgid.raw, sizeof(in6)); + rdma_get_ll_mac(&in6, smac); memcpy(sqp->ud_header.eth.smac_h, smac, 6); if (!memcmp(sqp->ud_header.eth.smac_h, sqp->ud_header.eth.dmac_h, 6)) mlx->flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK); - if (!is_vlan) - sqp->ud_header.eth.type = cpu_to_be16(MLX4_IBOE_ETHERTYPE); - else { - u16 pcp; - - sqp->ud_header.vlan.type = cpu_to_be16(MLX4_IBOE_ETHERTYPE); - pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 27 & 3) << 13; + if (!is_vlan) { + sqp->ud_header.eth.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE); + } else { + sqp->ud_header.vlan.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE); sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp); } } else { sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0; if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE) sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE; } sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED); if (!sqp->qp.ibqp.qp_num) ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey); else ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->wr.ud.pkey_index, &pkey); sqp->ud_header.bth.pkey = cpu_to_be16(pkey); sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn); sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); sqp->ud_header.deth.qkey = cpu_to_be32(wr->wr.ud.remote_qkey & 0x80000000 ? sqp->qkey : wr->wr.ud.remote_qkey); sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num); header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf); if (0) { - printk(KERN_ERR "built UD header of size %d:\n", header_size); + pr_err("built UD header of size %d:\n", header_size); for (i = 0; i < header_size / 4; ++i) { if (i % 8 == 0) - printk(" [%02x] ", i * 4); - printk(" %08x", - be32_to_cpu(((__be32 *) sqp->header_buf)[i])); + pr_err(" [%02x] ", i * 4); + pr_cont(" %08x", + be32_to_cpu(((__be32 *) sqp->header_buf)[i])); if ((i + 1) % 8 == 0) - printk("\n"); + pr_cont("\n"); } - printk("\n"); + pr_err("\n"); } /* * Inline data segments may not cross a 64 byte boundary. If * our UD header is bigger than the space available up to the * next 64 byte boundary in the WQE, use two inline data * segments to hold the UD header. */ spc = MLX4_INLINE_ALIGN - - ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1)); + ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1)); if (header_size <= spc) { - inl->byte_count = cpu_to_be32(1 << 31 | header_size); + inl->byte_count = cpu_to_be32(1U << 31 | header_size); memcpy(inl + 1, sqp->header_buf, header_size); i = 1; } else { - inl->byte_count = cpu_to_be32(1 << 31 | spc); + inl->byte_count = cpu_to_be32(1U << 31 | spc); memcpy(inl + 1, sqp->header_buf, spc); inl = (void *) (inl + 1) + spc; memcpy(inl + 1, sqp->header_buf + spc, header_size - spc); /* * Need a barrier here to make sure all the data is * visible before the byte_count field is set. * Otherwise the HCA prefetcher could grab the 64-byte * chunk with this inline segment and get a valid (!= * 0xffffffff) byte count but stale data, and end up * generating a packet with bad headers. * * The first inline segment's byte_count field doesn't * need a barrier, because it comes after a * control/MLX segment and therefore is at an offset * of 16 mod 64. */ wmb(); - inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc)); + inl->byte_count = cpu_to_be32(1U << 31 | (header_size - spc)); i = 2; } *mlx_seg_len = - ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16); + ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16); return 0; } static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq) { unsigned cur; struct mlx4_ib_cq *cq; cur = wq->head - wq->tail; if (likely(cur + nreq < wq->max_post)) return 0; cq = to_mcq(ib_cq); spin_lock(&cq->lock); cur = wq->head - wq->tail; spin_unlock(&cq->lock); return cur + nreq >= wq->max_post; } static __be32 convert_access(int acc) { - return (acc & IB_ACCESS_REMOTE_ATOMIC ? cpu_to_be32(MLX4_WQE_FMR_PERM_ATOMIC) : 0) | - (acc & IB_ACCESS_REMOTE_WRITE ? cpu_to_be32(MLX4_WQE_FMR_PERM_REMOTE_WRITE) : 0) | - (acc & IB_ACCESS_REMOTE_READ ? cpu_to_be32(MLX4_WQE_FMR_PERM_REMOTE_READ) : 0) | - (acc & IB_ACCESS_LOCAL_WRITE ? cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_WRITE) : 0) | + return (acc & IB_ACCESS_REMOTE_ATOMIC ? cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC) : 0) | + (acc & IB_ACCESS_REMOTE_WRITE ? cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE) : 0) | + (acc & IB_ACCESS_REMOTE_READ ? cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ) : 0) | + (acc & IB_ACCESS_LOCAL_WRITE ? cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_WRITE) : 0) | cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_READ); } static void set_fmr_seg(struct mlx4_wqe_fmr_seg *fseg, struct ib_send_wr *wr) { struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(wr->wr.fast_reg.page_list); int i; for (i = 0; i < wr->wr.fast_reg.page_list_len; ++i) mfrpl->mapped_page_list[i] = cpu_to_be64(wr->wr.fast_reg.page_list->page_list[i] | MLX4_MTT_FLAG_PRESENT); fseg->flags = convert_access(wr->wr.fast_reg.access_flags); fseg->mem_key = cpu_to_be32(wr->wr.fast_reg.rkey); fseg->buf_list = cpu_to_be64(mfrpl->map); fseg->start_addr = cpu_to_be64(wr->wr.fast_reg.iova_start); fseg->reg_len = cpu_to_be64(wr->wr.fast_reg.length); fseg->offset = 0; /* XXX -- is this just for ZBVA? */ fseg->page_size = cpu_to_be32(wr->wr.fast_reg.page_shift); fseg->reserved[0] = 0; fseg->reserved[1] = 0; } static void set_local_inv_seg(struct mlx4_wqe_local_inval_seg *iseg, u32 rkey) { - iseg->flags = 0; - iseg->mem_key = cpu_to_be32(rkey); - iseg->guest_id = 0; - iseg->pa = 0; + iseg->mem_key = cpu_to_be32(rkey); + + iseg->reserved1 = 0; + iseg->reserved2 = 0; + iseg->reserved3[0] = 0; + iseg->reserved3[1] = 0; } static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg, u64 remote_addr, u32 rkey) { rseg->raddr = cpu_to_be64(remote_addr); rseg->rkey = cpu_to_be32(rkey); rseg->reserved = 0; } static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg, struct ib_send_wr *wr) { if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap); aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add); } else if (wr->opcode == IB_WR_MASKED_ATOMIC_FETCH_AND_ADD) { aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add); aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add_mask); } else { aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add); aseg->compare = 0; } } static void set_masked_atomic_seg(struct mlx4_wqe_masked_atomic_seg *aseg, struct ib_send_wr *wr) { aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap); aseg->swap_add_mask = cpu_to_be64(wr->wr.atomic.swap_mask); aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add); aseg->compare_mask = cpu_to_be64(wr->wr.atomic.compare_add_mask); } static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg, - struct ib_send_wr *wr, __be16 *vlan) + struct ib_send_wr *wr) { memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av)); dseg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn); dseg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey); dseg->vlan = to_mah(wr->wr.ud.ah)->av.eth.vlan; memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->av.eth.mac, 6); - *vlan = dseg->vlan; } +static void set_tunnel_datagram_seg(struct mlx4_ib_dev *dev, + struct mlx4_wqe_datagram_seg *dseg, + struct ib_send_wr *wr, enum ib_qp_type qpt) +{ + union mlx4_ext_av *av = &to_mah(wr->wr.ud.ah)->av; + struct mlx4_av sqp_av = {0}; + int port = *((u8 *) &av->ib.port_pd) & 0x3; + + /* force loopback */ + sqp_av.port_pd = av->ib.port_pd | cpu_to_be32(0x80000000); + sqp_av.g_slid = av->ib.g_slid & 0x7f; /* no GRH */ + sqp_av.sl_tclass_flowlabel = av->ib.sl_tclass_flowlabel & + cpu_to_be32(0xf0000000); + + memcpy(dseg->av, &sqp_av, sizeof (struct mlx4_av)); + /* This function used only for sending on QP1 proxies */ + dseg->dqpn = cpu_to_be32(dev->dev->caps.qp1_tunnel[port - 1]); + /* Use QKEY from the QP context, which is set by master */ + dseg->qkey = cpu_to_be32(IB_QP_SET_QKEY); +} + +static void build_tunnel_header(struct ib_send_wr *wr, void *wqe, unsigned *mlx_seg_len) +{ + struct mlx4_wqe_inline_seg *inl = wqe; + struct mlx4_ib_tunnel_header hdr; + struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah); + int spc; + int i; + + memcpy(&hdr.av, &ah->av, sizeof hdr.av); + hdr.remote_qpn = cpu_to_be32(wr->wr.ud.remote_qpn); + hdr.pkey_index = cpu_to_be16(wr->wr.ud.pkey_index); + hdr.qkey = cpu_to_be32(wr->wr.ud.remote_qkey); + + spc = MLX4_INLINE_ALIGN - + ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1)); + if (sizeof (hdr) <= spc) { + memcpy(inl + 1, &hdr, sizeof (hdr)); + wmb(); + inl->byte_count = cpu_to_be32(1U << 31 | sizeof (hdr)); + i = 1; + } else { + memcpy(inl + 1, &hdr, spc); + wmb(); + inl->byte_count = cpu_to_be32(1U << 31 | spc); + + inl = (void *) (inl + 1) + spc; + memcpy(inl + 1, (void *) &hdr + spc, sizeof (hdr) - spc); + wmb(); + inl->byte_count = cpu_to_be32(1U << 31 | (sizeof (hdr) - spc)); + i = 2; + } + + *mlx_seg_len = + ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + sizeof (hdr), 16); +} + static void set_mlx_icrc_seg(void *dseg) { u32 *t = dseg; struct mlx4_wqe_inline_seg *iseg = dseg; t[1] = 0; /* * Need a barrier here before writing the byte_count field to * make sure that all the data is visible before the * byte_count field is set. Otherwise, if the segment begins * a new cacheline, the HCA prefetcher could grab the 64-byte * chunk and get a valid (!= * 0xffffffff) byte count but * stale data, and end up sending the wrong data. */ wmb(); iseg->byte_count = cpu_to_be32((1 << 31) | 4); } static void set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg) { dseg->lkey = cpu_to_be32(sg->lkey); dseg->addr = cpu_to_be64(sg->addr); /* * Need a barrier here before writing the byte_count field to * make sure that all the data is visible before the * byte_count field is set. Otherwise, if the segment begins * a new cacheline, the HCA prefetcher could grab the 64-byte * chunk and get a valid (!= * 0xffffffff) byte count but * stale data, and end up sending the wrong data. */ wmb(); dseg->byte_count = cpu_to_be32(sg->length); } static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg) { dseg->byte_count = cpu_to_be32(sg->length); dseg->lkey = cpu_to_be32(sg->lkey); dseg->addr = cpu_to_be64(sg->addr); } static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_send_wr *wr, struct mlx4_ib_qp *qp, unsigned *lso_seg_len, - __be32 *lso_hdr_sz, int *blh) + __be32 *lso_hdr_sz, __be32 *blh) { unsigned halign = ALIGN(sizeof *wqe + wr->wr.ud.hlen, 16); - *blh = unlikely(halign > 64) ? 1 : 0; + if (unlikely(halign > MLX4_IB_CACHE_LINE_SIZE)) + *blh = cpu_to_be32(1 << 6); if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) && wr->num_sge > qp->sq.max_gs - (halign >> 4))) return -EINVAL; memcpy(wqe->header, wr->wr.ud.header, wr->wr.ud.hlen); *lso_hdr_sz = cpu_to_be32((wr->wr.ud.mss - wr->wr.ud.hlen) << 16 | wr->wr.ud.hlen); *lso_seg_len = halign; return 0; } static __be32 send_ieth(struct ib_send_wr *wr) { switch (wr->opcode) { case IB_WR_SEND_WITH_IMM: case IB_WR_RDMA_WRITE_WITH_IMM: return wr->ex.imm_data; case IB_WR_SEND_WITH_INV: return cpu_to_be32(wr->ex.invalidate_rkey); default: return 0; } } +static void add_zero_len_inline(void *wqe) +{ + struct mlx4_wqe_inline_seg *inl = wqe; + memset(wqe, 0, 16); + inl->byte_count = cpu_to_be32(1 << 31); +} + static int lay_inline_data(struct mlx4_ib_qp *qp, struct ib_send_wr *wr, void *wqe, int *sz) { struct mlx4_wqe_inline_seg *seg; void *addr; int len, seg_len; int num_seg; int off, to_copy; int i; int inl = 0; seg = wqe; wqe += sizeof *seg; off = ((unsigned long)wqe) & (unsigned long)(MLX4_INLINE_ALIGN - 1); num_seg = 0; seg_len = 0; for (i = 0; i < wr->num_sge; ++i) { addr = (void *) (unsigned long)(wr->sg_list[i].addr); len = wr->sg_list[i].length; inl += len; if (inl > qp->max_inline_data) { inl = 0; return -1; } while (len >= MLX4_INLINE_ALIGN - off) { to_copy = MLX4_INLINE_ALIGN - off; memcpy(wqe, addr, to_copy); len -= to_copy; wqe += to_copy; addr += to_copy; seg_len += to_copy; wmb(); /* see comment below */ seg->byte_count = htonl(MLX4_INLINE_SEG | seg_len); seg_len = 0; seg = wqe; wqe += sizeof *seg; off = sizeof *seg; ++num_seg; } memcpy(wqe, addr, len); wqe += len; seg_len += len; off += len; } if (seg_len) { ++num_seg; /* * Need a barrier here to make sure * all the data is visible before the * byte_count field is set. Otherwise * the HCA prefetcher could grab the * 64-byte chunk with this inline * segment and get a valid (!= * 0xffffffff) byte count but stale * data, and end up sending the wrong * data. */ wmb(); seg->byte_count = htonl(MLX4_INLINE_SEG | seg_len); } *sz = (inl + num_seg * sizeof *seg + 15) / 16; return 0; } /* * Avoid using memcpy() to copy to BlueFlame page, since memcpy() * implementations may use move-string-buffer assembler instructions, * which do not guarantee order of copying. */ -static void mlx4_bf_copy(unsigned long *dst, unsigned long *src, unsigned bytecnt) +static void mlx4_bf_copy(unsigned long *dst, unsigned long *src, + unsigned bytecnt) { __iowrite64_copy(dst, src, bytecnt / 8); } int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, struct ib_send_wr **bad_wr) { struct mlx4_ib_qp *qp = to_mqp(ibqp); void *wqe; - struct mlx4_wqe_ctrl_seg *ctrl; + struct mlx4_wqe_ctrl_seg *uninitialized_var(ctrl); struct mlx4_wqe_data_seg *dseg; unsigned long flags; int nreq; int err = 0; unsigned ind; int uninitialized_var(stamp); int uninitialized_var(size); unsigned uninitialized_var(seglen); __be32 dummy; __be32 *lso_wqe; __be32 uninitialized_var(lso_hdr_sz); + __be32 blh; int i; - int blh = 0; - __be16 vlan = 0; int inl = 0; - - ctrl = NULL; spin_lock_irqsave(&qp->sq.lock, flags); ind = qp->sq_next_wqe; for (nreq = 0; wr; ++nreq, wr = wr->next) { lso_wqe = &dummy; + blh = 0; if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { - mlx4_ib_dbg("QP 0x%x: WQE overflow", ibqp->qp_num); err = -ENOMEM; *bad_wr = wr; goto out; } if (unlikely(wr->num_sge > qp->sq.max_gs)) { - mlx4_ib_dbg("QP 0x%x: too many sg entries (%d)", - ibqp->qp_num, wr->num_sge); err = -EINVAL; *bad_wr = wr; goto out; } ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); *((u32 *) (&ctrl->vlan_tag)) = 0; qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id; ctrl->srcrb_flags = (wr->send_flags & IB_SEND_SIGNALED ? cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) | (wr->send_flags & IB_SEND_SOLICITED ? cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) | ((wr->send_flags & IB_SEND_IP_CSUM) ? cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM | MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) | qp->sq_signal_bits; ctrl->imm = send_ieth(wr); wqe += sizeof *ctrl; size = sizeof *ctrl / 16; - switch (ibqp->qp_type) { - case IB_QPT_XRC: - ctrl->srcrb_flags |= - cpu_to_be32(wr->xrc_remote_srq_num << 8); - /* fall thru */ - case IB_QPT_RC: - case IB_QPT_UC: + switch (qp->mlx4_ib_qp_type) { + case MLX4_IB_QPT_RC: + case MLX4_IB_QPT_UC: switch (wr->opcode) { case IB_WR_ATOMIC_CMP_AND_SWP: case IB_WR_ATOMIC_FETCH_AND_ADD: case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD: set_raddr_seg(wqe, wr->wr.atomic.remote_addr, wr->wr.atomic.rkey); wqe += sizeof (struct mlx4_wqe_raddr_seg); set_atomic_seg(wqe, wr); wqe += sizeof (struct mlx4_wqe_atomic_seg); size += (sizeof (struct mlx4_wqe_raddr_seg) + sizeof (struct mlx4_wqe_atomic_seg)) / 16; break; case IB_WR_MASKED_ATOMIC_CMP_AND_SWP: set_raddr_seg(wqe, wr->wr.atomic.remote_addr, wr->wr.atomic.rkey); wqe += sizeof (struct mlx4_wqe_raddr_seg); set_masked_atomic_seg(wqe, wr); wqe += sizeof (struct mlx4_wqe_masked_atomic_seg); size += (sizeof (struct mlx4_wqe_raddr_seg) + sizeof (struct mlx4_wqe_masked_atomic_seg)) / 16; break; case IB_WR_RDMA_READ: case IB_WR_RDMA_WRITE: case IB_WR_RDMA_WRITE_WITH_IMM: set_raddr_seg(wqe, wr->wr.rdma.remote_addr, wr->wr.rdma.rkey); wqe += sizeof (struct mlx4_wqe_raddr_seg); size += sizeof (struct mlx4_wqe_raddr_seg) / 16; break; case IB_WR_LOCAL_INV: ctrl->srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER); set_local_inv_seg(wqe, wr->ex.invalidate_rkey); wqe += sizeof (struct mlx4_wqe_local_inval_seg); size += sizeof (struct mlx4_wqe_local_inval_seg) / 16; break; case IB_WR_FAST_REG_MR: ctrl->srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER); set_fmr_seg(wqe, wr); wqe += sizeof (struct mlx4_wqe_fmr_seg); size += sizeof (struct mlx4_wqe_fmr_seg) / 16; break; default: /* No extra segments required for sends */ break; } break; - case IB_QPT_UD: - set_datagram_seg(wqe, wr, &vlan); + case MLX4_IB_QPT_TUN_SMI_OWNER: + err = build_sriov_qp0_header(to_msqp(qp), wr, ctrl, &seglen); + if (unlikely(err)) { + *bad_wr = wr; + goto out; + } + wqe += seglen; + size += seglen / 16; + break; + case MLX4_IB_QPT_TUN_SMI: + case MLX4_IB_QPT_TUN_GSI: + /* this is a UD qp used in MAD responses to slaves. */ + set_datagram_seg(wqe, wr); + /* set the forced-loopback bit in the data seg av */ + *(__be32 *) wqe |= cpu_to_be32(0x80000000); wqe += sizeof (struct mlx4_wqe_datagram_seg); size += sizeof (struct mlx4_wqe_datagram_seg) / 16; + break; + case MLX4_IB_QPT_UD: + set_datagram_seg(wqe, wr); + wqe += sizeof (struct mlx4_wqe_datagram_seg); + size += sizeof (struct mlx4_wqe_datagram_seg) / 16; if (wr->opcode == IB_WR_LSO) { err = build_lso_seg(wqe, wr, qp, &seglen, &lso_hdr_sz, &blh); if (unlikely(err)) { *bad_wr = wr; goto out; } lso_wqe = (__be32 *) wqe; wqe += seglen; size += seglen / 16; } break; - case IB_QPT_SMI: - case IB_QPT_GSI: - err = build_mlx_header(to_msqp(qp), wr, ctrl, &seglen); + case MLX4_IB_QPT_PROXY_SMI_OWNER: + if (unlikely(!mlx4_is_master(to_mdev(ibqp->device)->dev))) { + err = -ENOSYS; + *bad_wr = wr; + goto out; + } + err = build_sriov_qp0_header(to_msqp(qp), wr, ctrl, &seglen); if (unlikely(err)) { *bad_wr = wr; goto out; } wqe += seglen; size += seglen / 16; + /* to start tunnel header on a cache-line boundary */ + add_zero_len_inline(wqe); + wqe += 16; + size++; + build_tunnel_header(wr, wqe, &seglen); + wqe += seglen; + size += seglen / 16; break; + case MLX4_IB_QPT_PROXY_SMI: + /* don't allow QP0 sends on guests */ + err = -ENOSYS; + *bad_wr = wr; + goto out; + case MLX4_IB_QPT_PROXY_GSI: + /* If we are tunneling special qps, this is a UD qp. + * In this case we first add a UD segment targeting + * the tunnel qp, and then add a header with address + * information */ + set_tunnel_datagram_seg(to_mdev(ibqp->device), wqe, wr, ibqp->qp_type); + wqe += sizeof (struct mlx4_wqe_datagram_seg); + size += sizeof (struct mlx4_wqe_datagram_seg) / 16; + build_tunnel_header(wr, wqe, &seglen); + wqe += seglen; + size += seglen / 16; + break; - case IB_QPT_RAW_ETY: - err = build_raw_ety_header(to_msqp(qp), wr, ctrl, - &seglen); + case MLX4_IB_QPT_SMI: + case MLX4_IB_QPT_GSI: + err = build_mlx_header(to_msqp(qp), wr, ctrl, &seglen); if (unlikely(err)) { *bad_wr = wr; goto out; } wqe += seglen; size += seglen / 16; break; default: break; } /* * Write data segments in reverse order, so as to * overwrite cacheline stamp last within each * cacheline. This avoids issues with WQE * prefetching. */ - dseg = wqe; dseg += wr->num_sge - 1; /* Add one more inline data segment for ICRC for MLX sends */ - if (unlikely(qp->ibqp.qp_type == IB_QPT_SMI || - qp->ibqp.qp_type == IB_QPT_GSI)) { + if (unlikely(qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI || + qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI || + qp->mlx4_ib_qp_type & + (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))) { set_mlx_icrc_seg(dseg + 1); size += sizeof (struct mlx4_wqe_data_seg) / 16; } if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) { int sz; err = lay_inline_data(qp, wr, wqe, &sz); if (!err) { inl = 1; size += sz; } } else { - size += wr->num_sge * (sizeof (struct mlx4_wqe_data_seg) / 16); + size += wr->num_sge * + (sizeof(struct mlx4_wqe_data_seg) / 16); for (i = wr->num_sge - 1; i >= 0; --i, --dseg) set_data_seg(dseg, wr->sg_list + i); } /* * Possibly overwrite stamping in cacheline with LSO * segment only after making sure all data segments * are written. */ wmb(); *lso_wqe = lso_hdr_sz; - ctrl->fence_size = (wr->send_flags & IB_SEND_FENCE ? MLX4_WQE_CTRL_FENCE : 0) | size; - if (vlan) { - ctrl->ins_vlan = 1 << 6; - ctrl->vlan_tag = vlan; - } - /* * Make sure descriptor is fully written before * setting ownership bit (because HW can start * executing as soon as we do). */ wmb(); - if (wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) { + if (wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) { + *bad_wr = wr; err = -EINVAL; goto out; } ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] | - (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0) | - (blh ? cpu_to_be32(1 << 6) : 0); + (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0) | blh; stamp = ind + qp->sq_spare_wqes; ind += DIV_ROUND_UP(size * 16, 1U << qp->sq.wqe_shift); /* * We can improve latency by not stamping the last * send queue WQE until after ringing the doorbell, so * only stamp here if there are still more WQEs to post. * * Same optimization applies to padding with NOP wqe * in case of WQE shrinking (used to prevent wrap-around * in the middle of WR). */ if (wr->next) { stamp_send_wqe(qp, stamp, size * 16); ind = pad_wraparound(qp, ind); } } out: if (nreq == 1 && inl && size > 1 && size < qp->bf.buf_size / 16) { ctrl->owner_opcode |= htonl((qp->sq_next_wqe & 0xffff) << 8); + /* We set above doorbell_qpn bits to 0 as part of vlan + * tag initialization, so |= should be correct. + */ *(u32 *) (&ctrl->vlan_tag) |= qp->doorbell_qpn; /* * Make sure that descriptor is written to memory * before writing to BlueFlame page. */ wmb(); ++qp->sq.head; mlx4_bf_copy(qp->bf.reg + qp->bf.offset, (unsigned long *) ctrl, ALIGN(size * 16, 64)); wc_wmb(); qp->bf.offset ^= qp->bf.buf_size; } else if (nreq) { qp->sq.head += nreq; /* * Make sure that descriptors are written before * doorbell record. */ wmb(); writel(qp->doorbell_qpn, qp->bf.uar->map + MLX4_SEND_DOORBELL); /* * Make sure doorbells don't leak out of SQ spinlock * and reach the HCA out of order. */ mmiowb(); } if (likely(nreq)) { stamp_send_wqe(qp, stamp, size * 16); ind = pad_wraparound(qp, ind); qp->sq_next_wqe = ind; } spin_unlock_irqrestore(&qp->sq.lock, flags); return err; } int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, struct ib_recv_wr **bad_wr) { struct mlx4_ib_qp *qp = to_mqp(ibqp); struct mlx4_wqe_data_seg *scat; unsigned long flags; int err = 0; int nreq; int ind; + int max_gs; int i; + max_gs = qp->rq.max_gs; spin_lock_irqsave(&qp->rq.lock, flags); ind = qp->rq.head & (qp->rq.wqe_cnt - 1); for (nreq = 0; wr; ++nreq, wr = wr->next) { if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) { - mlx4_ib_dbg("QP 0x%x: WQE overflow", ibqp->qp_num); err = -ENOMEM; *bad_wr = wr; goto out; } if (unlikely(wr->num_sge > qp->rq.max_gs)) { - mlx4_ib_dbg("QP 0x%x: too many sg entries (%d)", - ibqp->qp_num, wr->num_sge); err = -EINVAL; *bad_wr = wr; goto out; } scat = get_recv_wqe(qp, ind); + if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER | + MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) { + ib_dma_sync_single_for_device(ibqp->device, + qp->sqp_proxy_rcv[ind].map, + sizeof (struct mlx4_ib_proxy_sqp_hdr), + DMA_FROM_DEVICE); + scat->byte_count = + cpu_to_be32(sizeof (struct mlx4_ib_proxy_sqp_hdr)); + /* use dma lkey from upper layer entry */ + scat->lkey = cpu_to_be32(wr->sg_list->lkey); + scat->addr = cpu_to_be64(qp->sqp_proxy_rcv[ind].map); + scat++; + max_gs--; + } + for (i = 0; i < wr->num_sge; ++i) __set_data_seg(scat + i, wr->sg_list + i); - if (i < qp->rq.max_gs) { + if (i < max_gs) { scat[i].byte_count = 0; scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY); scat[i].addr = 0; } qp->rq.wrid[ind] = wr->wr_id; ind = (ind + 1) & (qp->rq.wqe_cnt - 1); } out: if (likely(nreq)) { qp->rq.head += nreq; /* * Make sure that descriptors are written before * doorbell record. */ wmb(); *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff); } spin_unlock_irqrestore(&qp->rq.lock, flags); return err; } static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state) { switch (mlx4_state) { case MLX4_QP_STATE_RST: return IB_QPS_RESET; case MLX4_QP_STATE_INIT: return IB_QPS_INIT; case MLX4_QP_STATE_RTR: return IB_QPS_RTR; case MLX4_QP_STATE_RTS: return IB_QPS_RTS; case MLX4_QP_STATE_SQ_DRAINING: case MLX4_QP_STATE_SQD: return IB_QPS_SQD; case MLX4_QP_STATE_SQER: return IB_QPS_SQE; case MLX4_QP_STATE_ERR: return IB_QPS_ERR; default: return -1; } } static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state) { switch (mlx4_mig_state) { case MLX4_QP_PM_ARMED: return IB_MIG_ARMED; case MLX4_QP_PM_REARM: return IB_MIG_REARM; case MLX4_QP_PM_MIGRATED: return IB_MIG_MIGRATED; default: return -1; } } static int to_ib_qp_access_flags(int mlx4_flags) { int ib_flags = 0; if (mlx4_flags & MLX4_QP_BIT_RRE) ib_flags |= IB_ACCESS_REMOTE_READ; if (mlx4_flags & MLX4_QP_BIT_RWE) ib_flags |= IB_ACCESS_REMOTE_WRITE; if (mlx4_flags & MLX4_QP_BIT_RAE) ib_flags |= IB_ACCESS_REMOTE_ATOMIC; return ib_flags; } -static void to_ib_ah_attr(struct mlx4_ib_dev *ib_dev, struct ib_ah_attr *ib_ah_attr, - struct mlx4_qp_path *path) +static void to_ib_ah_attr(struct mlx4_ib_dev *ibdev, struct ib_ah_attr *ib_ah_attr, + struct mlx4_qp_path *path) { - struct mlx4_dev *dev = ib_dev->dev; + struct mlx4_dev *dev = ibdev->dev; int is_eth; memset(ib_ah_attr, 0, sizeof *ib_ah_attr); ib_ah_attr->port_num = path->sched_queue & 0x40 ? 2 : 1; if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->caps.num_ports) return; - is_eth = rdma_port_get_link_layer(&ib_dev->ib_dev, ib_ah_attr->port_num) == + is_eth = rdma_port_get_link_layer(&ibdev->ib_dev, ib_ah_attr->port_num) == IB_LINK_LAYER_ETHERNET; if (is_eth) ib_ah_attr->sl = ((path->sched_queue >> 3) & 0x7) | ((path->sched_queue & 4) << 1); else ib_ah_attr->sl = (path->sched_queue >> 2) & 0xf; ib_ah_attr->dlid = be16_to_cpu(path->rlid); - ib_ah_attr->src_path_bits = path->grh_mylmc & 0x7f; ib_ah_attr->static_rate = path->static_rate ? path->static_rate - 5 : 0; ib_ah_attr->ah_flags = (path->grh_mylmc & (1 << 7)) ? IB_AH_GRH : 0; if (ib_ah_attr->ah_flags) { ib_ah_attr->grh.sgid_index = path->mgid_index; ib_ah_attr->grh.hop_limit = path->hop_limit; ib_ah_attr->grh.traffic_class = (be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff; ib_ah_attr->grh.flow_label = be32_to_cpu(path->tclass_flowlabel) & 0xfffff; memcpy(ib_ah_attr->grh.dgid.raw, path->rgid, sizeof ib_ah_attr->grh.dgid.raw); } } int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) { struct mlx4_ib_dev *dev = to_mdev(ibqp->device); struct mlx4_ib_qp *qp = to_mqp(ibqp); struct mlx4_qp_context context; int mlx4_state; int err = 0; mutex_lock(&qp->mutex); if (qp->state == IB_QPS_RESET) { qp_attr->qp_state = IB_QPS_RESET; goto done; } err = mlx4_qp_query(dev->dev, &qp->mqp, &context); if (err) { err = -EINVAL; goto out; } mlx4_state = be32_to_cpu(context.flags) >> 28; qp->state = to_ib_qp_state(mlx4_state); qp_attr->qp_state = qp->state; qp_attr->path_mtu = context.mtu_msgmax >> 5; qp_attr->path_mig_state = to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3); qp_attr->qkey = be32_to_cpu(context.qkey); qp_attr->rq_psn = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff; qp_attr->sq_psn = be32_to_cpu(context.next_send_psn) & 0xffffff; qp_attr->dest_qp_num = be32_to_cpu(context.remote_qpn) & 0xffffff; qp_attr->qp_access_flags = to_ib_qp_access_flags(be32_to_cpu(context.params2)); - if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC || - qp->ibqp.qp_type == IB_QPT_XRC) { + if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) { to_ib_ah_attr(dev, &qp_attr->ah_attr, &context.pri_path); to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context.alt_path); qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f; qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num; } qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f; if (qp_attr->qp_state == IB_QPS_INIT) qp_attr->port_num = qp->port; else qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1; /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ qp_attr->sq_draining = mlx4_state == MLX4_QP_STATE_SQ_DRAINING; qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context.params1) >> 21) & 0x7); qp_attr->max_dest_rd_atomic = 1 << ((be32_to_cpu(context.params2) >> 21) & 0x7); qp_attr->min_rnr_timer = (be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f; qp_attr->timeout = context.pri_path.ackto >> 3; qp_attr->retry_cnt = (be32_to_cpu(context.params1) >> 16) & 0x7; qp_attr->rnr_retry = (be32_to_cpu(context.params1) >> 13) & 0x7; qp_attr->alt_timeout = context.alt_path.ackto >> 3; done: qp_attr->cur_qp_state = qp_attr->qp_state; qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt; qp_attr->cap.max_recv_sge = qp->rq.max_gs; if (!ibqp->uobject) { qp_attr->cap.max_send_wr = qp->sq.wqe_cnt; qp_attr->cap.max_send_sge = qp->sq.max_gs; } else { qp_attr->cap.max_send_wr = 0; qp_attr->cap.max_send_sge = 0; } /* * We don't support inline sends for kernel QPs (yet), and we * don't know what userspace's value should be. */ qp_attr->cap.max_inline_data = 0; qp_init_attr->cap = qp_attr->cap; qp_init_attr->create_flags = 0; if (qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK; if (qp->flags & MLX4_IB_QP_LSO) qp_init_attr->create_flags |= IB_QP_CREATE_IPOIB_UD_LSO; -out: - mutex_unlock(&qp->mutex); - return err; -} + if (qp->flags & MLX4_IB_QP_NETIF) + qp_init_attr->create_flags |= IB_QP_CREATE_NETIF_QP; -int mlx4_ib_create_xrc_rcv_qp(struct ib_qp_init_attr *init_attr, - u32 *qp_num) -{ - struct mlx4_ib_dev *dev = to_mdev(init_attr->xrc_domain->device); - struct mlx4_ib_xrcd *xrcd = to_mxrcd(init_attr->xrc_domain); - struct mlx4_ib_qp *qp; - struct ib_qp *ibqp; - struct mlx4_ib_xrc_reg_entry *ctx_entry; - unsigned long flags; - int err; + qp_init_attr->sq_sig_type = + qp->sq_signal_bits == cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) ? + IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; - if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) - return -ENOSYS; - - if (init_attr->qp_type != IB_QPT_XRC) - return -EINVAL; - - ctx_entry = kmalloc(sizeof *ctx_entry, GFP_KERNEL); - if (!ctx_entry) - return -ENOMEM; - - qp = kzalloc(sizeof *qp, GFP_KERNEL); - if (!qp) { - kfree(ctx_entry); - return -ENOMEM; - } - mutex_lock(&dev->xrc_reg_mutex); - qp->flags = MLX4_IB_XRC_RCV; - qp->xrcdn = to_mxrcd(init_attr->xrc_domain)->xrcdn; - INIT_LIST_HEAD(&qp->xrc_reg_list); - err = create_qp_common(dev, xrcd->pd, init_attr, NULL, 0, qp); - if (err) { - mutex_unlock(&dev->xrc_reg_mutex); - kfree(ctx_entry); - kfree(qp); - return err; - } - - ibqp = &qp->ibqp; - /* set the ibpq attributes which will be used by the mlx4 module */ - ibqp->qp_num = qp->mqp.qpn; - ibqp->device = init_attr->xrc_domain->device; - ibqp->pd = xrcd->pd; - ibqp->send_cq = ibqp->recv_cq = xrcd->cq; - ibqp->event_handler = init_attr->event_handler; - ibqp->qp_context = init_attr->qp_context; - ibqp->qp_type = init_attr->qp_type; - ibqp->xrcd = init_attr->xrc_domain; - - mutex_lock(&qp->mutex); - ctx_entry->context = init_attr->qp_context; - spin_lock_irqsave(&qp->xrc_reg_list_lock, flags); - list_add_tail(&ctx_entry->list, &qp->xrc_reg_list); - spin_unlock_irqrestore(&qp->xrc_reg_list_lock, flags); - mutex_unlock(&qp->mutex); - mutex_unlock(&dev->xrc_reg_mutex); - *qp_num = qp->mqp.qpn; - return 0; -} - -int mlx4_ib_modify_xrc_rcv_qp(struct ib_xrcd *ibxrcd, u32 qp_num, - struct ib_qp_attr *attr, int attr_mask) -{ - struct mlx4_ib_dev *dev = to_mdev(ibxrcd->device); - struct mlx4_ib_xrcd *xrcd = to_mxrcd(ibxrcd); - struct mlx4_qp *mqp; - struct mlx4_ib_qp *mibqp; - int err = -EINVAL; - - if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) - return -ENOSYS; - - mutex_lock(&dev->xrc_reg_mutex); - mqp = mlx4_qp_lookup_lock(dev->dev, qp_num); - if (unlikely(!mqp)) { - printk(KERN_WARNING "mlx4_ib_reg_xrc_rcv_qp: " - "unknown QPN %06x\n", qp_num); - goto err_out; - } - - mibqp = to_mibqp(mqp); - - if (!(mibqp->flags & MLX4_IB_XRC_RCV) || !mibqp->ibqp.xrcd || - xrcd->xrcdn != to_mxrcd(mibqp->ibqp.xrcd)->xrcdn) - goto err_out; - - err = mlx4_ib_modify_qp(&mibqp->ibqp, attr, attr_mask, NULL); - mutex_unlock(&dev->xrc_reg_mutex); - return err; - -err_out: - mutex_unlock(&dev->xrc_reg_mutex); - return err; -} - -int mlx4_ib_query_xrc_rcv_qp(struct ib_xrcd *ibxrcd, u32 qp_num, - struct ib_qp_attr *qp_attr, int qp_attr_mask, - struct ib_qp_init_attr *qp_init_attr) -{ - struct mlx4_ib_dev *dev = to_mdev(ibxrcd->device); - struct mlx4_ib_xrcd *xrcd = to_mxrcd(ibxrcd); - struct mlx4_ib_qp *qp; - struct mlx4_qp *mqp; - struct mlx4_qp_context context; - int mlx4_state; - int err = -EINVAL; - - if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)) - return -ENOSYS; - - mutex_lock(&dev->xrc_reg_mutex); - mqp = mlx4_qp_lookup_lock(dev->dev, qp_num); - if (unlikely(!mqp)) { - printk(KERN_WARNING "mlx4_ib_reg_xrc_rcv_qp: " - "unknown QPN %06x\n", qp_num); - goto err_out; - } - - qp = to_mibqp(mqp); - if (!(qp->flags & MLX4_IB_XRC_RCV) || !(qp->ibqp.xrcd) || - xrcd->xrcdn != to_mxrcd(qp->ibqp.xrcd)->xrcdn) - goto err_out; - - if (qp->state == IB_QPS_RESET) { - qp_attr->qp_state = IB_QPS_RESET; - goto done; - } - - err = mlx4_qp_query(dev->dev, mqp, &context); - if (err) - goto err_out; - - mlx4_state = be32_to_cpu(context.flags) >> 28; - - qp_attr->qp_state = to_ib_qp_state(mlx4_state); - qp_attr->path_mtu = context.mtu_msgmax >> 5; - qp_attr->path_mig_state = - to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3); - qp_attr->qkey = be32_to_cpu(context.qkey); - qp_attr->rq_psn = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff; - qp_attr->sq_psn = be32_to_cpu(context.next_send_psn) & 0xffffff; - qp_attr->dest_qp_num = be32_to_cpu(context.remote_qpn) & 0xffffff; - qp_attr->qp_access_flags = - to_ib_qp_access_flags(be32_to_cpu(context.params2)); - - if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC || - qp->ibqp.qp_type == IB_QPT_XRC) { - to_ib_ah_attr(dev, &qp_attr->ah_attr, &context.pri_path); - to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, - &context.alt_path); - qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f; - qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num; - } - - qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f; - if (qp_attr->qp_state == IB_QPS_INIT) - qp_attr->port_num = qp->port; + qp_init_attr->qpg_type = ibqp->qpg_type; + if (ibqp->qpg_type == IB_QPG_PARENT) + qp_init_attr->cap.qpg_tss_mask_sz = qp->qpg_data->qpg_tss_mask_sz; else - qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1; + qp_init_attr->cap.qpg_tss_mask_sz = 0; - /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ - qp_attr->sq_draining = mlx4_state == MLX4_QP_STATE_SQ_DRAINING; - - qp_attr->max_rd_atomic = - 1 << ((be32_to_cpu(context.params1) >> 21) & 0x7); - - qp_attr->max_dest_rd_atomic = - 1 << ((be32_to_cpu(context.params2) >> 21) & 0x7); - qp_attr->min_rnr_timer = - (be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f; - qp_attr->timeout = context.pri_path.ackto >> 3; - qp_attr->retry_cnt = (be32_to_cpu(context.params1) >> 16) & 0x7; - qp_attr->rnr_retry = (be32_to_cpu(context.params1) >> 13) & 0x7; - qp_attr->alt_timeout = context.alt_path.ackto >> 3; - -done: - qp_attr->cur_qp_state = qp_attr->qp_state; - qp_attr->cap.max_recv_wr = 0; - qp_attr->cap.max_recv_sge = 0; - qp_attr->cap.max_send_wr = 0; - qp_attr->cap.max_send_sge = 0; - qp_attr->cap.max_inline_data = 0; - qp_init_attr->cap = qp_attr->cap; - - mutex_unlock(&dev->xrc_reg_mutex); - return 0; - -err_out: - mutex_unlock(&dev->xrc_reg_mutex); - return err; -} - -int mlx4_ib_reg_xrc_rcv_qp(struct ib_xrcd *xrcd, void *context, u32 qp_num) -{ - - struct mlx4_ib_xrcd *mxrcd = to_mxrcd(xrcd); - - struct mlx4_qp *mqp; - struct mlx4_ib_qp *mibqp; - struct mlx4_ib_xrc_reg_entry *ctx_entry, *tmp; - unsigned long flags; - int err = -EINVAL; - - mutex_lock(&to_mdev(xrcd->device)->xrc_reg_mutex); - mqp = mlx4_qp_lookup_lock(to_mdev(xrcd->device)->dev, qp_num); - if (unlikely(!mqp)) { - printk(KERN_WARNING "mlx4_ib_reg_xrc_rcv_qp: " - "unknown QPN %06x\n", qp_num); - goto err_out; - } - - mibqp = to_mibqp(mqp); - - if (!(mibqp->flags & MLX4_IB_XRC_RCV) || !(mibqp->ibqp.xrcd) || - mxrcd->xrcdn != to_mxrcd(mibqp->ibqp.xrcd)->xrcdn) - goto err_out; - - ctx_entry = kmalloc(sizeof *ctx_entry, GFP_KERNEL); - if (!ctx_entry) { - err = -ENOMEM; - goto err_out; - } - - mutex_lock(&mibqp->mutex); - list_for_each_entry(tmp, &mibqp->xrc_reg_list, list) - if (tmp->context == context) { - mutex_unlock(&mibqp->mutex); - kfree(ctx_entry); - mutex_unlock(&to_mdev(xrcd->device)->xrc_reg_mutex); - return 0; - } - - ctx_entry->context = context; - spin_lock_irqsave(&mibqp->xrc_reg_list_lock, flags); - list_add_tail(&ctx_entry->list, &mibqp->xrc_reg_list); - spin_unlock_irqrestore(&mibqp->xrc_reg_list_lock, flags); - mutex_unlock(&mibqp->mutex); - mutex_unlock(&to_mdev(xrcd->device)->xrc_reg_mutex); - return 0; - -err_out: - mutex_unlock(&to_mdev(xrcd->device)->xrc_reg_mutex); - return err; -} - -int mlx4_ib_unreg_xrc_rcv_qp(struct ib_xrcd *xrcd, void *context, u32 qp_num) -{ - - struct mlx4_ib_xrcd *mxrcd = to_mxrcd(xrcd); - - struct mlx4_qp *mqp; - struct mlx4_ib_qp *mibqp; - struct mlx4_ib_xrc_reg_entry *ctx_entry, *tmp; - unsigned long flags; - int found = 0; - int err = -EINVAL; - - mutex_lock(&to_mdev(xrcd->device)->xrc_reg_mutex); - mqp = mlx4_qp_lookup_lock(to_mdev(xrcd->device)->dev, qp_num); - if (unlikely(!mqp)) { - printk(KERN_WARNING "mlx4_ib_unreg_xrc_rcv_qp: " - "unknown QPN %06x\n", qp_num); - goto err_out; - } - - mibqp = to_mibqp(mqp); - - if (!(mibqp->flags & MLX4_IB_XRC_RCV) || - mxrcd->xrcdn != (mibqp->xrcdn & 0xffff)) - goto err_out; - - mutex_lock(&mibqp->mutex); - spin_lock_irqsave(&mibqp->xrc_reg_list_lock, flags); - list_for_each_entry_safe(ctx_entry, tmp, &mibqp->xrc_reg_list, list) - if (ctx_entry->context == context) { - found = 1; - list_del(&ctx_entry->list); - spin_unlock_irqrestore(&mibqp->xrc_reg_list_lock, flags); - kfree(ctx_entry); - break; - } - - if (!found) - spin_unlock_irqrestore(&mibqp->xrc_reg_list_lock, flags); - mutex_unlock(&mibqp->mutex); - if (!found) - goto err_out; - - /* destroy the QP if the registration list is empty */ - if (list_empty(&mibqp->xrc_reg_list)) - mlx4_ib_destroy_qp(&mibqp->ibqp); - - mutex_unlock(&to_mdev(xrcd->device)->xrc_reg_mutex); - return 0; - -err_out: - mutex_unlock(&to_mdev(xrcd->device)->xrc_reg_mutex); +out: + mutex_unlock(&qp->mutex); return err; } Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/srq.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/srq.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/srq.c (revision 273864) @@ -1,401 +1,368 @@ /* * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include +#include #include "mlx4_ib.h" #include "user.h" static void *get_wqe(struct mlx4_ib_srq *srq, int n) { return mlx4_buf_offset(&srq->buf, n << srq->msrq.wqe_shift); } static void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type) { struct ib_event event; struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; if (ibsrq->event_handler) { event.device = ibsrq->device; event.element.srq = ibsrq; switch (type) { case MLX4_EVENT_TYPE_SRQ_LIMIT: event.event = IB_EVENT_SRQ_LIMIT_REACHED; break; case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR: event.event = IB_EVENT_SRQ_ERR; break; default: - printk(KERN_WARNING "mlx4_ib: Unexpected event type %d " + pr_warn("Unexpected event type %d " "on SRQ %06x\n", type, srq->srqn); return; } ibsrq->event_handler(&event, ibsrq->srq_context); } } -struct ib_srq *mlx4_ib_create_xrc_srq(struct ib_pd *pd, - struct ib_cq *xrc_cq, - struct ib_xrcd *xrcd, - struct ib_srq_init_attr *init_attr, - struct ib_udata *udata) +struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(pd->device); struct mlx4_ib_srq *srq; struct mlx4_wqe_srq_next_seg *next; - u32 cqn; - u16 xrcdn; + struct mlx4_wqe_data_seg *scatter; + u32 cqn; + u16 xrcdn; int desc_size; int buf_size; int err; int i; /* Sanity check SRQ size before proceeding */ if (init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes || - init_attr->attr.max_sge > dev->dev->caps.max_srq_sge) { - mlx4_ib_dbg("a size param is out of range. " - "max_wr = 0x%x, max_sge = 0x%x", - init_attr->attr.max_wr, init_attr->attr.max_sge); + init_attr->attr.max_sge > dev->dev->caps.max_srq_sge) return ERR_PTR(-EINVAL); - } - srq = kzalloc(sizeof *srq, GFP_KERNEL); + srq = kmalloc(sizeof *srq, GFP_KERNEL); if (!srq) return ERR_PTR(-ENOMEM); mutex_init(&srq->mutex); spin_lock_init(&srq->lock); srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); srq->msrq.max_gs = init_attr->attr.max_sge; desc_size = max(32UL, roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg) + srq->msrq.max_gs * sizeof (struct mlx4_wqe_data_seg))); srq->msrq.wqe_shift = ilog2(desc_size); buf_size = srq->msrq.max * desc_size; if (pd->uobject) { struct mlx4_ib_create_srq ucmd; if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { err = -EFAULT; goto err_srq; } srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, buf_size, 0, 0); if (IS_ERR(srq->umem)) { err = PTR_ERR(srq->umem); goto err_srq; } err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem), ilog2(srq->umem->page_size), &srq->mtt); if (err) goto err_buf; err = mlx4_ib_umem_write_mtt(dev, &srq->mtt, srq->umem); if (err) goto err_mtt; err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context), ucmd.db_addr, &srq->db); if (err) goto err_mtt; } else { - struct mlx4_wqe_data_seg *scatter; - err = mlx4_db_alloc(dev->dev, &srq->db, 0); if (err) goto err_srq; *srq->db.db = 0; if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &srq->buf)) { err = -ENOMEM; goto err_db; } srq->head = 0; srq->tail = srq->msrq.max - 1; srq->wqe_ctr = 0; for (i = 0; i < srq->msrq.max; ++i) { next = get_wqe(srq, i); next->next_wqe_index = cpu_to_be16((i + 1) & (srq->msrq.max - 1)); for (scatter = (void *) (next + 1); (void *) scatter < (void *) next + desc_size; ++scatter) scatter->lkey = cpu_to_be32(MLX4_INVALID_LKEY); } err = mlx4_mtt_init(dev->dev, srq->buf.npages, srq->buf.page_shift, &srq->mtt); if (err) goto err_buf; err = mlx4_buf_write_mtt(dev->dev, &srq->mtt, &srq->buf); if (err) goto err_mtt; srq->wrid = kmalloc(srq->msrq.max * sizeof (u64), GFP_KERNEL); if (!srq->wrid) { err = -ENOMEM; goto err_mtt; } } - cqn = xrc_cq ? (u32) (to_mcq(xrc_cq)->mcq.cqn) : 0; - xrcdn = xrcd ? (u16) (to_mxrcd(xrcd)->xrcdn) : + cqn = (init_attr->srq_type == IB_SRQT_XRC) ? + to_mcq(init_attr->ext.xrc.cq)->mcq.cqn : 0; + xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? + to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn : (u16) dev->dev->caps.reserved_xrcds; - err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, cqn, xrcdn, &srq->mtt, srq->db.dma, &srq->msrq); if (err) goto err_wrid; srq->msrq.event = mlx4_ib_srq_event; + srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; - if (pd->uobject) { + if (pd->uobject) if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) { err = -EFAULT; goto err_wrid; } - } else - srq->ibsrq.xrc_srq_num = srq->msrq.srqn; init_attr->attr.max_wr = srq->msrq.max - 1; return &srq->ibsrq; err_wrid: if (pd->uobject) mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db); else kfree(srq->wrid); err_mtt: mlx4_mtt_cleanup(dev->dev, &srq->mtt); err_buf: if (pd->uobject) ib_umem_release(srq->umem); else mlx4_buf_free(dev->dev, buf_size, &srq->buf); err_db: if (!pd->uobject) mlx4_db_free(dev->dev, &srq->db); err_srq: kfree(srq); return ERR_PTR(err); } int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); struct mlx4_ib_srq *srq = to_msrq(ibsrq); int ret; /* We don't support resizing SRQs (yet?) */ - if (attr_mask & IB_SRQ_MAX_WR) { - mlx4_ib_dbg("resize not yet supported"); + if (attr_mask & IB_SRQ_MAX_WR) return -EINVAL; - } if (attr_mask & IB_SRQ_LIMIT) { - if (attr->srq_limit >= srq->msrq.max){ - mlx4_ib_dbg("limit (0x%x) too high", attr->srq_limit); + if (attr->srq_limit >= srq->msrq.max) return -EINVAL; - } mutex_lock(&srq->mutex); ret = mlx4_srq_arm(dev->dev, &srq->msrq, attr->srq_limit); mutex_unlock(&srq->mutex); if (ret) return ret; } return 0; } -struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, - struct ib_srq_init_attr *init_attr, - struct ib_udata *udata) -{ - return mlx4_ib_create_xrc_srq(pd, NULL, NULL, init_attr, udata); -} - int mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) { struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); struct mlx4_ib_srq *srq = to_msrq(ibsrq); int ret; int limit_watermark; ret = mlx4_srq_query(dev->dev, &srq->msrq, &limit_watermark); if (ret) return ret; srq_attr->srq_limit = limit_watermark; srq_attr->max_wr = srq->msrq.max - 1; srq_attr->max_sge = srq->msrq.max_gs; return 0; } int mlx4_ib_destroy_srq(struct ib_srq *srq) { struct mlx4_ib_dev *dev = to_mdev(srq->device); struct mlx4_ib_srq *msrq = to_msrq(srq); - struct mlx4_ib_cq *cq; - mlx4_srq_invalidate(dev->dev, &msrq->msrq); - - if (srq->xrc_cq && !srq->uobject) { - cq = to_mcq(srq->xrc_cq); - spin_lock_irq(&cq->lock); - __mlx4_ib_cq_clean(cq, -1, msrq); - mlx4_srq_remove(dev->dev, &msrq->msrq); - spin_unlock_irq(&cq->lock); - } else - mlx4_srq_remove(dev->dev, &msrq->msrq); - mlx4_srq_free(dev->dev, &msrq->msrq); mlx4_mtt_cleanup(dev->dev, &msrq->mtt); if (srq->uobject) { mlx4_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db); ib_umem_release(msrq->umem); } else { kfree(msrq->wrid); mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift, &msrq->buf); mlx4_db_free(dev->dev, &msrq->db); } kfree(msrq); return 0; } void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index) { struct mlx4_wqe_srq_next_seg *next; /* always called with interrupts disabled. */ spin_lock(&srq->lock); next = get_wqe(srq, srq->tail); next->next_wqe_index = cpu_to_be16(wqe_index); srq->tail = wqe_index; spin_unlock(&srq->lock); } int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, struct ib_recv_wr **bad_wr) { struct mlx4_ib_srq *srq = to_msrq(ibsrq); struct mlx4_wqe_srq_next_seg *next; struct mlx4_wqe_data_seg *scat; unsigned long flags; int err = 0; int nreq; int i; spin_lock_irqsave(&srq->lock, flags); for (nreq = 0; wr; ++nreq, wr = wr->next) { if (unlikely(wr->num_sge > srq->msrq.max_gs)) { - mlx4_ib_dbg("srq num 0x%x: num s/g entries too large (%d)", - srq->msrq.srqn, wr->num_sge); err = -EINVAL; *bad_wr = wr; break; } if (unlikely(srq->head == srq->tail)) { - mlx4_ib_dbg("srq num 0x%x: No entries available to post.", - srq->msrq.srqn); err = -ENOMEM; *bad_wr = wr; break; } srq->wrid[srq->head] = wr->wr_id; next = get_wqe(srq, srq->head); srq->head = be16_to_cpu(next->next_wqe_index); scat = (struct mlx4_wqe_data_seg *) (next + 1); for (i = 0; i < wr->num_sge; ++i) { scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); } if (i < srq->msrq.max_gs) { scat[i].byte_count = 0; scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY); scat[i].addr = 0; } } if (likely(nreq)) { srq->wqe_ctr += nreq; /* * Make sure that descriptors are written before * doorbell record. */ wmb(); *srq->db.db = cpu_to_be32(srq->wqe_ctr); } spin_unlock_irqrestore(&srq->lock, flags); return err; } Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/sysfs.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/sysfs.c (nonexistent) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/sysfs.c (revision 273864) @@ -0,0 +1,801 @@ +/* + * Copyright (c) 2012 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/*#include "core_priv.h"*/ +#include "mlx4_ib.h" +#include +#include +#include + +#include +/*show_admin_alias_guid returns the administratively assigned value of that GUID. + * Values returned in buf parameter string: + * 0 - requests opensm to assign a value. + * ffffffffffffffff - delete this entry. + * other - value assigned by administrator. + */ +static ssize_t show_admin_alias_guid(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int record_num;/*0-15*/ + int guid_index_in_rec; /*0 - 7*/ + struct mlx4_ib_iov_sysfs_attr *mlx4_ib_iov_dentry = + container_of(attr, struct mlx4_ib_iov_sysfs_attr, dentry); + struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry->ctx; + struct mlx4_ib_dev *mdev = port->dev; + + record_num = mlx4_ib_iov_dentry->entry_num / 8 ; + guid_index_in_rec = mlx4_ib_iov_dentry->entry_num % 8 ; + + return sprintf(buf, "%llx\n", (long long) + be64_to_cpu(*(__be64 *)&mdev->sriov.alias_guid. + ports_guid[port->num - 1]. + all_rec_per_port[record_num]. + all_recs[8 * guid_index_in_rec])); +} + +/* store_admin_alias_guid stores the (new) administratively assigned value of that GUID. + * Values in buf parameter string: + * 0 - requests opensm to assign a value. + * 0xffffffffffffffff - delete this entry. + * other - guid value assigned by the administrator. + */ +static ssize_t store_admin_alias_guid(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int record_num;/*0-15*/ + int guid_index_in_rec; /*0 - 7*/ + struct mlx4_ib_iov_sysfs_attr *mlx4_ib_iov_dentry = + container_of(attr, struct mlx4_ib_iov_sysfs_attr, dentry); + struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry->ctx; + struct mlx4_ib_dev *mdev = port->dev; + u64 sysadmin_ag_val; + + record_num = mlx4_ib_iov_dentry->entry_num / 8; + guid_index_in_rec = mlx4_ib_iov_dentry->entry_num % 8; + if (0 == record_num && 0 == guid_index_in_rec) { + pr_err("GUID 0 block 0 is RO\n"); + return count; + } + sscanf(buf, "%llx", &sysadmin_ag_val); + *(__be64 *)&mdev->sriov.alias_guid.ports_guid[port->num - 1]. + all_rec_per_port[record_num]. + all_recs[GUID_REC_SIZE * guid_index_in_rec] = + cpu_to_be64(sysadmin_ag_val); + + /* Change the state to be pending for update */ + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].status + = MLX4_GUID_INFO_STATUS_IDLE ; + + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].method + = MLX4_GUID_INFO_RECORD_SET; + + switch (sysadmin_ag_val) { + case MLX4_GUID_FOR_DELETE_VAL: + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].method + = MLX4_GUID_INFO_RECORD_DELETE; + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].ownership + = MLX4_GUID_SYSADMIN_ASSIGN; + break; + /* The sysadmin requests the SM to re-assign */ + case MLX4_NOT_SET_GUID: + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].ownership + = MLX4_GUID_DRIVER_ASSIGN; + break; + /* The sysadmin requests a specific value.*/ + default: + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].ownership + = MLX4_GUID_SYSADMIN_ASSIGN; + break; + } + + /* set the record index */ + mdev->sriov.alias_guid.ports_guid[port->num - 1].all_rec_per_port[record_num].guid_indexes + = mlx4_ib_get_aguid_comp_mask_from_ix(guid_index_in_rec); + + mlx4_ib_init_alias_guid_work(mdev, port->num - 1); + + return count; +} + +static ssize_t show_port_gid(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mlx4_ib_iov_sysfs_attr *mlx4_ib_iov_dentry = + container_of(attr, struct mlx4_ib_iov_sysfs_attr, dentry); + struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry->ctx; + struct mlx4_ib_dev *mdev = port->dev; + union ib_gid gid; + ssize_t ret; + + ret = __mlx4_ib_query_gid(&mdev->ib_dev, port->num, + mlx4_ib_iov_dentry->entry_num, &gid, 1); + if (ret) + return ret; + ret = sprintf(buf, "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n", + be16_to_cpu(((__be16 *) gid.raw)[0]), + be16_to_cpu(((__be16 *) gid.raw)[1]), + be16_to_cpu(((__be16 *) gid.raw)[2]), + be16_to_cpu(((__be16 *) gid.raw)[3]), + be16_to_cpu(((__be16 *) gid.raw)[4]), + be16_to_cpu(((__be16 *) gid.raw)[5]), + be16_to_cpu(((__be16 *) gid.raw)[6]), + be16_to_cpu(((__be16 *) gid.raw)[7])); + return ret; +} + +static ssize_t show_phys_port_pkey(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mlx4_ib_iov_sysfs_attr *mlx4_ib_iov_dentry = + container_of(attr, struct mlx4_ib_iov_sysfs_attr, dentry); + struct mlx4_ib_iov_port *port = mlx4_ib_iov_dentry->ctx; + struct mlx4_ib_dev *mdev = port->dev; + u16 pkey; + ssize_t ret; + + ret = __mlx4_ib_query_pkey(&mdev->ib_dev, port->num, + mlx4_ib_iov_dentry->entry_num, &pkey, 1); + if (ret) + return ret; + + return sprintf(buf, "0x%04x\n", pkey); +} + +#define DENTRY_REMOVE(_dentry) \ +do { \ + sysfs_remove_file((_dentry)->kobj, &(_dentry)->dentry.attr); \ +} while (0); + +static int create_sysfs_entry(void *_ctx, struct mlx4_ib_iov_sysfs_attr *_dentry, + char *_name, struct kobject *_kobj, + ssize_t (*show)(struct device *dev, + struct device_attribute *attr, + char *buf), + ssize_t (*store)(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) + ) +{ + int ret = 0; + struct mlx4_ib_iov_sysfs_attr *vdentry = _dentry; + + vdentry->ctx = _ctx; + vdentry->dentry.show = show; + vdentry->dentry.store = store; + sysfs_attr_init(&vdentry->dentry.attr); + vdentry->dentry.attr.name = vdentry->name; + vdentry->dentry.attr.mode = 0; + vdentry->kobj = _kobj; + snprintf(vdentry->name, 15, "%s", _name); + + if (vdentry->dentry.store) + vdentry->dentry.attr.mode |= S_IWUSR; + + if (vdentry->dentry.show) + vdentry->dentry.attr.mode |= S_IRUGO; + + ret = sysfs_create_file(vdentry->kobj, &vdentry->dentry.attr); + if (ret) { + pr_err("failed to create %s\n", vdentry->dentry.attr.name); + vdentry->ctx = NULL; + return ret; + } + + return ret; +} + +int add_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num, + struct attribute *attr) +{ + struct mlx4_ib_iov_port *port = &device->iov_ports[port_num - 1]; + int ret; + + ret = sysfs_create_file(port->mcgs_parent, attr); + if (ret) + pr_err("failed to create %s\n", attr->name); + + return ret; +} + +void del_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num, + struct attribute *attr) +{ + struct mlx4_ib_iov_port *port = &device->iov_ports[port_num - 1]; + + sysfs_remove_file(port->mcgs_parent, attr); +} + +static int add_port_entries(struct mlx4_ib_dev *device, int port_num) +{ + int i; + char buff[10]; + struct mlx4_ib_iov_port *port = NULL; + int ret = 0 ; + struct ib_port_attr attr; + + /* get the physical gid and pkey table sizes.*/ + ret = __mlx4_ib_query_port(&device->ib_dev, port_num, &attr, 1); + if (ret) + goto err; + + port = &device->iov_ports[port_num - 1]; + port->dev = device; + port->num = port_num; + /* Directory structure: + * iov - + * port num - + * admin_guids + * gids (operational) + * mcg_table + */ + port->dentr_ar = kzalloc(sizeof (struct mlx4_ib_iov_sysfs_attr_ar), + GFP_KERNEL); + if (!port->dentr_ar) { + ret = -ENOMEM; + goto err; + } + sprintf(buff, "%d", port_num); + port->cur_port = kobject_create_and_add(buff, + kobject_get(device->ports_parent)); + if (!port->cur_port) { + ret = -ENOMEM; + goto kobj_create_err; + } + /* admin GUIDs */ + port->admin_alias_parent = kobject_create_and_add("admin_guids", + kobject_get(port->cur_port)); + if (!port->admin_alias_parent) { + ret = -ENOMEM; + goto err_admin_guids; + } + for (i = 0 ; i < attr.gid_tbl_len; i++) { + sprintf(buff, "%d", i); + port->dentr_ar->dentries[i].entry_num = i; + ret = create_sysfs_entry(port, &port->dentr_ar->dentries[i], + buff, port->admin_alias_parent, + show_admin_alias_guid, store_admin_alias_guid); + if (ret) + goto err_admin_alias_parent; + } + + /* gids subdirectory (operational gids) */ + port->gids_parent = kobject_create_and_add("gids", + kobject_get(port->cur_port)); + if (!port->gids_parent) { + ret = -ENOMEM; + goto err_gids; + } + + for (i = 0 ; i < attr.gid_tbl_len; i++) { + sprintf(buff, "%d", i); + port->dentr_ar->dentries[attr.gid_tbl_len + i].entry_num = i; + ret = create_sysfs_entry(port, + &port->dentr_ar->dentries[attr.gid_tbl_len + i], + buff, + port->gids_parent, show_port_gid, NULL); + if (ret) + goto err_gids_parent; + } + + /* physical port pkey table */ + port->pkeys_parent = + kobject_create_and_add("pkeys", kobject_get(port->cur_port)); + if (!port->pkeys_parent) { + ret = -ENOMEM; + goto err_pkeys; + } + + for (i = 0 ; i < attr.pkey_tbl_len; i++) { + sprintf(buff, "%d", i); + port->dentr_ar->dentries[2 * attr.gid_tbl_len + i].entry_num = i; + ret = create_sysfs_entry(port, + &port->dentr_ar->dentries[2 * attr.gid_tbl_len + i], + buff, port->pkeys_parent, + show_phys_port_pkey, NULL); + if (ret) + goto err_pkeys_parent; + } + + /* MCGs table */ + port->mcgs_parent = + kobject_create_and_add("mcgs", kobject_get(port->cur_port)); + if (!port->mcgs_parent) { + ret = -ENOMEM; + goto err_mcgs; + } + return 0; + +err_mcgs: + kobject_put(port->cur_port); + +err_pkeys_parent: + kobject_put(port->pkeys_parent); + +err_pkeys: + kobject_put(port->cur_port); + +err_gids_parent: + kobject_put(port->gids_parent); + +err_gids: + kobject_put(port->cur_port); + +err_admin_alias_parent: + kobject_put(port->admin_alias_parent); + +err_admin_guids: + kobject_put(port->cur_port); + kobject_put(port->cur_port); /* once more for create_and_add buff */ + +kobj_create_err: + kobject_put(device->ports_parent); + kfree(port->dentr_ar); + +err: + pr_err("add_port_entries FAILED: for port:%d, error: %d\n", + port_num, ret); + return ret; +} + +static void get_name(struct mlx4_ib_dev *dev, char *name, int i, int max) +{ + char base_name[9]; + + /* pci_name format is: bus:dev:func -> xxxx:yy:zz.n */ + strlcpy(name, pci_name(dev->dev->pdev), max); + strncpy(base_name, name, 8); /*till xxxx:yy:*/ + base_name[8] = '\0'; + /* with no ARI only 3 last bits are used so when the fn is higher than 8 + * need to add it to the dev num, so count in the last number will be + * modulo 8 */ + sprintf(name, "%s%.2d.%d", base_name, (i/8), (i%8)); +} + +struct mlx4_port { + struct kobject kobj; + struct mlx4_ib_dev *dev; + struct attribute_group pkey_group; + struct attribute_group gid_group; + u8 port_num; + int slave; +}; + + +static void mlx4_port_release(struct kobject *kobj) +{ + struct mlx4_port *p = container_of(kobj, struct mlx4_port, kobj); + struct attribute *a; + int i; + + for (i = 0; (a = p->pkey_group.attrs[i]); ++i) + kfree(a); + kfree(p->pkey_group.attrs); + for (i = 0; (a = p->gid_group.attrs[i]); ++i) + kfree(a); + kfree(p->gid_group.attrs); + kfree(p); +} + +struct port_attribute { + struct attribute attr; + ssize_t (*show)(struct mlx4_port *, struct port_attribute *, char *buf); + ssize_t (*store)(struct mlx4_port *, struct port_attribute *, + const char *buf, size_t count); +}; + +static ssize_t port_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct port_attribute *port_attr = + container_of(attr, struct port_attribute, attr); + struct mlx4_port *p = container_of(kobj, struct mlx4_port, kobj); + + if (!port_attr->show) + return -EIO; + return port_attr->show(p, port_attr, buf); +} + +static ssize_t port_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t size) +{ + struct port_attribute *port_attr = + container_of(attr, struct port_attribute, attr); + struct mlx4_port *p = container_of(kobj, struct mlx4_port, kobj); + + if (!port_attr->store) + return -EIO; + return port_attr->store(p, port_attr, buf, size); +} + +static const struct sysfs_ops port_sysfs_ops = { + .show = port_attr_show, + .store = port_attr_store, +}; + +static struct kobj_type port_type = { + .release = mlx4_port_release, + .sysfs_ops = &port_sysfs_ops, +}; + +struct port_table_attribute { + struct port_attribute attr; + char name[8]; + int index; +}; + +static ssize_t show_port_pkey(struct mlx4_port *p, struct port_attribute *attr, + char *buf) +{ + struct port_table_attribute *tab_attr = + container_of(attr, struct port_table_attribute, attr); + ssize_t ret = -ENODEV; + + if (p->dev->pkeys.virt2phys_pkey[p->slave][p->port_num - 1][tab_attr->index] >= + (p->dev->dev->caps.pkey_table_len[p->port_num])) + ret = sprintf(buf, "none\n"); + else + ret = sprintf(buf, "%d\n", + p->dev->pkeys.virt2phys_pkey[p->slave] + [p->port_num - 1][tab_attr->index]); + return ret; +} + +static ssize_t store_port_pkey(struct mlx4_port *p, struct port_attribute *attr, + const char *buf, size_t count) +{ + struct port_table_attribute *tab_attr = + container_of(attr, struct port_table_attribute, attr); + int idx; + int err; + + /* do not allow remapping Dom0 virtual pkey table */ + if (p->slave == mlx4_master_func_num(p->dev->dev)) + return -EINVAL; + + if (!strncasecmp(buf, "no", 2)) + idx = p->dev->dev->phys_caps.pkey_phys_table_len[p->port_num] - 1; + else if (sscanf(buf, "%i", &idx) != 1 || + idx >= p->dev->dev->caps.pkey_table_len[p->port_num] || + idx < 0) + return -EINVAL; + + p->dev->pkeys.virt2phys_pkey[p->slave][p->port_num - 1] + [tab_attr->index] = idx; + mlx4_sync_pkey_table(p->dev->dev, p->slave, p->port_num, + tab_attr->index, idx); + err = mlx4_gen_pkey_eqe(p->dev->dev, p->slave, p->port_num); + if (err) { + pr_err("mlx4_gen_pkey_eqe failed for slave %d," + " port %d, index %d\n", p->slave, p->port_num, idx); + return err; + } + return count; +} + +static ssize_t show_port_gid_idx(struct mlx4_port *p, + struct port_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", p->slave); +} + +static struct attribute ** +alloc_group_attrs(ssize_t (*show)(struct mlx4_port *, + struct port_attribute *, char *buf), + ssize_t (*store)(struct mlx4_port *, struct port_attribute *, + const char *buf, size_t count), + int len) +{ + struct attribute **tab_attr; + struct port_table_attribute *element; + int i; + + tab_attr = kcalloc(1 + len, sizeof (struct attribute *), GFP_KERNEL); + if (!tab_attr) + return NULL; + + for (i = 0; i < len; i++) { + element = kzalloc(sizeof (struct port_table_attribute), + GFP_KERNEL); + if (!element) + goto err; + if (snprintf(element->name, sizeof (element->name), + "%d", i) >= sizeof (element->name)) { + kfree(element); + goto err; + } + sysfs_attr_init(&element->attr.attr); + element->attr.attr.name = element->name; + if (store) { + element->attr.attr.mode = S_IWUSR | S_IRUGO; + element->attr.store = store; + } else + element->attr.attr.mode = S_IRUGO; + + element->attr.show = show; + element->index = i; + tab_attr[i] = &element->attr.attr; + } + return tab_attr; + +err: + while (--i >= 0) + kfree(tab_attr[i]); + kfree(tab_attr); + return NULL; +} + +static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) +{ + struct mlx4_port *p; + int i; + int ret; + int is_eth = rdma_port_get_link_layer(&dev->ib_dev, port_num) == + IB_LINK_LAYER_ETHERNET; + + p = kzalloc(sizeof *p, GFP_KERNEL); + if (!p) + return -ENOMEM; + + p->dev = dev; + p->port_num = port_num; + p->slave = slave; + + ret = kobject_init_and_add(&p->kobj, &port_type, + kobject_get(dev->dev_ports_parent[slave]), + "%d", port_num); + if (ret) + goto err_alloc; + + p->pkey_group.name = "pkey_idx"; + if (is_eth) + p->pkey_group.attrs = + alloc_group_attrs(show_port_pkey, NULL, + dev->dev->caps.pkey_table_len[port_num]); + else + p->pkey_group.attrs = + alloc_group_attrs(show_port_pkey, store_port_pkey, + dev->dev->caps.pkey_table_len[port_num]); + if (!p->pkey_group.attrs) + goto err_alloc; + + ret = sysfs_create_group(&p->kobj, &p->pkey_group); + if (ret) + goto err_free_pkey; + + p->gid_group.name = "gid_idx"; + p->gid_group.attrs = alloc_group_attrs(show_port_gid_idx, NULL, 1); + if (!p->gid_group.attrs) + goto err_free_pkey; + + ret = sysfs_create_group(&p->kobj, &p->gid_group); + if (ret) + goto err_free_gid; + + list_add_tail(&p->kobj.entry, &dev->pkeys.pkey_port_list[slave]); + return 0; + +err_free_gid: + kfree(p->gid_group.attrs[0]); + kfree(p->gid_group.attrs); + +err_free_pkey: + for (i = 0; i < dev->dev->caps.pkey_table_len[port_num]; ++i) + kfree(p->pkey_group.attrs[i]); + kfree(p->pkey_group.attrs); + +err_alloc: + kobject_put(dev->dev_ports_parent[slave]); + kfree(p); + return ret; +} + +static int register_one_pkey_tree(struct mlx4_ib_dev *dev, int slave) +{ + char name[32]; + int err; + int port; + struct kobject *p, *t; + struct mlx4_port *mport; + + get_name(dev, name, slave, sizeof name); + + dev->pkeys.device_parent[slave] = + kobject_create_and_add(name, kobject_get(dev->iov_parent)); + + if (!dev->pkeys.device_parent[slave]) { + err = -ENOMEM; + goto fail_dev; + } + + INIT_LIST_HEAD(&dev->pkeys.pkey_port_list[slave]); + + dev->dev_ports_parent[slave] = + kobject_create_and_add("ports", + kobject_get(dev->pkeys.device_parent[slave])); + + if (!dev->dev_ports_parent[slave]) { + err = -ENOMEM; + goto err_ports; + } + + for (port = 1; port <= dev->dev->caps.num_ports; ++port) { + err = add_port(dev, port, slave); + if (err) + goto err_add; + } + return 0; + +err_add: + list_for_each_entry_safe(p, t, + &dev->pkeys.pkey_port_list[slave], + entry) { + list_del(&p->entry); + mport = container_of(p, struct mlx4_port, kobj); + sysfs_remove_group(p, &mport->pkey_group); + sysfs_remove_group(p, &mport->gid_group); + kobject_put(p); + } + kobject_put(dev->dev_ports_parent[slave]); + +err_ports: + kobject_put(dev->pkeys.device_parent[slave]); + /* extra put for the device_parent create_and_add */ + kobject_put(dev->pkeys.device_parent[slave]); + +fail_dev: + kobject_put(dev->iov_parent); + return err; +} + +static int register_pkey_tree(struct mlx4_ib_dev *device) +{ + int i; + + if (!mlx4_is_master(device->dev)) + return 0; + + for (i = 0; i <= device->dev->num_vfs; ++i) + register_one_pkey_tree(device, i); + + return 0; +} + +static void unregister_pkey_tree(struct mlx4_ib_dev *device) +{ + int slave; + struct kobject *p, *t; + struct mlx4_port *port; + + if (!mlx4_is_master(device->dev)) + return; + + for (slave = device->dev->num_vfs; slave >= 0; --slave) { + list_for_each_entry_safe(p, t, + &device->pkeys.pkey_port_list[slave], + entry) { + list_del(&p->entry); + port = container_of(p, struct mlx4_port, kobj); + sysfs_remove_group(p, &port->pkey_group); + sysfs_remove_group(p, &port->gid_group); + kobject_put(p); + kobject_put(device->dev_ports_parent[slave]); + } + kobject_put(device->dev_ports_parent[slave]); + kobject_put(device->pkeys.device_parent[slave]); + kobject_put(device->pkeys.device_parent[slave]); + kobject_put(device->iov_parent); + } +} + +int mlx4_ib_device_register_sysfs(struct mlx4_ib_dev *dev) +{ + int i; + int ret = 0; + + if (!mlx4_is_master(dev->dev)) + return 0; + + dev->iov_parent = + kobject_create_and_add("iov", + kobject_get(dev->ib_dev.ports_parent->parent)); + if (!dev->iov_parent) { + ret = -ENOMEM; + goto err; + } + dev->ports_parent = + kobject_create_and_add("ports", + kobject_get(dev->iov_parent)); + if (!dev->iov_parent) { + ret = -ENOMEM; + goto err_ports; + } + + for (i = 1; i <= dev->ib_dev.phys_port_cnt; ++i) { + ret = add_port_entries(dev, i); + if (ret) + goto err_add_entries; + } + + ret = register_pkey_tree(dev); + if (ret) + goto err_add_entries; + return 0; + +err_add_entries: + kobject_put(dev->ports_parent); + +err_ports: + kobject_put(dev->iov_parent); +err: + kobject_put(dev->ib_dev.ports_parent->parent); + pr_err("mlx4_ib_device_register_sysfs error (%d)\n", ret); + return ret; +} + +static void unregister_alias_guid_tree(struct mlx4_ib_dev *device) +{ + struct mlx4_ib_iov_port *p; + int i; + + if (!mlx4_is_master(device->dev)) + return; + + for (i = 0; i < device->dev->caps.num_ports; i++) { + p = &device->iov_ports[i]; + kobject_put(p->admin_alias_parent); + kobject_put(p->gids_parent); + kobject_put(p->pkeys_parent); + kobject_put(p->mcgs_parent); + kobject_put(p->cur_port); + kobject_put(p->cur_port); + kobject_put(p->cur_port); + kobject_put(p->cur_port); + kobject_put(p->cur_port); + kobject_put(p->dev->ports_parent); + kfree(p->dentr_ar); + } +} + +void mlx4_ib_device_unregister_sysfs(struct mlx4_ib_dev *device) +{ + unregister_alias_guid_tree(device); + unregister_pkey_tree(device); + kobject_put(device->ports_parent); + kobject_put(device->iov_parent); + kobject_put(device->iov_parent); + kobject_put(device->ib_dev.ports_parent->parent); +} Property changes on: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/sysfs.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/user.h =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/user.h (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/user.h (revision 273864) @@ -1,97 +1,107 @@ /* * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef MLX4_IB_USER_H #define MLX4_IB_USER_H #include /* * Increment this value if any changes that break userspace ABI * compatibility are made. */ -#define MLX4_IB_UVERBS_ABI_VERSION 3 +#define MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION 3 +#define MLX4_IB_UVERBS_ABI_VERSION 4 + /* * Make sure that all structs defined in this file remain laid out so * that they pack the same way on 32-bit and 64-bit architectures (to * avoid incompatibility between 32-bit userspace and 64-bit kernels). * In particular do not use pointer types -- pass pointers in __u64 * instead. */ +struct mlx4_ib_alloc_ucontext_resp_v3 { + __u32 qp_tab_size; + __u16 bf_reg_size; + __u16 bf_regs_per_page; +}; + struct mlx4_ib_alloc_ucontext_resp { + __u32 dev_caps; __u32 qp_tab_size; __u16 bf_reg_size; __u16 bf_regs_per_page; + __u32 cqe_size; }; struct mlx4_ib_alloc_pd_resp { __u32 pdn; __u32 reserved; }; struct mlx4_ib_create_cq { __u64 buf_addr; __u64 db_addr; }; struct mlx4_ib_create_cq_resp { __u32 cqn; __u32 reserved; }; struct mlx4_ib_resize_cq { __u64 buf_addr; }; struct mlx4_ib_create_srq { __u64 buf_addr; __u64 db_addr; }; struct mlx4_ib_create_srq_resp { __u32 srqn; __u32 reserved; }; struct mlx4_ib_create_qp { __u64 buf_addr; __u64 db_addr; __u8 log_sq_bb_count; __u8 log_sq_stride; __u8 sq_no_prefetch; __u8 reserved[5]; }; #endif /* MLX4_IB_USER_H */ Index: stable/9/sys/ofed/drivers/infiniband/hw/mlx4/wc.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mlx4/wc.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mlx4/wc.c (revision 273864) @@ -1,74 +1,73 @@ /* * Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include "wc.h" #if defined(__i386__) || defined(__x86_64__) pgprot_t pgprot_wc(pgprot_t _prot) { return pgprot_writecombine(_prot); } int mlx4_wc_enabled(void) { return 1; } #elif defined(CONFIG_PPC64) pgprot_t pgprot_wc(pgprot_t _prot) { return __pgprot((pgprot_val(_prot) | _PAGE_NO_CACHE) & ~(pgprot_t)_PAGE_GUARDED); } int mlx4_wc_enabled(void) { return 1; } #else /* !(defined(__i386__) || defined(__x86_64__)) */ pgprot_t pgprot_wc(pgprot_t _prot) { return pgprot_noncached(_prot); } int mlx4_wc_enabled(void) { return 0; } #endif - Index: stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_allocator.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_allocator.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_allocator.c (revision 273864) @@ -1,301 +1,300 @@ /* * Copyright (c) 2004 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include -#include #include "mthca_dev.h" /* Trivial bitmap-based allocator */ u32 mthca_alloc(struct mthca_alloc *alloc) { unsigned long flags; u32 obj; spin_lock_irqsave(&alloc->lock, flags); obj = find_next_zero_bit(alloc->table, alloc->max, alloc->last); if (obj >= alloc->max) { alloc->top = (alloc->top + alloc->max) & alloc->mask; obj = find_first_zero_bit(alloc->table, alloc->max); } if (obj < alloc->max) { set_bit(obj, alloc->table); obj |= alloc->top; } else obj = -1; spin_unlock_irqrestore(&alloc->lock, flags); return obj; } void mthca_free(struct mthca_alloc *alloc, u32 obj) { unsigned long flags; obj &= alloc->max - 1; spin_lock_irqsave(&alloc->lock, flags); clear_bit(obj, alloc->table); alloc->last = min(alloc->last, obj); alloc->top = (alloc->top + alloc->max) & alloc->mask; spin_unlock_irqrestore(&alloc->lock, flags); } int mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask, u32 reserved) { int i; /* num must be a power of 2 */ if (num != 1 << (ffs(num) - 1)) return -EINVAL; alloc->last = 0; alloc->top = 0; alloc->max = num; alloc->mask = mask; spin_lock_init(&alloc->lock); alloc->table = kmalloc(BITS_TO_LONGS(num) * sizeof (long), GFP_KERNEL); if (!alloc->table) return -ENOMEM; bitmap_zero(alloc->table, num); for (i = 0; i < reserved; ++i) set_bit(i, alloc->table); return 0; } void mthca_alloc_cleanup(struct mthca_alloc *alloc) { kfree(alloc->table); } /* * Array of pointers with lazy allocation of leaf pages. Callers of * _get, _set and _clear methods must use a lock or otherwise * serialize access to the array. */ #define MTHCA_ARRAY_MASK (PAGE_SIZE / sizeof (void *) - 1) void *mthca_array_get(struct mthca_array *array, int index) { int p = (index * sizeof (void *)) >> PAGE_SHIFT; if (array->page_list[p].page) return array->page_list[p].page[index & MTHCA_ARRAY_MASK]; else return NULL; } int mthca_array_set(struct mthca_array *array, int index, void *value) { int p = (index * sizeof (void *)) >> PAGE_SHIFT; /* Allocate with GFP_ATOMIC because we'll be called with locks held. */ if (!array->page_list[p].page) array->page_list[p].page = (void **) get_zeroed_page(GFP_ATOMIC); if (!array->page_list[p].page) return -ENOMEM; array->page_list[p].page[index & MTHCA_ARRAY_MASK] = value; ++array->page_list[p].used; return 0; } void mthca_array_clear(struct mthca_array *array, int index) { int p = (index * sizeof (void *)) >> PAGE_SHIFT; if (--array->page_list[p].used == 0) { free_page((unsigned long) array->page_list[p].page); array->page_list[p].page = NULL; } else array->page_list[p].page[index & MTHCA_ARRAY_MASK] = NULL; if (array->page_list[p].used < 0) pr_debug("Array %p index %d page %d with ref count %d < 0\n", array, index, p, array->page_list[p].used); } int mthca_array_init(struct mthca_array *array, int nent) { int npage = (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE; int i; array->page_list = kmalloc(npage * sizeof *array->page_list, GFP_KERNEL); if (!array->page_list) return -ENOMEM; for (i = 0; i < npage; ++i) { array->page_list[i].page = NULL; array->page_list[i].used = 0; } return 0; } void mthca_array_cleanup(struct mthca_array *array, int nent) { int i; for (i = 0; i < (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE; ++i) free_page((unsigned long) array->page_list[i].page); kfree(array->page_list); } /* * Handling for queue buffers -- we allocate a bunch of memory and * register it in a memory region at HCA virtual address 0. If the * requested size is > max_direct, we split the allocation into * multiple pages, so we don't require too much contiguous memory. */ int mthca_buf_alloc(struct mthca_dev *dev, int size, int max_direct, union mthca_buf *buf, int *is_direct, struct mthca_pd *pd, int hca_write, struct mthca_mr *mr) { int err = -ENOMEM; int npages, shift; u64 *dma_list = NULL; dma_addr_t t; int i; if (size <= max_direct) { *is_direct = 1; npages = 1; shift = get_order(size) + PAGE_SHIFT; buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, size, &t, GFP_KERNEL); if (!buf->direct.buf) return -ENOMEM; pci_unmap_addr_set(&buf->direct, mapping, t); memset(buf->direct.buf, 0, size); while (t & ((1 << shift) - 1)) { --shift; npages *= 2; } dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); if (!dma_list) goto err_free; for (i = 0; i < npages; ++i) dma_list[i] = t + i * (1 << shift); } else { *is_direct = 0; npages = (size + PAGE_SIZE - 1) / PAGE_SIZE; shift = PAGE_SHIFT; dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); if (!dma_list) return -ENOMEM; buf->page_list = kmalloc(npages * sizeof *buf->page_list, GFP_KERNEL); if (!buf->page_list) goto err_out; for (i = 0; i < npages; ++i) buf->page_list[i].buf = NULL; for (i = 0; i < npages; ++i) { buf->page_list[i].buf = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, &t, GFP_KERNEL); if (!buf->page_list[i].buf) goto err_free; dma_list[i] = t; pci_unmap_addr_set(&buf->page_list[i], mapping, t); clear_page(buf->page_list[i].buf); } } err = mthca_mr_alloc_phys(dev, pd->pd_num, dma_list, shift, npages, 0, size, MTHCA_MPT_FLAG_LOCAL_READ | (hca_write ? MTHCA_MPT_FLAG_LOCAL_WRITE : 0), mr); if (err) goto err_free; kfree(dma_list); return 0; err_free: mthca_buf_free(dev, size, buf, *is_direct, NULL); err_out: kfree(dma_list); return err; } void mthca_buf_free(struct mthca_dev *dev, int size, union mthca_buf *buf, int is_direct, struct mthca_mr *mr) { int i; if (mr) mthca_free_mr(dev, mr); if (is_direct) dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, pci_unmap_addr(&buf->direct, mapping)); else { for (i = 0; i < (size + PAGE_SIZE - 1) / PAGE_SIZE; ++i) dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf->page_list[i].buf, pci_unmap_addr(&buf->page_list[i], mapping)); kfree(buf->page_list); } } Index: stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.c (revision 273864) @@ -1,1931 +1,1931 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Mellanox Technologies. All rights reserved. * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #include #include #include "mthca_dev.h" #include "mthca_config_reg.h" #include "mthca_cmd.h" #include "mthca_memfree.h" #define CMD_POLL_TOKEN 0xffff enum { HCR_IN_PARAM_OFFSET = 0x00, HCR_IN_MODIFIER_OFFSET = 0x08, HCR_OUT_PARAM_OFFSET = 0x0c, HCR_TOKEN_OFFSET = 0x14, HCR_STATUS_OFFSET = 0x18, HCR_OPMOD_SHIFT = 12, HCA_E_BIT = 22, HCR_GO_BIT = 23 }; enum { /* initialization and general commands */ CMD_SYS_EN = 0x1, CMD_SYS_DIS = 0x2, CMD_MAP_FA = 0xfff, CMD_UNMAP_FA = 0xffe, CMD_RUN_FW = 0xff6, CMD_MOD_STAT_CFG = 0x34, CMD_QUERY_DEV_LIM = 0x3, CMD_QUERY_FW = 0x4, CMD_ENABLE_LAM = 0xff8, CMD_DISABLE_LAM = 0xff7, CMD_QUERY_DDR = 0x5, CMD_QUERY_ADAPTER = 0x6, CMD_INIT_HCA = 0x7, CMD_CLOSE_HCA = 0x8, CMD_INIT_IB = 0x9, CMD_CLOSE_IB = 0xa, CMD_QUERY_HCA = 0xb, CMD_SET_IB = 0xc, CMD_ACCESS_DDR = 0x2e, CMD_MAP_ICM = 0xffa, CMD_UNMAP_ICM = 0xff9, CMD_MAP_ICM_AUX = 0xffc, CMD_UNMAP_ICM_AUX = 0xffb, CMD_SET_ICM_SIZE = 0xffd, /* TPT commands */ CMD_SW2HW_MPT = 0xd, CMD_QUERY_MPT = 0xe, CMD_HW2SW_MPT = 0xf, CMD_READ_MTT = 0x10, CMD_WRITE_MTT = 0x11, CMD_SYNC_TPT = 0x2f, /* EQ commands */ CMD_MAP_EQ = 0x12, CMD_SW2HW_EQ = 0x13, CMD_HW2SW_EQ = 0x14, CMD_QUERY_EQ = 0x15, /* CQ commands */ CMD_SW2HW_CQ = 0x16, CMD_HW2SW_CQ = 0x17, CMD_QUERY_CQ = 0x18, CMD_RESIZE_CQ = 0x2c, /* SRQ commands */ CMD_SW2HW_SRQ = 0x35, CMD_HW2SW_SRQ = 0x36, CMD_QUERY_SRQ = 0x37, CMD_ARM_SRQ = 0x40, /* QP/EE commands */ CMD_RST2INIT_QPEE = 0x19, CMD_INIT2RTR_QPEE = 0x1a, CMD_RTR2RTS_QPEE = 0x1b, CMD_RTS2RTS_QPEE = 0x1c, CMD_SQERR2RTS_QPEE = 0x1d, CMD_2ERR_QPEE = 0x1e, CMD_RTS2SQD_QPEE = 0x1f, CMD_SQD2SQD_QPEE = 0x38, CMD_SQD2RTS_QPEE = 0x20, CMD_ERR2RST_QPEE = 0x21, CMD_QUERY_QPEE = 0x22, CMD_INIT2INIT_QPEE = 0x2d, CMD_SUSPEND_QPEE = 0x32, CMD_UNSUSPEND_QPEE = 0x33, /* special QPs and management commands */ CMD_CONF_SPECIAL_QP = 0x23, CMD_MAD_IFC = 0x24, /* multicast commands */ CMD_READ_MGM = 0x25, CMD_WRITE_MGM = 0x26, CMD_MGID_HASH = 0x27, /* miscellaneous commands */ CMD_DIAG_RPRT = 0x30, CMD_NOP = 0x31, /* debug commands */ CMD_QUERY_DEBUG_MSG = 0x2a, CMD_SET_DEBUG_MSG = 0x2b, }; /* * According to Mellanox code, FW may be starved and never complete * commands. So we can't use strict timeouts described in PRM -- we * just arbitrarily select 60 seconds for now. */ #if 0 /* * Round up and add 1 to make sure we get the full wait time (since we * will be starting in the middle of a jiffy) */ enum { CMD_TIME_CLASS_A = (HZ + 999) / 1000 + 1, CMD_TIME_CLASS_B = (HZ + 99) / 100 + 1, CMD_TIME_CLASS_C = (HZ + 9) / 10 + 1, CMD_TIME_CLASS_D = 60 * HZ }; #else #define CMD_TIME_CLASS_A (60 * HZ) #define CMD_TIME_CLASS_B (60 * HZ) #define CMD_TIME_CLASS_C (60 * HZ) #define CMD_TIME_CLASS_D (60 * HZ) #endif #define GO_BIT_TIMEOUT (HZ * 10) struct mthca_cmd_context { struct completion done; int result; int next; u64 out_param; u16 token; u8 status; }; static int fw_cmd_doorbell = 0; module_param(fw_cmd_doorbell, int, 0644); MODULE_PARM_DESC(fw_cmd_doorbell, "post FW commands through doorbell page if nonzero " "(and supported by FW)"); static inline int go_bit(struct mthca_dev *dev) { return readl(dev->hcr + HCR_STATUS_OFFSET) & swab32(1 << HCR_GO_BIT); } static void mthca_cmd_post_dbell(struct mthca_dev *dev, u64 in_param, u64 out_param, u32 in_modifier, u8 op_modifier, u16 op, u16 token) { void __iomem *ptr = dev->cmd.dbell_map; u16 *offs = dev->cmd.dbell_offsets; __raw_writel((__force u32) cpu_to_be32(in_param >> 32), ptr + offs[0]); wmb(); __raw_writel((__force u32) cpu_to_be32(in_param & 0xfffffffful), ptr + offs[1]); wmb(); __raw_writel((__force u32) cpu_to_be32(in_modifier), ptr + offs[2]); wmb(); __raw_writel((__force u32) cpu_to_be32(out_param >> 32), ptr + offs[3]); wmb(); __raw_writel((__force u32) cpu_to_be32(out_param & 0xfffffffful), ptr + offs[4]); wmb(); __raw_writel((__force u32) cpu_to_be32(token << 16), ptr + offs[5]); wmb(); __raw_writel((__force u32) cpu_to_be32((1 << HCR_GO_BIT) | (1 << HCA_E_BIT) | (op_modifier << HCR_OPMOD_SHIFT) | op), ptr + offs[6]); wmb(); __raw_writel((__force u32) 0, ptr + offs[7]); wmb(); } static int mthca_cmd_post_hcr(struct mthca_dev *dev, u64 in_param, u64 out_param, u32 in_modifier, u8 op_modifier, u16 op, u16 token, int event) { if (event) { unsigned long end = jiffies + GO_BIT_TIMEOUT; while (go_bit(dev) && time_before(jiffies, end)) sched_yield(); } if (go_bit(dev)) return -EAGAIN; /* * We use writel (instead of something like memcpy_toio) * because writes of less than 32 bits to the HCR don't work * (and some architectures such as ia64 implement memcpy_toio * in terms of writeb). */ __raw_writel((__force u32) cpu_to_be32(in_param >> 32), dev->hcr + 0 * 4); __raw_writel((__force u32) cpu_to_be32(in_param & 0xfffffffful), dev->hcr + 1 * 4); __raw_writel((__force u32) cpu_to_be32(in_modifier), dev->hcr + 2 * 4); __raw_writel((__force u32) cpu_to_be32(out_param >> 32), dev->hcr + 3 * 4); __raw_writel((__force u32) cpu_to_be32(out_param & 0xfffffffful), dev->hcr + 4 * 4); __raw_writel((__force u32) cpu_to_be32(token << 16), dev->hcr + 5 * 4); /* __raw_writel may not order writes. */ wmb(); __raw_writel((__force u32) cpu_to_be32((1 << HCR_GO_BIT) | (event ? (1 << HCA_E_BIT) : 0) | (op_modifier << HCR_OPMOD_SHIFT) | op), dev->hcr + 6 * 4); return 0; } static int mthca_cmd_post(struct mthca_dev *dev, u64 in_param, u64 out_param, u32 in_modifier, u8 op_modifier, u16 op, u16 token, int event) { int err = 0; mutex_lock(&dev->cmd.hcr_mutex); if (event && dev->cmd.flags & MTHCA_CMD_POST_DOORBELLS && fw_cmd_doorbell) mthca_cmd_post_dbell(dev, in_param, out_param, in_modifier, op_modifier, op, token); else err = mthca_cmd_post_hcr(dev, in_param, out_param, in_modifier, op_modifier, op, token, event); /* * Make sure that our HCR writes don't get mixed in with * writes from another CPU starting a FW command. */ mmiowb(); mutex_unlock(&dev->cmd.hcr_mutex); return err; } static int mthca_cmd_poll(struct mthca_dev *dev, u64 in_param, u64 *out_param, int out_is_imm, u32 in_modifier, u8 op_modifier, u16 op, unsigned long timeout, u8 *status) { int err = 0; unsigned long end; down(&dev->cmd.poll_sem); err = mthca_cmd_post(dev, in_param, out_param ? *out_param : 0, in_modifier, op_modifier, op, CMD_POLL_TOKEN, 0); if (err) goto out; end = timeout + jiffies; while (go_bit(dev) && time_before(jiffies, end)) sched_yield(); if (go_bit(dev)) { err = -EBUSY; goto out; } if (out_is_imm) *out_param = (u64) be32_to_cpu((__force __be32) __raw_readl(dev->hcr + HCR_OUT_PARAM_OFFSET)) << 32 | (u64) be32_to_cpu((__force __be32) __raw_readl(dev->hcr + HCR_OUT_PARAM_OFFSET + 4)); *status = be32_to_cpu((__force __be32) __raw_readl(dev->hcr + HCR_STATUS_OFFSET)) >> 24; out: up(&dev->cmd.poll_sem); return err; } void mthca_cmd_event(struct mthca_dev *dev, u16 token, u8 status, u64 out_param) { struct mthca_cmd_context *context = &dev->cmd.context[token & dev->cmd.token_mask]; /* previously timed out command completing at long last */ if (token != context->token) return; context->result = 0; context->status = status; context->out_param = out_param; complete(&context->done); } static int mthca_cmd_wait(struct mthca_dev *dev, u64 in_param, u64 *out_param, int out_is_imm, u32 in_modifier, u8 op_modifier, u16 op, unsigned long timeout, u8 *status) { int err = 0; struct mthca_cmd_context *context; down(&dev->cmd.event_sem); spin_lock(&dev->cmd.context_lock); BUG_ON(dev->cmd.free_head < 0); context = &dev->cmd.context[dev->cmd.free_head]; context->token += dev->cmd.token_mask + 1; dev->cmd.free_head = context->next; spin_unlock(&dev->cmd.context_lock); init_completion(&context->done); err = mthca_cmd_post(dev, in_param, out_param ? *out_param : 0, in_modifier, op_modifier, op, context->token, 1); if (err) goto out; if (!wait_for_completion_timeout(&context->done, timeout)) { err = -EBUSY; goto out; } err = context->result; if (err) goto out; *status = context->status; if (*status) mthca_dbg(dev, "Command %02x completed with status %02x\n", op, *status); if (out_is_imm) *out_param = context->out_param; out: spin_lock(&dev->cmd.context_lock); context->next = dev->cmd.free_head; dev->cmd.free_head = context - dev->cmd.context; spin_unlock(&dev->cmd.context_lock); up(&dev->cmd.event_sem); return err; } /* Invoke a command with an output mailbox */ static int mthca_cmd_box(struct mthca_dev *dev, u64 in_param, u64 out_param, u32 in_modifier, u8 op_modifier, u16 op, unsigned long timeout, u8 *status) { if (dev->cmd.flags & MTHCA_CMD_USE_EVENTS) return mthca_cmd_wait(dev, in_param, &out_param, 0, in_modifier, op_modifier, op, timeout, status); else return mthca_cmd_poll(dev, in_param, &out_param, 0, in_modifier, op_modifier, op, timeout, status); } /* Invoke a command with no output parameter */ static int mthca_cmd(struct mthca_dev *dev, u64 in_param, u32 in_modifier, u8 op_modifier, u16 op, unsigned long timeout, u8 *status) { return mthca_cmd_box(dev, in_param, 0, in_modifier, op_modifier, op, timeout, status); } /* * Invoke a command with an immediate output parameter (and copy the * output into the caller's out_param pointer after the command * executes). */ static int mthca_cmd_imm(struct mthca_dev *dev, u64 in_param, u64 *out_param, u32 in_modifier, u8 op_modifier, u16 op, unsigned long timeout, u8 *status) { if (dev->cmd.flags & MTHCA_CMD_USE_EVENTS) return mthca_cmd_wait(dev, in_param, out_param, 1, in_modifier, op_modifier, op, timeout, status); else return mthca_cmd_poll(dev, in_param, out_param, 1, in_modifier, op_modifier, op, timeout, status); } int mthca_cmd_init(struct mthca_dev *dev) { mutex_init(&dev->cmd.hcr_mutex); sema_init(&dev->cmd.poll_sem, 1); dev->cmd.flags = 0; dev->hcr = ioremap(pci_resource_start(dev->pdev, 0) + MTHCA_HCR_BASE, MTHCA_HCR_SIZE); if (!dev->hcr) { mthca_err(dev, "Couldn't map command register."); return -ENOMEM; } dev->cmd.pool = pci_pool_create("mthca_cmd", dev->pdev, MTHCA_MAILBOX_SIZE, MTHCA_MAILBOX_SIZE, 0); if (!dev->cmd.pool) { iounmap(dev->hcr); return -ENOMEM; } return 0; } void mthca_cmd_cleanup(struct mthca_dev *dev) { pci_pool_destroy(dev->cmd.pool); iounmap(dev->hcr); if (dev->cmd.flags & MTHCA_CMD_POST_DOORBELLS) iounmap(dev->cmd.dbell_map); } /* * Switch to using events to issue FW commands (should be called after * event queue to command events has been initialized). */ int mthca_cmd_use_events(struct mthca_dev *dev) { int i; dev->cmd.context = kmalloc(dev->cmd.max_cmds * sizeof (struct mthca_cmd_context), GFP_KERNEL); if (!dev->cmd.context) return -ENOMEM; for (i = 0; i < dev->cmd.max_cmds; ++i) { dev->cmd.context[i].token = i; dev->cmd.context[i].next = i + 1; } dev->cmd.context[dev->cmd.max_cmds - 1].next = -1; dev->cmd.free_head = 0; sema_init(&dev->cmd.event_sem, dev->cmd.max_cmds); spin_lock_init(&dev->cmd.context_lock); for (dev->cmd.token_mask = 1; dev->cmd.token_mask < dev->cmd.max_cmds; dev->cmd.token_mask <<= 1) ; /* nothing */ --dev->cmd.token_mask; dev->cmd.flags |= MTHCA_CMD_USE_EVENTS; down(&dev->cmd.poll_sem); return 0; } /* * Switch back to polling (used when shutting down the device) */ void mthca_cmd_use_polling(struct mthca_dev *dev) { int i; dev->cmd.flags &= ~MTHCA_CMD_USE_EVENTS; for (i = 0; i < dev->cmd.max_cmds; ++i) down(&dev->cmd.event_sem); kfree(dev->cmd.context); up(&dev->cmd.poll_sem); } struct mthca_mailbox *mthca_alloc_mailbox(struct mthca_dev *dev, gfp_t gfp_mask) { struct mthca_mailbox *mailbox; mailbox = kmalloc(sizeof *mailbox, gfp_mask); if (!mailbox) return ERR_PTR(-ENOMEM); mailbox->buf = pci_pool_alloc(dev->cmd.pool, gfp_mask, &mailbox->dma); if (!mailbox->buf) { kfree(mailbox); return ERR_PTR(-ENOMEM); } return mailbox; } void mthca_free_mailbox(struct mthca_dev *dev, struct mthca_mailbox *mailbox) { if (!mailbox) return; pci_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma); kfree(mailbox); } int mthca_SYS_EN(struct mthca_dev *dev, u8 *status) { u64 out; int ret; ret = mthca_cmd_imm(dev, 0, &out, 0, 0, CMD_SYS_EN, CMD_TIME_CLASS_D, status); if (*status == MTHCA_CMD_STAT_DDR_MEM_ERR) mthca_warn(dev, "SYS_EN DDR error: syn=%x, sock=%d, " "sladdr=%d, SPD source=%s\n", (int) (out >> 6) & 0xf, (int) (out >> 4) & 3, (int) (out >> 1) & 7, (int) out & 1 ? "NVMEM" : "DIMM"); return ret; } int mthca_SYS_DIS(struct mthca_dev *dev, u8 *status) { return mthca_cmd(dev, 0, 0, 0, CMD_SYS_DIS, CMD_TIME_CLASS_C, status); } static int mthca_map_cmd(struct mthca_dev *dev, u16 op, struct mthca_icm *icm, u64 virt, u8 *status) { struct mthca_mailbox *mailbox; struct mthca_icm_iter iter; __be64 *pages; int lg; int nent = 0; int i; int err = 0; int ts = 0, tc = 0; mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); memset(mailbox->buf, 0, MTHCA_MAILBOX_SIZE); pages = mailbox->buf; for (mthca_icm_first(icm, &iter); !mthca_icm_last(&iter); mthca_icm_next(&iter)) { /* * We have to pass pages that are aligned to their * size, so find the least significant 1 in the * address or size and use that as our log2 size. */ lg = ffs(mthca_icm_addr(&iter) | mthca_icm_size(&iter)) - 1; if (lg < MTHCA_ICM_PAGE_SHIFT) { mthca_warn(dev, "Got FW area not aligned to %d (%llx/%lx).\n", MTHCA_ICM_PAGE_SIZE, (unsigned long long) mthca_icm_addr(&iter), mthca_icm_size(&iter)); err = -EINVAL; goto out; } for (i = 0; i < mthca_icm_size(&iter) >> lg; ++i) { if (virt != -1) { pages[nent * 2] = cpu_to_be64(virt); virt += 1 << lg; } pages[nent * 2 + 1] = cpu_to_be64((mthca_icm_addr(&iter) + (i << lg)) | (lg - MTHCA_ICM_PAGE_SHIFT)); ts += 1 << (lg - 10); ++tc; if (++nent == MTHCA_MAILBOX_SIZE / 16) { err = mthca_cmd(dev, mailbox->dma, nent, 0, op, CMD_TIME_CLASS_B, status); if (err || *status) goto out; nent = 0; } } } if (nent) err = mthca_cmd(dev, mailbox->dma, nent, 0, op, CMD_TIME_CLASS_B, status); switch (op) { case CMD_MAP_FA: mthca_dbg(dev, "Mapped %d chunks/%d KB for FW.\n", tc, ts); break; case CMD_MAP_ICM_AUX: mthca_dbg(dev, "Mapped %d chunks/%d KB for ICM aux.\n", tc, ts); break; case CMD_MAP_ICM: mthca_dbg(dev, "Mapped %d chunks/%d KB at %llx for ICM.\n", tc, ts, (unsigned long long) virt - (ts << 10)); break; } out: mthca_free_mailbox(dev, mailbox); return err; } int mthca_MAP_FA(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status) { return mthca_map_cmd(dev, CMD_MAP_FA, icm, -1, status); } int mthca_UNMAP_FA(struct mthca_dev *dev, u8 *status) { return mthca_cmd(dev, 0, 0, 0, CMD_UNMAP_FA, CMD_TIME_CLASS_B, status); } int mthca_RUN_FW(struct mthca_dev *dev, u8 *status) { return mthca_cmd(dev, 0, 0, 0, CMD_RUN_FW, CMD_TIME_CLASS_A, status); } static void mthca_setup_cmd_doorbells(struct mthca_dev *dev, u64 base) { unsigned long addr; u16 max_off = 0; int i; for (i = 0; i < 8; ++i) max_off = max(max_off, dev->cmd.dbell_offsets[i]); if ((base & PAGE_MASK) != ((base + max_off) & PAGE_MASK)) { mthca_warn(dev, "Firmware doorbell region at 0x%016llx, " "length 0x%x crosses a page boundary\n", (unsigned long long) base, max_off); return; } addr = pci_resource_start(dev->pdev, 2) + ((pci_resource_len(dev->pdev, 2) - 1) & base); dev->cmd.dbell_map = ioremap(addr, max_off + sizeof(u32)); if (!dev->cmd.dbell_map) return; dev->cmd.flags |= MTHCA_CMD_POST_DOORBELLS; mthca_dbg(dev, "Mapped doorbell page for posting FW commands\n"); } int mthca_QUERY_FW(struct mthca_dev *dev, u8 *status) { struct mthca_mailbox *mailbox; u32 *outbox; u64 base; u32 tmp; int err = 0; u8 lg; int i; #define QUERY_FW_OUT_SIZE 0x100 #define QUERY_FW_VER_OFFSET 0x00 #define QUERY_FW_MAX_CMD_OFFSET 0x0f #define QUERY_FW_ERR_START_OFFSET 0x30 #define QUERY_FW_ERR_SIZE_OFFSET 0x38 #define QUERY_FW_CMD_DB_EN_OFFSET 0x10 #define QUERY_FW_CMD_DB_OFFSET 0x50 #define QUERY_FW_CMD_DB_BASE 0x60 #define QUERY_FW_START_OFFSET 0x20 #define QUERY_FW_END_OFFSET 0x28 #define QUERY_FW_SIZE_OFFSET 0x00 #define QUERY_FW_CLR_INT_BASE_OFFSET 0x20 #define QUERY_FW_EQ_ARM_BASE_OFFSET 0x40 #define QUERY_FW_EQ_SET_CI_BASE_OFFSET 0x48 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); outbox = mailbox->buf; err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_QUERY_FW, CMD_TIME_CLASS_A, status); if (err) goto out; MTHCA_GET(dev->fw_ver, outbox, QUERY_FW_VER_OFFSET); /* * FW subminor version is at more significant bits than minor * version, so swap here. */ dev->fw_ver = (dev->fw_ver & 0xffff00000000ull) | ((dev->fw_ver & 0xffff0000ull) >> 16) | ((dev->fw_ver & 0x0000ffffull) << 16); MTHCA_GET(lg, outbox, QUERY_FW_MAX_CMD_OFFSET); dev->cmd.max_cmds = 1 << lg; mthca_dbg(dev, "FW version %012llx, max commands %d\n", (unsigned long long) dev->fw_ver, dev->cmd.max_cmds); MTHCA_GET(dev->catas_err.addr, outbox, QUERY_FW_ERR_START_OFFSET); MTHCA_GET(dev->catas_err.size, outbox, QUERY_FW_ERR_SIZE_OFFSET); mthca_dbg(dev, "Catastrophic error buffer at 0x%llx, size 0x%x\n", (unsigned long long) dev->catas_err.addr, dev->catas_err.size); MTHCA_GET(tmp, outbox, QUERY_FW_CMD_DB_EN_OFFSET); if (tmp & 0x1) { mthca_dbg(dev, "FW supports commands through doorbells\n"); MTHCA_GET(base, outbox, QUERY_FW_CMD_DB_BASE); for (i = 0; i < MTHCA_CMD_NUM_DBELL_DWORDS; ++i) MTHCA_GET(dev->cmd.dbell_offsets[i], outbox, QUERY_FW_CMD_DB_OFFSET + (i << 1)); mthca_setup_cmd_doorbells(dev, base); } if (mthca_is_memfree(dev)) { MTHCA_GET(dev->fw.arbel.fw_pages, outbox, QUERY_FW_SIZE_OFFSET); MTHCA_GET(dev->fw.arbel.clr_int_base, outbox, QUERY_FW_CLR_INT_BASE_OFFSET); MTHCA_GET(dev->fw.arbel.eq_arm_base, outbox, QUERY_FW_EQ_ARM_BASE_OFFSET); MTHCA_GET(dev->fw.arbel.eq_set_ci_base, outbox, QUERY_FW_EQ_SET_CI_BASE_OFFSET); mthca_dbg(dev, "FW size %d KB\n", dev->fw.arbel.fw_pages << 2); /* * Round up number of system pages needed in case * MTHCA_ICM_PAGE_SIZE < PAGE_SIZE. */ dev->fw.arbel.fw_pages = ALIGN(dev->fw.arbel.fw_pages, PAGE_SIZE / MTHCA_ICM_PAGE_SIZE) >> (PAGE_SHIFT - MTHCA_ICM_PAGE_SHIFT); mthca_dbg(dev, "Clear int @ %llx, EQ arm @ %llx, EQ set CI @ %llx\n", (unsigned long long) dev->fw.arbel.clr_int_base, (unsigned long long) dev->fw.arbel.eq_arm_base, (unsigned long long) dev->fw.arbel.eq_set_ci_base); } else { MTHCA_GET(dev->fw.tavor.fw_start, outbox, QUERY_FW_START_OFFSET); MTHCA_GET(dev->fw.tavor.fw_end, outbox, QUERY_FW_END_OFFSET); mthca_dbg(dev, "FW size %d KB (start %llx, end %llx)\n", (int) ((dev->fw.tavor.fw_end - dev->fw.tavor.fw_start) >> 10), (unsigned long long) dev->fw.tavor.fw_start, (unsigned long long) dev->fw.tavor.fw_end); } out: mthca_free_mailbox(dev, mailbox); return err; } int mthca_ENABLE_LAM(struct mthca_dev *dev, u8 *status) { struct mthca_mailbox *mailbox; u8 info; u32 *outbox; int err = 0; #define ENABLE_LAM_OUT_SIZE 0x100 #define ENABLE_LAM_START_OFFSET 0x00 #define ENABLE_LAM_END_OFFSET 0x08 #define ENABLE_LAM_INFO_OFFSET 0x13 #define ENABLE_LAM_INFO_HIDDEN_FLAG (1 << 4) #define ENABLE_LAM_INFO_ECC_MASK 0x3 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); outbox = mailbox->buf; err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_ENABLE_LAM, CMD_TIME_CLASS_C, status); if (err) goto out; if (*status == MTHCA_CMD_STAT_LAM_NOT_PRE) goto out; MTHCA_GET(dev->ddr_start, outbox, ENABLE_LAM_START_OFFSET); MTHCA_GET(dev->ddr_end, outbox, ENABLE_LAM_END_OFFSET); MTHCA_GET(info, outbox, ENABLE_LAM_INFO_OFFSET); if (!!(info & ENABLE_LAM_INFO_HIDDEN_FLAG) != !!(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) { mthca_info(dev, "FW reports that HCA-attached memory " "is %s hidden; does not match PCI config\n", (info & ENABLE_LAM_INFO_HIDDEN_FLAG) ? "" : "not"); } if (info & ENABLE_LAM_INFO_HIDDEN_FLAG) mthca_dbg(dev, "HCA-attached memory is hidden.\n"); mthca_dbg(dev, "HCA memory size %d KB (start %llx, end %llx)\n", (int) ((dev->ddr_end - dev->ddr_start) >> 10), (unsigned long long) dev->ddr_start, (unsigned long long) dev->ddr_end); out: mthca_free_mailbox(dev, mailbox); return err; } int mthca_DISABLE_LAM(struct mthca_dev *dev, u8 *status) { return mthca_cmd(dev, 0, 0, 0, CMD_SYS_DIS, CMD_TIME_CLASS_C, status); } int mthca_QUERY_DDR(struct mthca_dev *dev, u8 *status) { struct mthca_mailbox *mailbox; u8 info; u32 *outbox; int err = 0; #define QUERY_DDR_OUT_SIZE 0x100 #define QUERY_DDR_START_OFFSET 0x00 #define QUERY_DDR_END_OFFSET 0x08 #define QUERY_DDR_INFO_OFFSET 0x13 #define QUERY_DDR_INFO_HIDDEN_FLAG (1 << 4) #define QUERY_DDR_INFO_ECC_MASK 0x3 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); outbox = mailbox->buf; err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_QUERY_DDR, CMD_TIME_CLASS_A, status); if (err) goto out; MTHCA_GET(dev->ddr_start, outbox, QUERY_DDR_START_OFFSET); MTHCA_GET(dev->ddr_end, outbox, QUERY_DDR_END_OFFSET); MTHCA_GET(info, outbox, QUERY_DDR_INFO_OFFSET); if (!!(info & QUERY_DDR_INFO_HIDDEN_FLAG) != !!(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) { mthca_info(dev, "FW reports that HCA-attached memory " "is %s hidden; does not match PCI config\n", (info & QUERY_DDR_INFO_HIDDEN_FLAG) ? "" : "not"); } if (info & QUERY_DDR_INFO_HIDDEN_FLAG) mthca_dbg(dev, "HCA-attached memory is hidden.\n"); mthca_dbg(dev, "HCA memory size %d KB (start %llx, end %llx)\n", (int) ((dev->ddr_end - dev->ddr_start) >> 10), (unsigned long long) dev->ddr_start, (unsigned long long) dev->ddr_end); out: mthca_free_mailbox(dev, mailbox); return err; } int mthca_QUERY_DEV_LIM(struct mthca_dev *dev, struct mthca_dev_lim *dev_lim, u8 *status) { struct mthca_mailbox *mailbox; u32 *outbox; u8 field; u16 size; u16 stat_rate; int err; #define QUERY_DEV_LIM_OUT_SIZE 0x100 #define QUERY_DEV_LIM_MAX_SRQ_SZ_OFFSET 0x10 #define QUERY_DEV_LIM_MAX_QP_SZ_OFFSET 0x11 #define QUERY_DEV_LIM_RSVD_QP_OFFSET 0x12 #define QUERY_DEV_LIM_MAX_QP_OFFSET 0x13 #define QUERY_DEV_LIM_RSVD_SRQ_OFFSET 0x14 #define QUERY_DEV_LIM_MAX_SRQ_OFFSET 0x15 #define QUERY_DEV_LIM_RSVD_EEC_OFFSET 0x16 #define QUERY_DEV_LIM_MAX_EEC_OFFSET 0x17 #define QUERY_DEV_LIM_MAX_CQ_SZ_OFFSET 0x19 #define QUERY_DEV_LIM_RSVD_CQ_OFFSET 0x1a #define QUERY_DEV_LIM_MAX_CQ_OFFSET 0x1b #define QUERY_DEV_LIM_MAX_MPT_OFFSET 0x1d #define QUERY_DEV_LIM_RSVD_EQ_OFFSET 0x1e #define QUERY_DEV_LIM_MAX_EQ_OFFSET 0x1f #define QUERY_DEV_LIM_RSVD_MTT_OFFSET 0x20 #define QUERY_DEV_LIM_MAX_MRW_SZ_OFFSET 0x21 #define QUERY_DEV_LIM_RSVD_MRW_OFFSET 0x22 #define QUERY_DEV_LIM_MAX_MTT_SEG_OFFSET 0x23 #define QUERY_DEV_LIM_MAX_AV_OFFSET 0x27 #define QUERY_DEV_LIM_MAX_REQ_QP_OFFSET 0x29 #define QUERY_DEV_LIM_MAX_RES_QP_OFFSET 0x2b #define QUERY_DEV_LIM_MAX_RDMA_OFFSET 0x2f #define QUERY_DEV_LIM_RSZ_SRQ_OFFSET 0x33 #define QUERY_DEV_LIM_ACK_DELAY_OFFSET 0x35 #define QUERY_DEV_LIM_MTU_WIDTH_OFFSET 0x36 #define QUERY_DEV_LIM_VL_PORT_OFFSET 0x37 #define QUERY_DEV_LIM_MAX_GID_OFFSET 0x3b #define QUERY_DEV_LIM_RATE_SUPPORT_OFFSET 0x3c #define QUERY_DEV_LIM_MAX_PKEY_OFFSET 0x3f #define QUERY_DEV_LIM_FLAGS_OFFSET 0x44 #define QUERY_DEV_LIM_RSVD_UAR_OFFSET 0x48 #define QUERY_DEV_LIM_UAR_SZ_OFFSET 0x49 #define QUERY_DEV_LIM_PAGE_SZ_OFFSET 0x4b #define QUERY_DEV_LIM_MAX_SG_OFFSET 0x51 #define QUERY_DEV_LIM_MAX_DESC_SZ_OFFSET 0x52 #define QUERY_DEV_LIM_MAX_SG_RQ_OFFSET 0x55 #define QUERY_DEV_LIM_MAX_DESC_SZ_RQ_OFFSET 0x56 #define QUERY_DEV_LIM_MAX_QP_MCG_OFFSET 0x61 #define QUERY_DEV_LIM_RSVD_MCG_OFFSET 0x62 #define QUERY_DEV_LIM_MAX_MCG_OFFSET 0x63 #define QUERY_DEV_LIM_RSVD_PD_OFFSET 0x64 #define QUERY_DEV_LIM_MAX_PD_OFFSET 0x65 #define QUERY_DEV_LIM_RSVD_RDD_OFFSET 0x66 #define QUERY_DEV_LIM_MAX_RDD_OFFSET 0x67 #define QUERY_DEV_LIM_EEC_ENTRY_SZ_OFFSET 0x80 #define QUERY_DEV_LIM_QPC_ENTRY_SZ_OFFSET 0x82 #define QUERY_DEV_LIM_EEEC_ENTRY_SZ_OFFSET 0x84 #define QUERY_DEV_LIM_EQPC_ENTRY_SZ_OFFSET 0x86 #define QUERY_DEV_LIM_EQC_ENTRY_SZ_OFFSET 0x88 #define QUERY_DEV_LIM_CQC_ENTRY_SZ_OFFSET 0x8a #define QUERY_DEV_LIM_SRQ_ENTRY_SZ_OFFSET 0x8c #define QUERY_DEV_LIM_UAR_ENTRY_SZ_OFFSET 0x8e #define QUERY_DEV_LIM_MTT_ENTRY_SZ_OFFSET 0x90 #define QUERY_DEV_LIM_MPT_ENTRY_SZ_OFFSET 0x92 #define QUERY_DEV_LIM_PBL_SZ_OFFSET 0x96 #define QUERY_DEV_LIM_BMME_FLAGS_OFFSET 0x97 #define QUERY_DEV_LIM_RSVD_LKEY_OFFSET 0x98 #define QUERY_DEV_LIM_LAMR_OFFSET 0x9f #define QUERY_DEV_LIM_MAX_ICM_SZ_OFFSET 0xa0 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); outbox = mailbox->buf; err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_QUERY_DEV_LIM, CMD_TIME_CLASS_A, status); if (err) goto out; MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_QP_OFFSET); dev_lim->reserved_qps = 1 << (field & 0xf); MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_QP_OFFSET); dev_lim->max_qps = 1 << (field & 0x1f); MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_SRQ_OFFSET); dev_lim->reserved_srqs = 1 << (field >> 4); MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SRQ_OFFSET); dev_lim->max_srqs = 1 << (field & 0x1f); MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_EEC_OFFSET); dev_lim->reserved_eecs = 1 << (field & 0xf); MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_EEC_OFFSET); dev_lim->max_eecs = 1 << (field & 0x1f); MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_CQ_SZ_OFFSET); dev_lim->max_cq_sz = 1 << field; MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_CQ_OFFSET); dev_lim->reserved_cqs = 1 << (field & 0xf); MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_CQ_OFFSET); dev_lim->max_cqs = 1 << (field & 0x1f); MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_MPT_OFFSET); dev_lim->max_mpts = 1 << (field & 0x3f); MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_EQ_OFFSET); dev_lim->reserved_eqs = 1 << (field & 0xf); MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_EQ_OFFSET); dev_lim->max_eqs = 1 << (field & 0x7); MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_MTT_OFFSET); if (mthca_is_memfree(dev)) dev_lim->reserved_mtts = ALIGN((1 << (field >> 4)) * sizeof(u64), dev->limits.mtt_seg_size) / dev->limits.mtt_seg_size; else dev_lim->reserved_mtts = 1 << (field >> 4); MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_MRW_SZ_OFFSET); dev_lim->max_mrw_sz = 1 << field; MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_MRW_OFFSET); dev_lim->reserved_mrws = 1 << (field & 0xf); MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_MTT_SEG_OFFSET); dev_lim->max_mtt_seg = 1 << (field & 0x3f); MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_REQ_QP_OFFSET); dev_lim->max_requester_per_qp = 1 << (field & 0x3f); MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_RES_QP_OFFSET); dev_lim->max_responder_per_qp = 1 << (field & 0x3f); MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_RDMA_OFFSET); dev_lim->max_rdma_global = 1 << (field & 0x3f); MTHCA_GET(field, outbox, QUERY_DEV_LIM_ACK_DELAY_OFFSET); dev_lim->local_ca_ack_delay = field & 0x1f; MTHCA_GET(field, outbox, QUERY_DEV_LIM_MTU_WIDTH_OFFSET); dev_lim->max_mtu = field >> 4; dev_lim->max_port_width = field & 0xf; MTHCA_GET(field, outbox, QUERY_DEV_LIM_VL_PORT_OFFSET); dev_lim->max_vl = field >> 4; dev_lim->num_ports = field & 0xf; MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_GID_OFFSET); dev_lim->max_gids = 1 << (field & 0xf); MTHCA_GET(stat_rate, outbox, QUERY_DEV_LIM_RATE_SUPPORT_OFFSET); dev_lim->stat_rate_support = stat_rate; MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_PKEY_OFFSET); dev_lim->max_pkeys = 1 << (field & 0xf); MTHCA_GET(dev_lim->flags, outbox, QUERY_DEV_LIM_FLAGS_OFFSET); MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_UAR_OFFSET); dev_lim->reserved_uars = field >> 4; MTHCA_GET(field, outbox, QUERY_DEV_LIM_UAR_SZ_OFFSET); dev_lim->uar_size = 1 << ((field & 0x3f) + 20); MTHCA_GET(field, outbox, QUERY_DEV_LIM_PAGE_SZ_OFFSET); dev_lim->min_page_sz = 1 << field; MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SG_OFFSET); dev_lim->max_sg = field; MTHCA_GET(size, outbox, QUERY_DEV_LIM_MAX_DESC_SZ_OFFSET); dev_lim->max_desc_sz = size; MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_QP_MCG_OFFSET); dev_lim->max_qp_per_mcg = 1 << field; MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_MCG_OFFSET); dev_lim->reserved_mgms = field & 0xf; MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_MCG_OFFSET); dev_lim->max_mcgs = 1 << field; MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_PD_OFFSET); dev_lim->reserved_pds = field >> 4; MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_PD_OFFSET); dev_lim->max_pds = 1 << (field & 0x3f); MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_RDD_OFFSET); dev_lim->reserved_rdds = field >> 4; MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_RDD_OFFSET); dev_lim->max_rdds = 1 << (field & 0x3f); MTHCA_GET(size, outbox, QUERY_DEV_LIM_EEC_ENTRY_SZ_OFFSET); dev_lim->eec_entry_sz = size; MTHCA_GET(size, outbox, QUERY_DEV_LIM_QPC_ENTRY_SZ_OFFSET); dev_lim->qpc_entry_sz = size; MTHCA_GET(size, outbox, QUERY_DEV_LIM_EEEC_ENTRY_SZ_OFFSET); dev_lim->eeec_entry_sz = size; MTHCA_GET(size, outbox, QUERY_DEV_LIM_EQPC_ENTRY_SZ_OFFSET); dev_lim->eqpc_entry_sz = size; MTHCA_GET(size, outbox, QUERY_DEV_LIM_EQC_ENTRY_SZ_OFFSET); dev_lim->eqc_entry_sz = size; MTHCA_GET(size, outbox, QUERY_DEV_LIM_CQC_ENTRY_SZ_OFFSET); dev_lim->cqc_entry_sz = size; MTHCA_GET(size, outbox, QUERY_DEV_LIM_SRQ_ENTRY_SZ_OFFSET); dev_lim->srq_entry_sz = size; MTHCA_GET(size, outbox, QUERY_DEV_LIM_UAR_ENTRY_SZ_OFFSET); dev_lim->uar_scratch_entry_sz = size; if (mthca_is_memfree(dev)) { MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SRQ_SZ_OFFSET); dev_lim->max_srq_sz = 1 << field; MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_QP_SZ_OFFSET); dev_lim->max_qp_sz = 1 << field; MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSZ_SRQ_OFFSET); dev_lim->hca.arbel.resize_srq = field & 1; MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SG_RQ_OFFSET); dev_lim->max_sg = min_t(int, field, dev_lim->max_sg); MTHCA_GET(size, outbox, QUERY_DEV_LIM_MAX_DESC_SZ_RQ_OFFSET); dev_lim->max_desc_sz = min_t(int, size, dev_lim->max_desc_sz); MTHCA_GET(size, outbox, QUERY_DEV_LIM_MPT_ENTRY_SZ_OFFSET); dev_lim->mpt_entry_sz = size; MTHCA_GET(field, outbox, QUERY_DEV_LIM_PBL_SZ_OFFSET); dev_lim->hca.arbel.max_pbl_sz = 1 << (field & 0x3f); MTHCA_GET(dev_lim->hca.arbel.bmme_flags, outbox, QUERY_DEV_LIM_BMME_FLAGS_OFFSET); MTHCA_GET(dev_lim->hca.arbel.reserved_lkey, outbox, QUERY_DEV_LIM_RSVD_LKEY_OFFSET); MTHCA_GET(field, outbox, QUERY_DEV_LIM_LAMR_OFFSET); dev_lim->hca.arbel.lam_required = field & 1; MTHCA_GET(dev_lim->hca.arbel.max_icm_sz, outbox, QUERY_DEV_LIM_MAX_ICM_SZ_OFFSET); if (dev_lim->hca.arbel.bmme_flags & 1) mthca_dbg(dev, "Base MM extensions: yes " "(flags %d, max PBL %d, rsvd L_Key %08x)\n", dev_lim->hca.arbel.bmme_flags, dev_lim->hca.arbel.max_pbl_sz, dev_lim->hca.arbel.reserved_lkey); else mthca_dbg(dev, "Base MM extensions: no\n"); mthca_dbg(dev, "Max ICM size %lld MB\n", (unsigned long long) dev_lim->hca.arbel.max_icm_sz >> 20); } else { MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SRQ_SZ_OFFSET); dev_lim->max_srq_sz = (1 << field) - 1; MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_QP_SZ_OFFSET); dev_lim->max_qp_sz = (1 << field) - 1; MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_AV_OFFSET); dev_lim->hca.tavor.max_avs = 1 << (field & 0x3f); dev_lim->mpt_entry_sz = MTHCA_MPT_ENTRY_SIZE; } mthca_dbg(dev, "Max QPs: %d, reserved QPs: %d, entry size: %d\n", dev_lim->max_qps, dev_lim->reserved_qps, dev_lim->qpc_entry_sz); mthca_dbg(dev, "Max SRQs: %d, reserved SRQs: %d, entry size: %d\n", dev_lim->max_srqs, dev_lim->reserved_srqs, dev_lim->srq_entry_sz); mthca_dbg(dev, "Max CQs: %d, reserved CQs: %d, entry size: %d\n", dev_lim->max_cqs, dev_lim->reserved_cqs, dev_lim->cqc_entry_sz); mthca_dbg(dev, "Max EQs: %d, reserved EQs: %d, entry size: %d\n", dev_lim->max_eqs, dev_lim->reserved_eqs, dev_lim->eqc_entry_sz); mthca_dbg(dev, "reserved MPTs: %d, reserved MTTs: %d\n", dev_lim->reserved_mrws, dev_lim->reserved_mtts); mthca_dbg(dev, "Max PDs: %d, reserved PDs: %d, reserved UARs: %d\n", dev_lim->max_pds, dev_lim->reserved_pds, dev_lim->reserved_uars); mthca_dbg(dev, "Max QP/MCG: %d, reserved MGMs: %d\n", dev_lim->max_pds, dev_lim->reserved_mgms); mthca_dbg(dev, "Max CQEs: %d, max WQEs: %d, max SRQ WQEs: %d\n", dev_lim->max_cq_sz, dev_lim->max_qp_sz, dev_lim->max_srq_sz); mthca_dbg(dev, "Flags: %08x\n", dev_lim->flags); out: mthca_free_mailbox(dev, mailbox); return err; } static void get_board_id(void *vsd, char *board_id) { int i; #define VSD_OFFSET_SIG1 0x00 #define VSD_OFFSET_SIG2 0xde #define VSD_OFFSET_MLX_BOARD_ID 0xd0 #define VSD_OFFSET_TS_BOARD_ID 0x20 #define VSD_SIGNATURE_TOPSPIN 0x5ad memset(board_id, 0, MTHCA_BOARD_ID_LEN); if (be16_to_cpup(vsd + VSD_OFFSET_SIG1) == VSD_SIGNATURE_TOPSPIN && be16_to_cpup(vsd + VSD_OFFSET_SIG2) == VSD_SIGNATURE_TOPSPIN) { strlcpy(board_id, vsd + VSD_OFFSET_TS_BOARD_ID, MTHCA_BOARD_ID_LEN); } else { /* * The board ID is a string but the firmware byte * swaps each 4-byte word before passing it back to * us. Therefore we need to swab it before printing. */ for (i = 0; i < 4; ++i) ((u32 *) board_id)[i] = swab32(*(u32 *) (vsd + VSD_OFFSET_MLX_BOARD_ID + i * 4)); } } int mthca_QUERY_ADAPTER(struct mthca_dev *dev, struct mthca_adapter *adapter, u8 *status) { struct mthca_mailbox *mailbox; u32 *outbox; int err; #define QUERY_ADAPTER_OUT_SIZE 0x100 #define QUERY_ADAPTER_VENDOR_ID_OFFSET 0x00 #define QUERY_ADAPTER_DEVICE_ID_OFFSET 0x04 #define QUERY_ADAPTER_REVISION_ID_OFFSET 0x08 #define QUERY_ADAPTER_INTA_PIN_OFFSET 0x10 #define QUERY_ADAPTER_VSD_OFFSET 0x20 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); outbox = mailbox->buf; err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_QUERY_ADAPTER, CMD_TIME_CLASS_A, status); if (err) goto out; if (!mthca_is_memfree(dev)) { MTHCA_GET(adapter->vendor_id, outbox, QUERY_ADAPTER_VENDOR_ID_OFFSET); MTHCA_GET(adapter->device_id, outbox, QUERY_ADAPTER_DEVICE_ID_OFFSET); MTHCA_GET(adapter->revision_id, outbox, QUERY_ADAPTER_REVISION_ID_OFFSET); } MTHCA_GET(adapter->inta_pin, outbox, QUERY_ADAPTER_INTA_PIN_OFFSET); get_board_id(outbox + QUERY_ADAPTER_VSD_OFFSET / 4, adapter->board_id); out: mthca_free_mailbox(dev, mailbox); return err; } int mthca_INIT_HCA(struct mthca_dev *dev, struct mthca_init_hca_param *param, u8 *status) { struct mthca_mailbox *mailbox; __be32 *inbox; int err; #define INIT_HCA_IN_SIZE 0x200 #define INIT_HCA_FLAGS1_OFFSET 0x00c #define INIT_HCA_FLAGS2_OFFSET 0x014 #define INIT_HCA_QPC_OFFSET 0x020 #define INIT_HCA_QPC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x10) #define INIT_HCA_LOG_QP_OFFSET (INIT_HCA_QPC_OFFSET + 0x17) #define INIT_HCA_EEC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x20) #define INIT_HCA_LOG_EEC_OFFSET (INIT_HCA_QPC_OFFSET + 0x27) #define INIT_HCA_SRQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x28) #define INIT_HCA_LOG_SRQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x2f) #define INIT_HCA_CQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x30) #define INIT_HCA_LOG_CQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x37) #define INIT_HCA_EQPC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x40) #define INIT_HCA_EEEC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x50) #define INIT_HCA_EQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x60) #define INIT_HCA_LOG_EQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x67) #define INIT_HCA_RDB_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x70) #define INIT_HCA_UDAV_OFFSET 0x0b0 #define INIT_HCA_UDAV_LKEY_OFFSET (INIT_HCA_UDAV_OFFSET + 0x0) #define INIT_HCA_UDAV_PD_OFFSET (INIT_HCA_UDAV_OFFSET + 0x4) #define INIT_HCA_MCAST_OFFSET 0x0c0 #define INIT_HCA_MC_BASE_OFFSET (INIT_HCA_MCAST_OFFSET + 0x00) #define INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x12) #define INIT_HCA_MC_HASH_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x16) #define INIT_HCA_LOG_MC_TABLE_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x1b) #define INIT_HCA_TPT_OFFSET 0x0f0 #define INIT_HCA_MPT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x00) #define INIT_HCA_MTT_SEG_SZ_OFFSET (INIT_HCA_TPT_OFFSET + 0x09) #define INIT_HCA_LOG_MPT_SZ_OFFSET (INIT_HCA_TPT_OFFSET + 0x0b) #define INIT_HCA_MTT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x10) #define INIT_HCA_UAR_OFFSET 0x120 #define INIT_HCA_UAR_BASE_OFFSET (INIT_HCA_UAR_OFFSET + 0x00) #define INIT_HCA_UARC_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x09) #define INIT_HCA_LOG_UAR_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x0a) #define INIT_HCA_UAR_PAGE_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x0b) #define INIT_HCA_UAR_SCATCH_BASE_OFFSET (INIT_HCA_UAR_OFFSET + 0x10) #define INIT_HCA_UAR_CTX_BASE_OFFSET (INIT_HCA_UAR_OFFSET + 0x18) mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); inbox = mailbox->buf; memset(inbox, 0, INIT_HCA_IN_SIZE); if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT) MTHCA_PUT(inbox, 0x1, INIT_HCA_FLAGS1_OFFSET); #if defined(__LITTLE_ENDIAN) *(inbox + INIT_HCA_FLAGS2_OFFSET / 4) &= ~cpu_to_be32(1 << 1); #elif defined(__BIG_ENDIAN) *(inbox + INIT_HCA_FLAGS2_OFFSET / 4) |= cpu_to_be32(1 << 1); #else #error Host endianness not defined #endif /* Check port for UD address vector: */ *(inbox + INIT_HCA_FLAGS2_OFFSET / 4) |= cpu_to_be32(1); /* Enable IPoIB checksumming if we can: */ if (dev->device_cap_flags & IB_DEVICE_UD_IP_CSUM) *(inbox + INIT_HCA_FLAGS2_OFFSET / 4) |= cpu_to_be32(7 << 3); /* We leave wqe_quota, responder_exu, etc as 0 (default) */ /* QPC/EEC/CQC/EQC/RDB attributes */ MTHCA_PUT(inbox, param->qpc_base, INIT_HCA_QPC_BASE_OFFSET); MTHCA_PUT(inbox, param->log_num_qps, INIT_HCA_LOG_QP_OFFSET); MTHCA_PUT(inbox, param->eec_base, INIT_HCA_EEC_BASE_OFFSET); MTHCA_PUT(inbox, param->log_num_eecs, INIT_HCA_LOG_EEC_OFFSET); MTHCA_PUT(inbox, param->srqc_base, INIT_HCA_SRQC_BASE_OFFSET); MTHCA_PUT(inbox, param->log_num_srqs, INIT_HCA_LOG_SRQ_OFFSET); MTHCA_PUT(inbox, param->cqc_base, INIT_HCA_CQC_BASE_OFFSET); MTHCA_PUT(inbox, param->log_num_cqs, INIT_HCA_LOG_CQ_OFFSET); MTHCA_PUT(inbox, param->eqpc_base, INIT_HCA_EQPC_BASE_OFFSET); MTHCA_PUT(inbox, param->eeec_base, INIT_HCA_EEEC_BASE_OFFSET); MTHCA_PUT(inbox, param->eqc_base, INIT_HCA_EQC_BASE_OFFSET); MTHCA_PUT(inbox, param->log_num_eqs, INIT_HCA_LOG_EQ_OFFSET); MTHCA_PUT(inbox, param->rdb_base, INIT_HCA_RDB_BASE_OFFSET); /* UD AV attributes */ /* multicast attributes */ MTHCA_PUT(inbox, param->mc_base, INIT_HCA_MC_BASE_OFFSET); MTHCA_PUT(inbox, param->log_mc_entry_sz, INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET); MTHCA_PUT(inbox, param->mc_hash_sz, INIT_HCA_MC_HASH_SZ_OFFSET); MTHCA_PUT(inbox, param->log_mc_table_sz, INIT_HCA_LOG_MC_TABLE_SZ_OFFSET); /* TPT attributes */ MTHCA_PUT(inbox, param->mpt_base, INIT_HCA_MPT_BASE_OFFSET); if (!mthca_is_memfree(dev)) MTHCA_PUT(inbox, param->mtt_seg_sz, INIT_HCA_MTT_SEG_SZ_OFFSET); MTHCA_PUT(inbox, param->log_mpt_sz, INIT_HCA_LOG_MPT_SZ_OFFSET); MTHCA_PUT(inbox, param->mtt_base, INIT_HCA_MTT_BASE_OFFSET); /* UAR attributes */ { u8 uar_page_sz = PAGE_SHIFT - 12; MTHCA_PUT(inbox, uar_page_sz, INIT_HCA_UAR_PAGE_SZ_OFFSET); } MTHCA_PUT(inbox, param->uar_scratch_base, INIT_HCA_UAR_SCATCH_BASE_OFFSET); if (mthca_is_memfree(dev)) { MTHCA_PUT(inbox, param->log_uarc_sz, INIT_HCA_UARC_SZ_OFFSET); MTHCA_PUT(inbox, param->log_uar_sz, INIT_HCA_LOG_UAR_SZ_OFFSET); MTHCA_PUT(inbox, param->uarc_base, INIT_HCA_UAR_CTX_BASE_OFFSET); } err = mthca_cmd(dev, mailbox->dma, 0, 0, CMD_INIT_HCA, CMD_TIME_CLASS_D, status); mthca_free_mailbox(dev, mailbox); return err; } int mthca_INIT_IB(struct mthca_dev *dev, struct mthca_init_ib_param *param, int port, u8 *status) { struct mthca_mailbox *mailbox; u32 *inbox; int err; u32 flags; #define INIT_IB_IN_SIZE 56 #define INIT_IB_FLAGS_OFFSET 0x00 #define INIT_IB_FLAG_SIG (1 << 18) #define INIT_IB_FLAG_NG (1 << 17) #define INIT_IB_FLAG_G0 (1 << 16) #define INIT_IB_VL_SHIFT 4 #define INIT_IB_PORT_WIDTH_SHIFT 8 #define INIT_IB_MTU_SHIFT 12 #define INIT_IB_MAX_GID_OFFSET 0x06 #define INIT_IB_MAX_PKEY_OFFSET 0x0a #define INIT_IB_GUID0_OFFSET 0x10 #define INIT_IB_NODE_GUID_OFFSET 0x18 #define INIT_IB_SI_GUID_OFFSET 0x20 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); inbox = mailbox->buf; memset(inbox, 0, INIT_IB_IN_SIZE); flags = 0; flags |= param->set_guid0 ? INIT_IB_FLAG_G0 : 0; flags |= param->set_node_guid ? INIT_IB_FLAG_NG : 0; flags |= param->set_si_guid ? INIT_IB_FLAG_SIG : 0; flags |= param->vl_cap << INIT_IB_VL_SHIFT; flags |= param->port_width << INIT_IB_PORT_WIDTH_SHIFT; flags |= param->mtu_cap << INIT_IB_MTU_SHIFT; MTHCA_PUT(inbox, flags, INIT_IB_FLAGS_OFFSET); MTHCA_PUT(inbox, param->gid_cap, INIT_IB_MAX_GID_OFFSET); MTHCA_PUT(inbox, param->pkey_cap, INIT_IB_MAX_PKEY_OFFSET); MTHCA_PUT(inbox, param->guid0, INIT_IB_GUID0_OFFSET); MTHCA_PUT(inbox, param->node_guid, INIT_IB_NODE_GUID_OFFSET); MTHCA_PUT(inbox, param->si_guid, INIT_IB_SI_GUID_OFFSET); err = mthca_cmd(dev, mailbox->dma, port, 0, CMD_INIT_IB, CMD_TIME_CLASS_A, status); mthca_free_mailbox(dev, mailbox); return err; } int mthca_CLOSE_IB(struct mthca_dev *dev, int port, u8 *status) { return mthca_cmd(dev, 0, port, 0, CMD_CLOSE_IB, CMD_TIME_CLASS_A, status); } int mthca_CLOSE_HCA(struct mthca_dev *dev, int panic, u8 *status) { return mthca_cmd(dev, 0, 0, panic, CMD_CLOSE_HCA, CMD_TIME_CLASS_C, status); } int mthca_SET_IB(struct mthca_dev *dev, struct mthca_set_ib_param *param, int port, u8 *status) { struct mthca_mailbox *mailbox; u32 *inbox; int err; u32 flags = 0; #define SET_IB_IN_SIZE 0x40 #define SET_IB_FLAGS_OFFSET 0x00 #define SET_IB_FLAG_SIG (1 << 18) #define SET_IB_FLAG_RQK (1 << 0) #define SET_IB_CAP_MASK_OFFSET 0x04 #define SET_IB_SI_GUID_OFFSET 0x08 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); inbox = mailbox->buf; memset(inbox, 0, SET_IB_IN_SIZE); flags |= param->set_si_guid ? SET_IB_FLAG_SIG : 0; flags |= param->reset_qkey_viol ? SET_IB_FLAG_RQK : 0; MTHCA_PUT(inbox, flags, SET_IB_FLAGS_OFFSET); MTHCA_PUT(inbox, param->cap_mask, SET_IB_CAP_MASK_OFFSET); MTHCA_PUT(inbox, param->si_guid, SET_IB_SI_GUID_OFFSET); err = mthca_cmd(dev, mailbox->dma, port, 0, CMD_SET_IB, CMD_TIME_CLASS_B, status); mthca_free_mailbox(dev, mailbox); return err; } int mthca_MAP_ICM(struct mthca_dev *dev, struct mthca_icm *icm, u64 virt, u8 *status) { return mthca_map_cmd(dev, CMD_MAP_ICM, icm, virt, status); } int mthca_MAP_ICM_page(struct mthca_dev *dev, u64 dma_addr, u64 virt, u8 *status) { struct mthca_mailbox *mailbox; __be64 *inbox; int err; mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); inbox = mailbox->buf; inbox[0] = cpu_to_be64(virt); inbox[1] = cpu_to_be64(dma_addr); err = mthca_cmd(dev, mailbox->dma, 1, 0, CMD_MAP_ICM, CMD_TIME_CLASS_B, status); mthca_free_mailbox(dev, mailbox); if (!err) mthca_dbg(dev, "Mapped page at %llx to %llx for ICM.\n", (unsigned long long) dma_addr, (unsigned long long) virt); return err; } int mthca_UNMAP_ICM(struct mthca_dev *dev, u64 virt, u32 page_count, u8 *status) { mthca_dbg(dev, "Unmapping %d pages at %llx from ICM.\n", page_count, (unsigned long long) virt); return mthca_cmd(dev, virt, page_count, 0, CMD_UNMAP_ICM, CMD_TIME_CLASS_B, status); } int mthca_MAP_ICM_AUX(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status) { return mthca_map_cmd(dev, CMD_MAP_ICM_AUX, icm, -1, status); } int mthca_UNMAP_ICM_AUX(struct mthca_dev *dev, u8 *status) { return mthca_cmd(dev, 0, 0, 0, CMD_UNMAP_ICM_AUX, CMD_TIME_CLASS_B, status); } int mthca_SET_ICM_SIZE(struct mthca_dev *dev, u64 icm_size, u64 *aux_pages, u8 *status) { int ret = mthca_cmd_imm(dev, icm_size, aux_pages, 0, 0, CMD_SET_ICM_SIZE, CMD_TIME_CLASS_A, status); if (ret || status) return ret; /* * Round up number of system pages needed in case * MTHCA_ICM_PAGE_SIZE < PAGE_SIZE. */ *aux_pages = ALIGN(*aux_pages, PAGE_SIZE / MTHCA_ICM_PAGE_SIZE) >> (PAGE_SHIFT - MTHCA_ICM_PAGE_SHIFT); return 0; } int mthca_SW2HW_MPT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, int mpt_index, u8 *status) { return mthca_cmd(dev, mailbox->dma, mpt_index, 0, CMD_SW2HW_MPT, CMD_TIME_CLASS_B, status); } int mthca_HW2SW_MPT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, int mpt_index, u8 *status) { return mthca_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, mpt_index, !mailbox, CMD_HW2SW_MPT, CMD_TIME_CLASS_B, status); } int mthca_WRITE_MTT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, int num_mtt, u8 *status) { return mthca_cmd(dev, mailbox->dma, num_mtt, 0, CMD_WRITE_MTT, CMD_TIME_CLASS_B, status); } int mthca_SYNC_TPT(struct mthca_dev *dev, u8 *status) { return mthca_cmd(dev, 0, 0, 0, CMD_SYNC_TPT, CMD_TIME_CLASS_B, status); } int mthca_MAP_EQ(struct mthca_dev *dev, u64 event_mask, int unmap, int eq_num, u8 *status) { mthca_dbg(dev, "%s mask %016llx for eqn %d\n", unmap ? "Clearing" : "Setting", (unsigned long long) event_mask, eq_num); return mthca_cmd(dev, event_mask, (unmap << 31) | eq_num, 0, CMD_MAP_EQ, CMD_TIME_CLASS_B, status); } int mthca_SW2HW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, int eq_num, u8 *status) { return mthca_cmd(dev, mailbox->dma, eq_num, 0, CMD_SW2HW_EQ, CMD_TIME_CLASS_A, status); } int mthca_HW2SW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, int eq_num, u8 *status) { return mthca_cmd_box(dev, 0, mailbox->dma, eq_num, 0, CMD_HW2SW_EQ, CMD_TIME_CLASS_A, status); } int mthca_SW2HW_CQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, int cq_num, u8 *status) { return mthca_cmd(dev, mailbox->dma, cq_num, 0, CMD_SW2HW_CQ, CMD_TIME_CLASS_A, status); } int mthca_HW2SW_CQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, int cq_num, u8 *status) { return mthca_cmd_box(dev, 0, mailbox->dma, cq_num, 0, CMD_HW2SW_CQ, CMD_TIME_CLASS_A, status); } int mthca_RESIZE_CQ(struct mthca_dev *dev, int cq_num, u32 lkey, u8 log_size, u8 *status) { struct mthca_mailbox *mailbox; __be32 *inbox; int err; #define RESIZE_CQ_IN_SIZE 0x40 #define RESIZE_CQ_LOG_SIZE_OFFSET 0x0c #define RESIZE_CQ_LKEY_OFFSET 0x1c mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); inbox = mailbox->buf; memset(inbox, 0, RESIZE_CQ_IN_SIZE); /* * Leave start address fields zeroed out -- mthca assumes that * MRs for CQs always start at virtual address 0. */ MTHCA_PUT(inbox, log_size, RESIZE_CQ_LOG_SIZE_OFFSET); MTHCA_PUT(inbox, lkey, RESIZE_CQ_LKEY_OFFSET); err = mthca_cmd(dev, mailbox->dma, cq_num, 1, CMD_RESIZE_CQ, CMD_TIME_CLASS_B, status); mthca_free_mailbox(dev, mailbox); return err; } int mthca_SW2HW_SRQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, int srq_num, u8 *status) { return mthca_cmd(dev, mailbox->dma, srq_num, 0, CMD_SW2HW_SRQ, CMD_TIME_CLASS_A, status); } int mthca_HW2SW_SRQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, int srq_num, u8 *status) { return mthca_cmd_box(dev, 0, mailbox->dma, srq_num, 0, CMD_HW2SW_SRQ, CMD_TIME_CLASS_A, status); } int mthca_QUERY_SRQ(struct mthca_dev *dev, u32 num, struct mthca_mailbox *mailbox, u8 *status) { return mthca_cmd_box(dev, 0, mailbox->dma, num, 0, CMD_QUERY_SRQ, CMD_TIME_CLASS_A, status); } int mthca_ARM_SRQ(struct mthca_dev *dev, int srq_num, int limit, u8 *status) { return mthca_cmd(dev, limit, srq_num, 0, CMD_ARM_SRQ, CMD_TIME_CLASS_B, status); } int mthca_MODIFY_QP(struct mthca_dev *dev, enum ib_qp_state cur, enum ib_qp_state next, u32 num, int is_ee, struct mthca_mailbox *mailbox, u32 optmask, u8 *status) { static const u16 op[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = { [IB_QPS_RESET] = { [IB_QPS_RESET] = CMD_ERR2RST_QPEE, [IB_QPS_ERR] = CMD_2ERR_QPEE, [IB_QPS_INIT] = CMD_RST2INIT_QPEE, }, [IB_QPS_INIT] = { [IB_QPS_RESET] = CMD_ERR2RST_QPEE, [IB_QPS_ERR] = CMD_2ERR_QPEE, [IB_QPS_INIT] = CMD_INIT2INIT_QPEE, [IB_QPS_RTR] = CMD_INIT2RTR_QPEE, }, [IB_QPS_RTR] = { [IB_QPS_RESET] = CMD_ERR2RST_QPEE, [IB_QPS_ERR] = CMD_2ERR_QPEE, [IB_QPS_RTS] = CMD_RTR2RTS_QPEE, }, [IB_QPS_RTS] = { [IB_QPS_RESET] = CMD_ERR2RST_QPEE, [IB_QPS_ERR] = CMD_2ERR_QPEE, [IB_QPS_RTS] = CMD_RTS2RTS_QPEE, [IB_QPS_SQD] = CMD_RTS2SQD_QPEE, }, [IB_QPS_SQD] = { [IB_QPS_RESET] = CMD_ERR2RST_QPEE, [IB_QPS_ERR] = CMD_2ERR_QPEE, [IB_QPS_RTS] = CMD_SQD2RTS_QPEE, [IB_QPS_SQD] = CMD_SQD2SQD_QPEE, }, [IB_QPS_SQE] = { [IB_QPS_RESET] = CMD_ERR2RST_QPEE, [IB_QPS_ERR] = CMD_2ERR_QPEE, [IB_QPS_RTS] = CMD_SQERR2RTS_QPEE, }, [IB_QPS_ERR] = { [IB_QPS_RESET] = CMD_ERR2RST_QPEE, [IB_QPS_ERR] = CMD_2ERR_QPEE, } }; u8 op_mod = 0; int my_mailbox = 0; int err; if (op[cur][next] == CMD_ERR2RST_QPEE) { op_mod = 3; /* don't write outbox, any->reset */ /* For debugging */ if (!mailbox) { mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (!IS_ERR(mailbox)) { my_mailbox = 1; op_mod = 2; /* write outbox, any->reset */ } else mailbox = NULL; } err = mthca_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, (!!is_ee << 24) | num, op_mod, op[cur][next], CMD_TIME_CLASS_C, status); if (0 && mailbox) { int i; mthca_dbg(dev, "Dumping QP context:\n"); printk(" %08x\n", be32_to_cpup(mailbox->buf)); for (i = 0; i < 0x100 / 4; ++i) { if (i % 8 == 0) printk("[%02x] ", i * 4); printk(" %08x", be32_to_cpu(((__be32 *) mailbox->buf)[i + 2])); if ((i + 1) % 8 == 0) printk("\n"); } } if (my_mailbox) mthca_free_mailbox(dev, mailbox); } else { if (0) { int i; mthca_dbg(dev, "Dumping QP context:\n"); printk(" opt param mask: %08x\n", be32_to_cpup(mailbox->buf)); for (i = 0; i < 0x100 / 4; ++i) { if (i % 8 == 0) printk(" [%02x] ", i * 4); printk(" %08x", be32_to_cpu(((__be32 *) mailbox->buf)[i + 2])); if ((i + 1) % 8 == 0) printk("\n"); } } err = mthca_cmd(dev, mailbox->dma, optmask | (!!is_ee << 24) | num, op_mod, op[cur][next], CMD_TIME_CLASS_C, status); } return err; } int mthca_QUERY_QP(struct mthca_dev *dev, u32 num, int is_ee, struct mthca_mailbox *mailbox, u8 *status) { return mthca_cmd_box(dev, 0, mailbox->dma, (!!is_ee << 24) | num, 0, CMD_QUERY_QPEE, CMD_TIME_CLASS_A, status); } int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn, u8 *status) { u8 op_mod; switch (type) { case IB_QPT_SMI: op_mod = 0; break; case IB_QPT_GSI: op_mod = 1; break; case IB_QPT_RAW_IPV6: op_mod = 2; break; - case IB_QPT_RAW_ETY: + case IB_QPT_RAW_ETHERTYPE: op_mod = 3; break; default: return -EINVAL; } return mthca_cmd(dev, 0, qpn, op_mod, CMD_CONF_SPECIAL_QP, CMD_TIME_CLASS_B, status); } int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey, int port, struct ib_wc *in_wc, struct ib_grh *in_grh, void *in_mad, void *response_mad, u8 *status) { struct mthca_mailbox *inmailbox, *outmailbox; void *inbox; int err; u32 in_modifier = port; u8 op_modifier = 0; #define MAD_IFC_BOX_SIZE 0x400 #define MAD_IFC_MY_QPN_OFFSET 0x100 #define MAD_IFC_RQPN_OFFSET 0x108 #define MAD_IFC_SL_OFFSET 0x10c #define MAD_IFC_G_PATH_OFFSET 0x10d #define MAD_IFC_RLID_OFFSET 0x10e #define MAD_IFC_PKEY_OFFSET 0x112 #define MAD_IFC_GRH_OFFSET 0x140 inmailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(inmailbox)) return PTR_ERR(inmailbox); inbox = inmailbox->buf; outmailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(outmailbox)) { mthca_free_mailbox(dev, inmailbox); return PTR_ERR(outmailbox); } memcpy(inbox, in_mad, 256); /* * Key check traps can't be generated unless we have in_wc to * tell us where to send the trap. */ if (ignore_mkey || !in_wc) op_modifier |= 0x1; if (ignore_bkey || !in_wc) op_modifier |= 0x2; if (in_wc) { u8 val; memset(inbox + 256, 0, 256); MTHCA_PUT(inbox, in_wc->qp->qp_num, MAD_IFC_MY_QPN_OFFSET); MTHCA_PUT(inbox, in_wc->src_qp, MAD_IFC_RQPN_OFFSET); val = in_wc->sl << 4; MTHCA_PUT(inbox, val, MAD_IFC_SL_OFFSET); val = in_wc->dlid_path_bits | (in_wc->wc_flags & IB_WC_GRH ? 0x80 : 0); MTHCA_PUT(inbox, val, MAD_IFC_G_PATH_OFFSET); MTHCA_PUT(inbox, in_wc->slid, MAD_IFC_RLID_OFFSET); MTHCA_PUT(inbox, in_wc->pkey_index, MAD_IFC_PKEY_OFFSET); if (in_grh) memcpy(inbox + MAD_IFC_GRH_OFFSET, in_grh, 40); op_modifier |= 0x4; in_modifier |= in_wc->slid << 16; } err = mthca_cmd_box(dev, inmailbox->dma, outmailbox->dma, in_modifier, op_modifier, CMD_MAD_IFC, CMD_TIME_CLASS_C, status); if (!err && !*status) memcpy(response_mad, outmailbox->buf, 256); mthca_free_mailbox(dev, inmailbox); mthca_free_mailbox(dev, outmailbox); return err; } int mthca_READ_MGM(struct mthca_dev *dev, int index, struct mthca_mailbox *mailbox, u8 *status) { return mthca_cmd_box(dev, 0, mailbox->dma, index, 0, CMD_READ_MGM, CMD_TIME_CLASS_A, status); } int mthca_WRITE_MGM(struct mthca_dev *dev, int index, struct mthca_mailbox *mailbox, u8 *status) { return mthca_cmd(dev, mailbox->dma, index, 0, CMD_WRITE_MGM, CMD_TIME_CLASS_A, status); } int mthca_MGID_HASH(struct mthca_dev *dev, struct mthca_mailbox *mailbox, u16 *hash, u8 *status) { u64 imm; int err; err = mthca_cmd_imm(dev, mailbox->dma, &imm, 0, 0, CMD_MGID_HASH, CMD_TIME_CLASS_A, status); *hash = imm; return err; } int mthca_NOP(struct mthca_dev *dev, u8 *status) { return mthca_cmd(dev, 0, 0x1f, 0, CMD_NOP, msecs_to_jiffies(100), status); } Index: stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c (revision 273864) @@ -1,1360 +1,1359 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2005 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include -#include #include #include #include #include "mthca_dev.h" #include "mthca_config_reg.h" #include "mthca_cmd.h" #include "mthca_profile.h" #include "mthca_memfree.h" #include "mthca_wqe.h" MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("Mellanox InfiniBand HCA low-level driver"); MODULE_LICENSE("Dual BSD/GPL"); MODULE_VERSION(DRV_VERSION); #ifdef CONFIG_INFINIBAND_MTHCA_DEBUG int mthca_debug_level = 0; module_param_named(debug_level, mthca_debug_level, int, 0644); MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0"); #endif /* CONFIG_INFINIBAND_MTHCA_DEBUG */ #ifdef CONFIG_PCI_MSI static int msi_x = 1; module_param(msi_x, int, 0444); MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero"); #else /* CONFIG_PCI_MSI */ #define msi_x (0) #endif /* CONFIG_PCI_MSI */ static int tune_pci = 0; module_param(tune_pci, int, 0444); MODULE_PARM_DESC(tune_pci, "increase PCI burst from the default set by BIOS if nonzero"); DEFINE_MUTEX(mthca_device_mutex); #define MTHCA_DEFAULT_NUM_QP (1 << 16) #define MTHCA_DEFAULT_RDB_PER_QP (1 << 2) #define MTHCA_DEFAULT_NUM_CQ (1 << 16) #define MTHCA_DEFAULT_NUM_MCG (1 << 13) #define MTHCA_DEFAULT_NUM_MPT (1 << 17) #define MTHCA_DEFAULT_NUM_MTT (1 << 20) #define MTHCA_DEFAULT_NUM_UDAV (1 << 15) #define MTHCA_DEFAULT_NUM_RESERVED_MTTS (1 << 18) #define MTHCA_DEFAULT_NUM_UARC_SIZE (1 << 18) static struct mthca_profile hca_profile = { .num_qp = MTHCA_DEFAULT_NUM_QP, .rdb_per_qp = MTHCA_DEFAULT_RDB_PER_QP, .num_cq = MTHCA_DEFAULT_NUM_CQ, .num_mcg = MTHCA_DEFAULT_NUM_MCG, .num_mpt = MTHCA_DEFAULT_NUM_MPT, .num_mtt = MTHCA_DEFAULT_NUM_MTT, .num_udav = MTHCA_DEFAULT_NUM_UDAV, /* Tavor only */ .fmr_reserved_mtts = MTHCA_DEFAULT_NUM_RESERVED_MTTS, /* Tavor only */ .uarc_size = MTHCA_DEFAULT_NUM_UARC_SIZE, /* Arbel only */ }; module_param_named(num_qp, hca_profile.num_qp, int, 0444); MODULE_PARM_DESC(num_qp, "maximum number of QPs per HCA"); module_param_named(rdb_per_qp, hca_profile.rdb_per_qp, int, 0444); MODULE_PARM_DESC(rdb_per_qp, "number of RDB buffers per QP"); module_param_named(num_cq, hca_profile.num_cq, int, 0444); MODULE_PARM_DESC(num_cq, "maximum number of CQs per HCA"); module_param_named(num_mcg, hca_profile.num_mcg, int, 0444); MODULE_PARM_DESC(num_mcg, "maximum number of multicast groups per HCA"); module_param_named(num_mpt, hca_profile.num_mpt, int, 0444); MODULE_PARM_DESC(num_mpt, "maximum number of memory protection table entries per HCA"); module_param_named(num_mtt, hca_profile.num_mtt, int, 0444); MODULE_PARM_DESC(num_mtt, "maximum number of memory translation table segments per HCA"); module_param_named(num_udav, hca_profile.num_udav, int, 0444); MODULE_PARM_DESC(num_udav, "maximum number of UD address vectors per HCA"); module_param_named(fmr_reserved_mtts, hca_profile.fmr_reserved_mtts, int, 0444); MODULE_PARM_DESC(fmr_reserved_mtts, "number of memory translation table segments reserved for FMR"); static int log_mtts_per_seg; module_param_named(log_mtts_per_seg, log_mtts_per_seg, int, 0444); MODULE_PARM_DESC(log_mtts_per_seg, "Log2 number of MTT entries per segment (1-5)"); static char mthca_version[] __devinitdata = DRV_NAME ": Mellanox InfiniBand HCA driver v" DRV_VERSION " (" DRV_RELDATE ")\n"; static int mthca_tune_pci(struct mthca_dev *mdev) { if (!tune_pci) return 0; /* First try to max out Read Byte Count */ if (pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX)) { if (pcix_set_mmrbc(mdev->pdev, pcix_get_max_mmrbc(mdev->pdev))) { mthca_err(mdev, "Couldn't set PCI-X max read count, " "aborting.\n"); return -ENODEV; } } else if (!(mdev->mthca_flags & MTHCA_FLAG_PCIE)) mthca_info(mdev, "No PCI-X capability, not setting RBC.\n"); if (pci_find_capability(mdev->pdev, PCI_CAP_ID_EXP)) { if (pcie_set_readrq(mdev->pdev, 4096)) { mthca_err(mdev, "Couldn't write PCI Express read request, " "aborting.\n"); return -ENODEV; } } else if (mdev->mthca_flags & MTHCA_FLAG_PCIE) mthca_info(mdev, "No PCI Express capability, " "not setting Max Read Request Size.\n"); return 0; } static int mthca_dev_lim(struct mthca_dev *mdev, struct mthca_dev_lim *dev_lim) { int err; u8 status; mdev->limits.mtt_seg_size = (1 << log_mtts_per_seg) * 8; err = mthca_QUERY_DEV_LIM(mdev, dev_lim, &status); if (err) { mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n"); return err; } if (status) { mthca_err(mdev, "QUERY_DEV_LIM returned status 0x%02x, " "aborting.\n", status); return -EINVAL; } if (dev_lim->min_page_sz > PAGE_SIZE) { mthca_err(mdev, "HCA minimum page size of %d bigger than " "kernel PAGE_SIZE of %d, aborting.\n", dev_lim->min_page_sz, PAGE_SIZE); return -ENODEV; } if (dev_lim->num_ports > MTHCA_MAX_PORTS) { mthca_err(mdev, "HCA has %d ports, but we only support %d, " "aborting.\n", dev_lim->num_ports, MTHCA_MAX_PORTS); return -ENODEV; } if (dev_lim->uar_size > pci_resource_len(mdev->pdev, 2)) { mthca_err(mdev, "HCA reported UAR size of 0x%x bigger than " "PCI resource 2 size of 0x%llx, aborting.\n", dev_lim->uar_size, (unsigned long long)pci_resource_len(mdev->pdev, 2)); return -ENODEV; } mdev->limits.num_ports = dev_lim->num_ports; mdev->limits.vl_cap = dev_lim->max_vl; mdev->limits.mtu_cap = dev_lim->max_mtu; mdev->limits.gid_table_len = dev_lim->max_gids; mdev->limits.pkey_table_len = dev_lim->max_pkeys; mdev->limits.local_ca_ack_delay = dev_lim->local_ca_ack_delay; /* * Need to allow for worst case send WQE overhead and check * whether max_desc_sz imposes a lower limit than max_sg; UD * send has the biggest overhead. */ mdev->limits.max_sg = min_t(int, dev_lim->max_sg, (dev_lim->max_desc_sz - sizeof (struct mthca_next_seg) - (mthca_is_memfree(mdev) ? sizeof (struct mthca_arbel_ud_seg) : sizeof (struct mthca_tavor_ud_seg))) / sizeof (struct mthca_data_seg)); mdev->limits.max_wqes = dev_lim->max_qp_sz; mdev->limits.max_qp_init_rdma = dev_lim->max_requester_per_qp; mdev->limits.reserved_qps = dev_lim->reserved_qps; mdev->limits.max_srq_wqes = dev_lim->max_srq_sz; mdev->limits.reserved_srqs = dev_lim->reserved_srqs; mdev->limits.reserved_eecs = dev_lim->reserved_eecs; mdev->limits.max_desc_sz = dev_lim->max_desc_sz; mdev->limits.max_srq_sge = mthca_max_srq_sge(mdev); /* * Subtract 1 from the limit because we need to allocate a * spare CQE so the HCA HW can tell the difference between an * empty CQ and a full CQ. */ mdev->limits.max_cqes = dev_lim->max_cq_sz - 1; mdev->limits.reserved_cqs = dev_lim->reserved_cqs; mdev->limits.reserved_eqs = dev_lim->reserved_eqs; mdev->limits.reserved_mtts = dev_lim->reserved_mtts; mdev->limits.reserved_mrws = dev_lim->reserved_mrws; mdev->limits.reserved_uars = dev_lim->reserved_uars; mdev->limits.reserved_pds = dev_lim->reserved_pds; mdev->limits.port_width_cap = dev_lim->max_port_width; mdev->limits.page_size_cap = ~(u32) (dev_lim->min_page_sz - 1); mdev->limits.flags = dev_lim->flags; /* * For old FW that doesn't return static rate support, use a * value of 0x3 (only static rate values of 0 or 1 are handled), * except on Sinai, where even old FW can handle static rate * values of 2 and 3. */ if (dev_lim->stat_rate_support) mdev->limits.stat_rate_support = dev_lim->stat_rate_support; else if (mdev->mthca_flags & MTHCA_FLAG_SINAI_OPT) mdev->limits.stat_rate_support = 0xf; else mdev->limits.stat_rate_support = 0x3; /* IB_DEVICE_RESIZE_MAX_WR not supported by driver. May be doable since hardware supports it for SRQ. IB_DEVICE_N_NOTIFY_CQ is supported by hardware but not by driver. IB_DEVICE_SRQ_RESIZE is supported by hardware but SRQ is not supported by driver. */ mdev->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT | IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_SYS_IMAGE_GUID | IB_DEVICE_RC_RNR_NAK_GEN; if (dev_lim->flags & DEV_LIM_FLAG_BAD_PKEY_CNTR) mdev->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR; if (dev_lim->flags & DEV_LIM_FLAG_BAD_QKEY_CNTR) mdev->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR; if (dev_lim->flags & DEV_LIM_FLAG_RAW_MULTI) mdev->device_cap_flags |= IB_DEVICE_RAW_MULTI; if (dev_lim->flags & DEV_LIM_FLAG_AUTO_PATH_MIG) mdev->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG; if (dev_lim->flags & DEV_LIM_FLAG_UD_AV_PORT_ENFORCE) mdev->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE; if (dev_lim->flags & DEV_LIM_FLAG_SRQ) mdev->mthca_flags |= MTHCA_FLAG_SRQ; if (mthca_is_memfree(mdev)) if (dev_lim->flags & DEV_LIM_FLAG_IPOIB_CSUM) mdev->device_cap_flags |= IB_DEVICE_UD_IP_CSUM; return 0; } static int mthca_init_tavor(struct mthca_dev *mdev) { s64 size; u8 status; int err; struct mthca_dev_lim dev_lim; struct mthca_profile profile; struct mthca_init_hca_param init_hca; err = mthca_SYS_EN(mdev, &status); if (err) { mthca_err(mdev, "SYS_EN command failed, aborting.\n"); return err; } if (status) { mthca_err(mdev, "SYS_EN returned status 0x%02x, " "aborting.\n", status); return -EINVAL; } err = mthca_QUERY_FW(mdev, &status); if (err) { mthca_err(mdev, "QUERY_FW command failed, aborting.\n"); goto err_disable; } if (status) { mthca_err(mdev, "QUERY_FW returned status 0x%02x, " "aborting.\n", status); err = -EINVAL; goto err_disable; } err = mthca_QUERY_DDR(mdev, &status); if (err) { mthca_err(mdev, "QUERY_DDR command failed, aborting.\n"); goto err_disable; } if (status) { mthca_err(mdev, "QUERY_DDR returned status 0x%02x, " "aborting.\n", status); err = -EINVAL; goto err_disable; } err = mthca_dev_lim(mdev, &dev_lim); if (err) { mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n"); goto err_disable; } profile = hca_profile; profile.num_uar = dev_lim.uar_size / PAGE_SIZE; profile.uarc_size = 0; if (mdev->mthca_flags & MTHCA_FLAG_SRQ) profile.num_srq = dev_lim.max_srqs; size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca); if (size < 0) { err = size; goto err_disable; } err = mthca_INIT_HCA(mdev, &init_hca, &status); if (err) { mthca_err(mdev, "INIT_HCA command failed, aborting.\n"); goto err_disable; } if (status) { mthca_err(mdev, "INIT_HCA returned status 0x%02x, " "aborting.\n", status); err = -EINVAL; goto err_disable; } return 0; err_disable: mthca_SYS_DIS(mdev, &status); return err; } static int mthca_load_fw(struct mthca_dev *mdev) { u8 status; int err; /* FIXME: use HCA-attached memory for FW if present */ mdev->fw.arbel.fw_icm = mthca_alloc_icm(mdev, mdev->fw.arbel.fw_pages, GFP_HIGHUSER | __GFP_NOWARN, 0); if (!mdev->fw.arbel.fw_icm) { mthca_err(mdev, "Couldn't allocate FW area, aborting.\n"); return -ENOMEM; } err = mthca_MAP_FA(mdev, mdev->fw.arbel.fw_icm, &status); if (err) { mthca_err(mdev, "MAP_FA command failed, aborting.\n"); goto err_free; } if (status) { mthca_err(mdev, "MAP_FA returned status 0x%02x, aborting.\n", status); err = -EINVAL; goto err_free; } err = mthca_RUN_FW(mdev, &status); if (err) { mthca_err(mdev, "RUN_FW command failed, aborting.\n"); goto err_unmap_fa; } if (status) { mthca_err(mdev, "RUN_FW returned status 0x%02x, aborting.\n", status); err = -EINVAL; goto err_unmap_fa; } return 0; err_unmap_fa: mthca_UNMAP_FA(mdev, &status); err_free: mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0); return err; } static int mthca_init_icm(struct mthca_dev *mdev, struct mthca_dev_lim *dev_lim, struct mthca_init_hca_param *init_hca, u64 icm_size) { u64 aux_pages; u8 status; int err; err = mthca_SET_ICM_SIZE(mdev, icm_size, &aux_pages, &status); if (err) { mthca_err(mdev, "SET_ICM_SIZE command failed, aborting.\n"); return err; } if (status) { mthca_err(mdev, "SET_ICM_SIZE returned status 0x%02x, " "aborting.\n", status); return -EINVAL; } mthca_dbg(mdev, "%lld KB of HCA context requires %lld KB aux memory.\n", (unsigned long long) icm_size >> 10, (unsigned long long) aux_pages << 2); mdev->fw.arbel.aux_icm = mthca_alloc_icm(mdev, aux_pages, GFP_HIGHUSER | __GFP_NOWARN, 0); if (!mdev->fw.arbel.aux_icm) { mthca_err(mdev, "Couldn't allocate aux memory, aborting.\n"); return -ENOMEM; } err = mthca_MAP_ICM_AUX(mdev, mdev->fw.arbel.aux_icm, &status); if (err) { mthca_err(mdev, "MAP_ICM_AUX command failed, aborting.\n"); goto err_free_aux; } if (status) { mthca_err(mdev, "MAP_ICM_AUX returned status 0x%02x, aborting.\n", status); err = -EINVAL; goto err_free_aux; } err = mthca_map_eq_icm(mdev, init_hca->eqc_base); if (err) { mthca_err(mdev, "Failed to map EQ context memory, aborting.\n"); goto err_unmap_aux; } /* CPU writes to non-reserved MTTs, while HCA might DMA to reserved mtts */ mdev->limits.reserved_mtts = ALIGN(mdev->limits.reserved_mtts * mdev->limits.mtt_seg_size, dma_get_cache_alignment()) / mdev->limits.mtt_seg_size; mdev->mr_table.mtt_table = mthca_alloc_icm_table(mdev, init_hca->mtt_base, mdev->limits.mtt_seg_size, mdev->limits.num_mtt_segs, mdev->limits.reserved_mtts, 1, 0); if (!mdev->mr_table.mtt_table) { mthca_err(mdev, "Failed to map MTT context memory, aborting.\n"); err = -ENOMEM; goto err_unmap_eq; } mdev->mr_table.mpt_table = mthca_alloc_icm_table(mdev, init_hca->mpt_base, dev_lim->mpt_entry_sz, mdev->limits.num_mpts, mdev->limits.reserved_mrws, 1, 1); if (!mdev->mr_table.mpt_table) { mthca_err(mdev, "Failed to map MPT context memory, aborting.\n"); err = -ENOMEM; goto err_unmap_mtt; } mdev->qp_table.qp_table = mthca_alloc_icm_table(mdev, init_hca->qpc_base, dev_lim->qpc_entry_sz, mdev->limits.num_qps, mdev->limits.reserved_qps, 0, 0); if (!mdev->qp_table.qp_table) { mthca_err(mdev, "Failed to map QP context memory, aborting.\n"); err = -ENOMEM; goto err_unmap_mpt; } mdev->qp_table.eqp_table = mthca_alloc_icm_table(mdev, init_hca->eqpc_base, dev_lim->eqpc_entry_sz, mdev->limits.num_qps, mdev->limits.reserved_qps, 0, 0); if (!mdev->qp_table.eqp_table) { mthca_err(mdev, "Failed to map EQP context memory, aborting.\n"); err = -ENOMEM; goto err_unmap_qp; } mdev->qp_table.rdb_table = mthca_alloc_icm_table(mdev, init_hca->rdb_base, MTHCA_RDB_ENTRY_SIZE, mdev->limits.num_qps << mdev->qp_table.rdb_shift, 0, 0, 0); if (!mdev->qp_table.rdb_table) { mthca_err(mdev, "Failed to map RDB context memory, aborting\n"); err = -ENOMEM; goto err_unmap_eqp; } mdev->cq_table.table = mthca_alloc_icm_table(mdev, init_hca->cqc_base, dev_lim->cqc_entry_sz, mdev->limits.num_cqs, mdev->limits.reserved_cqs, 0, 0); if (!mdev->cq_table.table) { mthca_err(mdev, "Failed to map CQ context memory, aborting.\n"); err = -ENOMEM; goto err_unmap_rdb; } if (mdev->mthca_flags & MTHCA_FLAG_SRQ) { mdev->srq_table.table = mthca_alloc_icm_table(mdev, init_hca->srqc_base, dev_lim->srq_entry_sz, mdev->limits.num_srqs, mdev->limits.reserved_srqs, 0, 0); if (!mdev->srq_table.table) { mthca_err(mdev, "Failed to map SRQ context memory, " "aborting.\n"); err = -ENOMEM; goto err_unmap_cq; } } /* * It's not strictly required, but for simplicity just map the * whole multicast group table now. The table isn't very big * and it's a lot easier than trying to track ref counts. */ mdev->mcg_table.table = mthca_alloc_icm_table(mdev, init_hca->mc_base, MTHCA_MGM_ENTRY_SIZE, mdev->limits.num_mgms + mdev->limits.num_amgms, mdev->limits.num_mgms + mdev->limits.num_amgms, 0, 0); if (!mdev->mcg_table.table) { mthca_err(mdev, "Failed to map MCG context memory, aborting.\n"); err = -ENOMEM; goto err_unmap_srq; } return 0; err_unmap_srq: if (mdev->mthca_flags & MTHCA_FLAG_SRQ) mthca_free_icm_table(mdev, mdev->srq_table.table); err_unmap_cq: mthca_free_icm_table(mdev, mdev->cq_table.table); err_unmap_rdb: mthca_free_icm_table(mdev, mdev->qp_table.rdb_table); err_unmap_eqp: mthca_free_icm_table(mdev, mdev->qp_table.eqp_table); err_unmap_qp: mthca_free_icm_table(mdev, mdev->qp_table.qp_table); err_unmap_mpt: mthca_free_icm_table(mdev, mdev->mr_table.mpt_table); err_unmap_mtt: mthca_free_icm_table(mdev, mdev->mr_table.mtt_table); err_unmap_eq: mthca_unmap_eq_icm(mdev); err_unmap_aux: mthca_UNMAP_ICM_AUX(mdev, &status); err_free_aux: mthca_free_icm(mdev, mdev->fw.arbel.aux_icm, 0); return err; } static void mthca_free_icms(struct mthca_dev *mdev) { u8 status; mthca_free_icm_table(mdev, mdev->mcg_table.table); if (mdev->mthca_flags & MTHCA_FLAG_SRQ) mthca_free_icm_table(mdev, mdev->srq_table.table); mthca_free_icm_table(mdev, mdev->cq_table.table); mthca_free_icm_table(mdev, mdev->qp_table.rdb_table); mthca_free_icm_table(mdev, mdev->qp_table.eqp_table); mthca_free_icm_table(mdev, mdev->qp_table.qp_table); mthca_free_icm_table(mdev, mdev->mr_table.mpt_table); mthca_free_icm_table(mdev, mdev->mr_table.mtt_table); mthca_unmap_eq_icm(mdev); mthca_UNMAP_ICM_AUX(mdev, &status); mthca_free_icm(mdev, mdev->fw.arbel.aux_icm, 0); } static int mthca_init_arbel(struct mthca_dev *mdev) { struct mthca_dev_lim dev_lim; struct mthca_profile profile; struct mthca_init_hca_param init_hca; s64 icm_size; u8 status; int err; err = mthca_QUERY_FW(mdev, &status); if (err) { mthca_err(mdev, "QUERY_FW command failed, aborting.\n"); return err; } if (status) { mthca_err(mdev, "QUERY_FW returned status 0x%02x, " "aborting.\n", status); return -EINVAL; } err = mthca_ENABLE_LAM(mdev, &status); if (err) { mthca_err(mdev, "ENABLE_LAM command failed, aborting.\n"); return err; } if (status == MTHCA_CMD_STAT_LAM_NOT_PRE) { mthca_dbg(mdev, "No HCA-attached memory (running in MemFree mode)\n"); mdev->mthca_flags |= MTHCA_FLAG_NO_LAM; } else if (status) { mthca_err(mdev, "ENABLE_LAM returned status 0x%02x, " "aborting.\n", status); return -EINVAL; } err = mthca_load_fw(mdev); if (err) { mthca_err(mdev, "Failed to start FW, aborting.\n"); goto err_disable; } err = mthca_dev_lim(mdev, &dev_lim); if (err) { mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n"); goto err_stop_fw; } profile = hca_profile; profile.num_uar = dev_lim.uar_size / PAGE_SIZE; profile.num_udav = 0; if (mdev->mthca_flags & MTHCA_FLAG_SRQ) profile.num_srq = dev_lim.max_srqs; icm_size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca); if (icm_size < 0) { err = icm_size; goto err_stop_fw; } err = mthca_init_icm(mdev, &dev_lim, &init_hca, icm_size); if (err) goto err_stop_fw; err = mthca_INIT_HCA(mdev, &init_hca, &status); if (err) { mthca_err(mdev, "INIT_HCA command failed, aborting.\n"); goto err_free_icm; } if (status) { mthca_err(mdev, "INIT_HCA returned status 0x%02x, " "aborting.\n", status); err = -EINVAL; goto err_free_icm; } return 0; err_free_icm: mthca_free_icms(mdev); err_stop_fw: mthca_UNMAP_FA(mdev, &status); mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0); err_disable: if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM)) mthca_DISABLE_LAM(mdev, &status); return err; } static void mthca_close_hca(struct mthca_dev *mdev) { u8 status; mthca_CLOSE_HCA(mdev, 0, &status); if (mthca_is_memfree(mdev)) { mthca_free_icms(mdev); mthca_UNMAP_FA(mdev, &status); mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0); if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM)) mthca_DISABLE_LAM(mdev, &status); } else mthca_SYS_DIS(mdev, &status); } static int mthca_init_hca(struct mthca_dev *mdev) { u8 status; int err; struct mthca_adapter adapter; if (mthca_is_memfree(mdev)) err = mthca_init_arbel(mdev); else err = mthca_init_tavor(mdev); if (err) return err; err = mthca_QUERY_ADAPTER(mdev, &adapter, &status); if (err) { mthca_err(mdev, "QUERY_ADAPTER command failed, aborting.\n"); goto err_close; } if (status) { mthca_err(mdev, "QUERY_ADAPTER returned status 0x%02x, " "aborting.\n", status); err = -EINVAL; goto err_close; } mdev->eq_table.inta_pin = adapter.inta_pin; if (!mthca_is_memfree(mdev)) mdev->rev_id = adapter.revision_id; memcpy(mdev->board_id, adapter.board_id, sizeof mdev->board_id); return 0; err_close: mthca_close_hca(mdev); return err; } static int mthca_setup_hca(struct mthca_dev *dev) { int err; u8 status; MTHCA_INIT_DOORBELL_LOCK(&dev->doorbell_lock); err = mthca_init_uar_table(dev); if (err) { mthca_err(dev, "Failed to initialize " "user access region table, aborting.\n"); return err; } err = mthca_uar_alloc(dev, &dev->driver_uar); if (err) { mthca_err(dev, "Failed to allocate driver access region, " "aborting.\n"); goto err_uar_table_free; } dev->kar = ioremap(dev->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE); if (!dev->kar) { mthca_err(dev, "Couldn't map kernel access region, " "aborting.\n"); err = -ENOMEM; goto err_uar_free; } err = mthca_init_pd_table(dev); if (err) { mthca_err(dev, "Failed to initialize " "protection domain table, aborting.\n"); goto err_kar_unmap; } err = mthca_init_mr_table(dev); if (err) { mthca_err(dev, "Failed to initialize " "memory region table, aborting.\n"); goto err_pd_table_free; } err = mthca_pd_alloc(dev, 1, &dev->driver_pd); if (err) { mthca_err(dev, "Failed to create driver PD, " "aborting.\n"); goto err_mr_table_free; } err = mthca_init_eq_table(dev); if (err) { mthca_err(dev, "Failed to initialize " "event queue table, aborting.\n"); goto err_pd_free; } err = mthca_cmd_use_events(dev); if (err) { mthca_err(dev, "Failed to switch to event-driven " "firmware commands, aborting.\n"); goto err_eq_table_free; } err = mthca_NOP(dev, &status); if (err || status) { if (dev->mthca_flags & MTHCA_FLAG_MSI_X) { mthca_warn(dev, "NOP command failed to generate interrupt " "(IRQ %d).\n", dev->eq_table.eq[MTHCA_EQ_CMD].msi_x_vector); mthca_warn(dev, "Trying again with MSI-X disabled.\n"); } else { mthca_err(dev, "NOP command failed to generate interrupt " "(IRQ %d), aborting.\n", dev->pdev->irq); mthca_err(dev, "BIOS or ACPI interrupt routing problem?\n"); } goto err_cmd_poll; } mthca_dbg(dev, "NOP command IRQ test passed\n"); err = mthca_init_cq_table(dev); if (err) { mthca_err(dev, "Failed to initialize " "completion queue table, aborting.\n"); goto err_cmd_poll; } err = mthca_init_srq_table(dev); if (err) { mthca_err(dev, "Failed to initialize " "shared receive queue table, aborting.\n"); goto err_cq_table_free; } err = mthca_init_qp_table(dev); if (err) { mthca_err(dev, "Failed to initialize " "queue pair table, aborting.\n"); goto err_srq_table_free; } err = mthca_init_av_table(dev); if (err) { mthca_err(dev, "Failed to initialize " "address vector table, aborting.\n"); goto err_qp_table_free; } err = mthca_init_mcg_table(dev); if (err) { mthca_err(dev, "Failed to initialize " "multicast group table, aborting.\n"); goto err_av_table_free; } return 0; err_av_table_free: mthca_cleanup_av_table(dev); err_qp_table_free: mthca_cleanup_qp_table(dev); err_srq_table_free: mthca_cleanup_srq_table(dev); err_cq_table_free: mthca_cleanup_cq_table(dev); err_cmd_poll: mthca_cmd_use_polling(dev); err_eq_table_free: mthca_cleanup_eq_table(dev); err_pd_free: mthca_pd_free(dev, &dev->driver_pd); err_mr_table_free: mthca_cleanup_mr_table(dev); err_pd_table_free: mthca_cleanup_pd_table(dev); err_kar_unmap: iounmap(dev->kar); err_uar_free: mthca_uar_free(dev, &dev->driver_uar); err_uar_table_free: mthca_cleanup_uar_table(dev); return err; } static int mthca_enable_msi_x(struct mthca_dev *mdev) { struct msix_entry entries[3]; int err; entries[0].entry = 0; entries[1].entry = 1; entries[2].entry = 2; err = pci_enable_msix(mdev->pdev, entries, ARRAY_SIZE(entries)); if (err) { if (err > 0) mthca_info(mdev, "Only %d MSI-X vectors available, " "not using MSI-X\n", err); return err; } mdev->eq_table.eq[MTHCA_EQ_COMP ].msi_x_vector = entries[0].vector; mdev->eq_table.eq[MTHCA_EQ_ASYNC].msi_x_vector = entries[1].vector; mdev->eq_table.eq[MTHCA_EQ_CMD ].msi_x_vector = entries[2].vector; return 0; } /* Types of supported HCA */ enum { TAVOR, /* MT23108 */ ARBEL_COMPAT, /* MT25208 in Tavor compat mode */ ARBEL_NATIVE, /* MT25208 with extended features */ SINAI /* MT25204 */ }; #define MTHCA_FW_VER(major, minor, subminor) \ (((u64) (major) << 32) | ((u64) (minor) << 16) | (u64) (subminor)) static struct { u64 latest_fw; u32 flags; } mthca_hca_table[] = { [TAVOR] = { .latest_fw = MTHCA_FW_VER(3, 5, 0), .flags = 0 }, [ARBEL_COMPAT] = { .latest_fw = MTHCA_FW_VER(4, 8, 200), .flags = MTHCA_FLAG_PCIE }, [ARBEL_NATIVE] = { .latest_fw = MTHCA_FW_VER(5, 3, 0), .flags = MTHCA_FLAG_MEMFREE | MTHCA_FLAG_PCIE }, [SINAI] = { .latest_fw = MTHCA_FW_VER(1, 2, 0), .flags = MTHCA_FLAG_MEMFREE | MTHCA_FLAG_PCIE | MTHCA_FLAG_SINAI_OPT } }; static int __mthca_init_one(struct pci_dev *pdev, int hca_type) { int ddr_hidden = 0; int err; struct mthca_dev *mdev; printk(KERN_INFO PFX "Initializing %s\n", pci_name(pdev)); err = pci_enable_device(pdev); if (err) { dev_err(&pdev->dev, "Cannot enable PCI device, " "aborting.\n"); return err; } /* * Check for BARs. We expect 0: 1MB, 2: 8MB, 4: DDR (may not * be present) */ if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) || pci_resource_len(pdev, 0) != 1 << 20) { dev_err(&pdev->dev, "Missing DCS, aborting.\n"); err = -ENODEV; goto err_disable_pdev; } if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) { dev_err(&pdev->dev, "Missing UAR, aborting.\n"); err = -ENODEV; goto err_disable_pdev; } if (!(pci_resource_flags(pdev, 4) & IORESOURCE_MEM)) ddr_hidden = 1; err = pci_request_regions(pdev, DRV_NAME); if (err) { dev_err(&pdev->dev, "Cannot obtain PCI resources, " "aborting.\n"); goto err_disable_pdev; } pci_set_master(pdev); err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); if (err) { dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n"); err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); if (err) { dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n"); goto err_free_res; } } err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); if (err) { dev_warn(&pdev->dev, "Warning: couldn't set 64-bit " "consistent PCI DMA mask.\n"); err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); if (err) { dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, " "aborting.\n"); goto err_free_res; } } mdev = (struct mthca_dev *) ib_alloc_device(sizeof *mdev); if (!mdev) { dev_err(&pdev->dev, "Device struct alloc failed, " "aborting.\n"); err = -ENOMEM; goto err_free_res; } mdev->pdev = pdev; mdev->mthca_flags = mthca_hca_table[hca_type].flags; if (ddr_hidden) mdev->mthca_flags |= MTHCA_FLAG_DDR_HIDDEN; /* * Now reset the HCA before we touch the PCI capabilities or * attempt a firmware command, since a boot ROM may have left * the HCA in an undefined state. */ err = mthca_reset(mdev); if (err) { mthca_err(mdev, "Failed to reset HCA, aborting.\n"); goto err_free_dev; } if (mthca_cmd_init(mdev)) { mthca_err(mdev, "Failed to init command interface, aborting.\n"); goto err_free_dev; } err = mthca_tune_pci(mdev); if (err) goto err_cmd; err = mthca_init_hca(mdev); if (err) goto err_cmd; if (mdev->fw_ver < mthca_hca_table[hca_type].latest_fw) { mthca_warn(mdev, "HCA FW version %d.%d.%03d is old (%d.%d.%03d is current).\n", (int) (mdev->fw_ver >> 32), (int) (mdev->fw_ver >> 16) & 0xffff, (int) (mdev->fw_ver & 0xffff), (int) (mthca_hca_table[hca_type].latest_fw >> 32), (int) (mthca_hca_table[hca_type].latest_fw >> 16) & 0xffff, (int) (mthca_hca_table[hca_type].latest_fw & 0xffff)); mthca_warn(mdev, "If you have problems, try updating your HCA FW.\n"); } if (msi_x && !mthca_enable_msi_x(mdev)) mdev->mthca_flags |= MTHCA_FLAG_MSI_X; err = mthca_setup_hca(mdev); if (err == -EBUSY && (mdev->mthca_flags & MTHCA_FLAG_MSI_X)) { if (mdev->mthca_flags & MTHCA_FLAG_MSI_X) pci_disable_msix(pdev); mdev->mthca_flags &= ~MTHCA_FLAG_MSI_X; err = mthca_setup_hca(mdev); } if (err) goto err_close; err = mthca_register_device(mdev); if (err) goto err_cleanup; err = mthca_create_agents(mdev); if (err) goto err_unregister; pci_set_drvdata(pdev, mdev); mdev->hca_type = hca_type; mdev->active = 1; return 0; err_unregister: mthca_unregister_device(mdev); err_cleanup: mthca_cleanup_mcg_table(mdev); mthca_cleanup_av_table(mdev); mthca_cleanup_qp_table(mdev); mthca_cleanup_srq_table(mdev); mthca_cleanup_cq_table(mdev); mthca_cmd_use_polling(mdev); mthca_cleanup_eq_table(mdev); mthca_pd_free(mdev, &mdev->driver_pd); mthca_cleanup_mr_table(mdev); mthca_cleanup_pd_table(mdev); mthca_cleanup_uar_table(mdev); err_close: if (mdev->mthca_flags & MTHCA_FLAG_MSI_X) pci_disable_msix(pdev); mthca_close_hca(mdev); err_cmd: mthca_cmd_cleanup(mdev); err_free_dev: ib_dealloc_device(&mdev->ib_dev); err_free_res: pci_release_regions(pdev); err_disable_pdev: pci_disable_device(pdev); pci_set_drvdata(pdev, NULL); return err; } static void __mthca_remove_one(struct pci_dev *pdev) { struct mthca_dev *mdev = pci_get_drvdata(pdev); u8 status; int p; if (mdev) { mthca_free_agents(mdev); mthca_unregister_device(mdev); for (p = 1; p <= mdev->limits.num_ports; ++p) mthca_CLOSE_IB(mdev, p, &status); mthca_cleanup_mcg_table(mdev); mthca_cleanup_av_table(mdev); mthca_cleanup_qp_table(mdev); mthca_cleanup_srq_table(mdev); mthca_cleanup_cq_table(mdev); mthca_cmd_use_polling(mdev); mthca_cleanup_eq_table(mdev); mthca_pd_free(mdev, &mdev->driver_pd); mthca_cleanup_mr_table(mdev); mthca_cleanup_pd_table(mdev); iounmap(mdev->kar); mthca_uar_free(mdev, &mdev->driver_uar); mthca_cleanup_uar_table(mdev); mthca_close_hca(mdev); mthca_cmd_cleanup(mdev); if (mdev->mthca_flags & MTHCA_FLAG_MSI_X) pci_disable_msix(pdev); ib_dealloc_device(&mdev->ib_dev); pci_release_regions(pdev); pci_disable_device(pdev); pci_set_drvdata(pdev, NULL); } } int __mthca_restart_one(struct pci_dev *pdev) { struct mthca_dev *mdev; int hca_type; mdev = pci_get_drvdata(pdev); if (!mdev) return -ENODEV; hca_type = mdev->hca_type; __mthca_remove_one(pdev); return __mthca_init_one(pdev, hca_type); } static int __devinit mthca_init_one(struct pci_dev *pdev, const struct pci_device_id *id) { static int mthca_version_printed = 0; int ret; mutex_lock(&mthca_device_mutex); if (!mthca_version_printed) { printk(KERN_INFO "%s", mthca_version); ++mthca_version_printed; } if (id->driver_data >= ARRAY_SIZE(mthca_hca_table)) { printk(KERN_ERR PFX "%s has invalid driver data %jx\n", pci_name(pdev), (uintmax_t)id->driver_data); mutex_unlock(&mthca_device_mutex); return -ENODEV; } ret = __mthca_init_one(pdev, id->driver_data); mutex_unlock(&mthca_device_mutex); return ret; } static void __devexit mthca_remove_one(struct pci_dev *pdev) { mutex_lock(&mthca_device_mutex); __mthca_remove_one(pdev); mutex_unlock(&mthca_device_mutex); } static struct pci_device_id mthca_pci_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_TAVOR), .driver_data = TAVOR }, { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_TAVOR), .driver_data = TAVOR }, { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT), .driver_data = ARBEL_COMPAT }, { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT), .driver_data = ARBEL_COMPAT }, { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL), .driver_data = ARBEL_NATIVE }, { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL), .driver_data = ARBEL_NATIVE }, { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI), .driver_data = SINAI }, { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI), .driver_data = SINAI }, { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI_OLD), .driver_data = SINAI }, { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI_OLD), .driver_data = SINAI }, { 0, } }; MODULE_DEVICE_TABLE(pci, mthca_pci_table); static struct pci_driver mthca_driver = { .name = DRV_NAME, .id_table = mthca_pci_table, .probe = mthca_init_one, .remove = __devexit_p(mthca_remove_one) }; static void __init __mthca_check_profile_val(const char *name, int *pval, int pval_default) { /* value must be positive and power of 2 */ int old_pval = *pval; if (old_pval <= 0) *pval = pval_default; else *pval = roundup_pow_of_two(old_pval); if (old_pval != *pval) { printk(KERN_WARNING PFX "Invalid value %d for %s in module parameter.\n", old_pval, name); printk(KERN_WARNING PFX "Corrected %s to %d.\n", name, *pval); } } #define mthca_check_profile_val(name, default) \ __mthca_check_profile_val(#name, &hca_profile.name, default) static void __init mthca_validate_profile(void) { mthca_check_profile_val(num_qp, MTHCA_DEFAULT_NUM_QP); mthca_check_profile_val(rdb_per_qp, MTHCA_DEFAULT_RDB_PER_QP); mthca_check_profile_val(num_cq, MTHCA_DEFAULT_NUM_CQ); mthca_check_profile_val(num_mcg, MTHCA_DEFAULT_NUM_MCG); mthca_check_profile_val(num_mpt, MTHCA_DEFAULT_NUM_MPT); mthca_check_profile_val(num_mtt, MTHCA_DEFAULT_NUM_MTT); mthca_check_profile_val(num_udav, MTHCA_DEFAULT_NUM_UDAV); mthca_check_profile_val(fmr_reserved_mtts, MTHCA_DEFAULT_NUM_RESERVED_MTTS); if (hca_profile.fmr_reserved_mtts >= hca_profile.num_mtt) { printk(KERN_WARNING PFX "Invalid fmr_reserved_mtts module parameter %d.\n", hca_profile.fmr_reserved_mtts); printk(KERN_WARNING PFX "(Must be smaller than num_mtt %d)\n", hca_profile.num_mtt); hca_profile.fmr_reserved_mtts = hca_profile.num_mtt / 2; printk(KERN_WARNING PFX "Corrected fmr_reserved_mtts to %d.\n", hca_profile.fmr_reserved_mtts); } if (log_mtts_per_seg == 0) log_mtts_per_seg = ilog2(MTHCA_MTT_SEG_SIZE / 8); if ((log_mtts_per_seg < 1) || (log_mtts_per_seg > 5)) { - printk(KERN_WARNING PFX "bad log_mtts_per_seg (%d). Using default - %ld\n", + printk(KERN_WARNING PFX "bad log_mtts_per_seg (%d). Using default - %d\n", log_mtts_per_seg, ilog2(MTHCA_MTT_SEG_SIZE / 8)); log_mtts_per_seg = ilog2(MTHCA_MTT_SEG_SIZE / 8); } } static int __init mthca_init(void) { int ret; mthca_validate_profile(); ret = mthca_catas_init(); if (ret) return ret; ret = pci_register_driver(&mthca_driver); if (ret < 0) { mthca_catas_cleanup(); return ret; } return 0; } static void __exit mthca_cleanup(void) { pci_unregister_driver(&mthca_driver); mthca_catas_cleanup(); } module_init_order(mthca_init, SI_ORDER_MIDDLE); module_exit(mthca_cleanup); Index: stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_memfree.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_memfree.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_memfree.c (revision 273864) @@ -1,879 +1,880 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Cisco Systems. All rights reserved. * Copyright (c) 2005 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #include "mthca_memfree.h" #include "mthca_dev.h" #include "mthca_cmd.h" /* * We allocate in as big chunks as we can, up to a maximum of 256 KB * per chunk. */ enum { MTHCA_ICM_ALLOC_SIZE = 1 << 18, MTHCA_TABLE_CHUNK_SIZE = 1 << 18 }; struct mthca_user_db_table { struct mutex mutex; struct { u64 uvirt; struct scatterlist mem; int refcount; } page[0]; }; static void mthca_free_icm_pages(struct mthca_dev *dev, struct mthca_icm_chunk *chunk) { int i; if (chunk->nsg > 0) pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages, PCI_DMA_BIDIRECTIONAL); for (i = 0; i < chunk->npages; ++i) __free_pages(sg_page(&chunk->mem[i]), get_order(chunk->mem[i].length)); } static void mthca_free_icm_coherent(struct mthca_dev *dev, struct mthca_icm_chunk *chunk) { int i; for (i = 0; i < chunk->npages; ++i) { dma_free_coherent(&dev->pdev->dev, chunk->mem[i].length, lowmem_page_address(sg_page(&chunk->mem[i])), sg_dma_address(&chunk->mem[i])); } } void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm, int coherent) { struct mthca_icm_chunk *chunk, *tmp; if (!icm) return; list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) { if (coherent) mthca_free_icm_coherent(dev, chunk); else mthca_free_icm_pages(dev, chunk); kfree(chunk); } kfree(icm); } static int mthca_alloc_icm_pages(struct scatterlist *mem, int order, gfp_t gfp_mask) { struct page *page; /* * Use __GFP_ZERO because buggy firmware assumes ICM pages are * cleared, and subtle failures are seen if they aren't. */ page = alloc_pages(gfp_mask | __GFP_ZERO, order); if (!page) return -ENOMEM; sg_set_page(mem, page, PAGE_SIZE << order, 0); return 0; } static int mthca_alloc_icm_coherent(struct device *dev, struct scatterlist *mem, int order, gfp_t gfp_mask) { void *buf = dma_alloc_coherent(dev, PAGE_SIZE << order, &sg_dma_address(mem), gfp_mask); if (!buf) return -ENOMEM; sg_set_buf(mem, buf, PAGE_SIZE << order); BUG_ON(mem->offset); sg_dma_len(mem) = PAGE_SIZE << order; return 0; } struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages, gfp_t gfp_mask, int coherent) { struct mthca_icm *icm; struct mthca_icm_chunk *chunk = NULL; int cur_order; int ret; /* We use sg_set_buf for coherent allocs, which assumes low memory */ BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM)); icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); if (!icm) return icm; icm->refcount = 0; INIT_LIST_HEAD(&icm->chunk_list); cur_order = get_order(MTHCA_ICM_ALLOC_SIZE); while (npages > 0) { if (!chunk) { chunk = kmalloc(sizeof *chunk, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); if (!chunk) goto fail; sg_init_table(chunk->mem, MTHCA_ICM_CHUNK_LEN); chunk->npages = 0; chunk->nsg = 0; list_add_tail(&chunk->list, &icm->chunk_list); } while (1 << cur_order > npages) --cur_order; if (coherent) ret = mthca_alloc_icm_coherent(&dev->pdev->dev, &chunk->mem[chunk->npages], cur_order, gfp_mask); else ret = mthca_alloc_icm_pages(&chunk->mem[chunk->npages], cur_order, gfp_mask); if (!ret) { ++chunk->npages; if (coherent) ++chunk->nsg; else if (chunk->npages == MTHCA_ICM_CHUNK_LEN) { chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, chunk->npages, PCI_DMA_BIDIRECTIONAL); if (chunk->nsg <= 0) goto fail; } if (chunk->npages == MTHCA_ICM_CHUNK_LEN) chunk = NULL; npages -= 1 << cur_order; } else { --cur_order; if (cur_order < 0) goto fail; } } if (!coherent && chunk) { chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, chunk->npages, PCI_DMA_BIDIRECTIONAL); if (chunk->nsg <= 0) goto fail; } return icm; fail: mthca_free_icm(dev, icm, coherent); return NULL; } int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj) { int i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE; int ret = 0; u8 status; mutex_lock(&table->mutex); if (table->icm[i]) { ++table->icm[i]->refcount; goto out; } table->icm[i] = mthca_alloc_icm(dev, MTHCA_TABLE_CHUNK_SIZE >> PAGE_SHIFT, (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN, table->coherent); if (!table->icm[i]) { ret = -ENOMEM; goto out; } if (mthca_MAP_ICM(dev, table->icm[i], table->virt + i * MTHCA_TABLE_CHUNK_SIZE, &status) || status) { mthca_free_icm(dev, table->icm[i], table->coherent); table->icm[i] = NULL; ret = -ENOMEM; goto out; } ++table->icm[i]->refcount; out: mutex_unlock(&table->mutex); return ret; } void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj) { int i; u8 status; if (!mthca_is_memfree(dev)) return; i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE; mutex_lock(&table->mutex); if (--table->icm[i]->refcount == 0) { mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE, MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE, &status); mthca_free_icm(dev, table->icm[i], table->coherent); table->icm[i] = NULL; } mutex_unlock(&table->mutex); } void *mthca_table_find(struct mthca_icm_table *table, int obj, dma_addr_t *dma_handle) { int idx, offset, dma_offset, i; struct mthca_icm_chunk *chunk; struct mthca_icm *icm; struct page *page = NULL; if (!table->lowmem) return NULL; mutex_lock(&table->mutex); idx = (obj & (table->num_obj - 1)) * table->obj_size; icm = table->icm[idx / MTHCA_TABLE_CHUNK_SIZE]; dma_offset = offset = idx % MTHCA_TABLE_CHUNK_SIZE; if (!icm) goto out; list_for_each_entry(chunk, &icm->chunk_list, list) { for (i = 0; i < chunk->npages; ++i) { if (dma_handle && dma_offset >= 0) { if (sg_dma_len(&chunk->mem[i]) > dma_offset) *dma_handle = sg_dma_address(&chunk->mem[i]) + dma_offset; dma_offset -= sg_dma_len(&chunk->mem[i]); } /* DMA mapping can merge pages but not split them, * so if we found the page, dma_handle has already * been assigned to. */ if (chunk->mem[i].length > offset) { page = sg_page(&chunk->mem[i]); goto out; } offset -= chunk->mem[i].length; } } out: mutex_unlock(&table->mutex); return page ? lowmem_page_address(page) + offset : NULL; } int mthca_table_get_range(struct mthca_dev *dev, struct mthca_icm_table *table, int start, int end) { int inc = MTHCA_TABLE_CHUNK_SIZE / table->obj_size; int i, err; for (i = start; i <= end; i += inc) { err = mthca_table_get(dev, table, i); if (err) goto fail; } return 0; fail: while (i > start) { i -= inc; mthca_table_put(dev, table, i); } return err; } void mthca_table_put_range(struct mthca_dev *dev, struct mthca_icm_table *table, int start, int end) { int i; if (!mthca_is_memfree(dev)) return; for (i = start; i <= end; i += MTHCA_TABLE_CHUNK_SIZE / table->obj_size) mthca_table_put(dev, table, i); } struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev, u64 virt, int obj_size, int nobj, int reserved, int use_lowmem, int use_coherent) { struct mthca_icm_table *table; int obj_per_chunk; int num_icm; unsigned chunk_size; int i; u8 status; obj_per_chunk = MTHCA_TABLE_CHUNK_SIZE / obj_size; num_icm = DIV_ROUND_UP(nobj, obj_per_chunk); table = kmalloc(sizeof *table + num_icm * sizeof *table->icm, GFP_KERNEL); if (!table) return NULL; table->virt = virt; table->num_icm = num_icm; table->num_obj = nobj; table->obj_size = obj_size; table->lowmem = use_lowmem; table->coherent = use_coherent; mutex_init(&table->mutex); for (i = 0; i < num_icm; ++i) table->icm[i] = NULL; for (i = 0; i * MTHCA_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) { chunk_size = MTHCA_TABLE_CHUNK_SIZE; if ((i + 1) * MTHCA_TABLE_CHUNK_SIZE > nobj * obj_size) chunk_size = nobj * obj_size - i * MTHCA_TABLE_CHUNK_SIZE; table->icm[i] = mthca_alloc_icm(dev, chunk_size >> PAGE_SHIFT, (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN, use_coherent); if (!table->icm[i]) goto err; if (mthca_MAP_ICM(dev, table->icm[i], virt + i * MTHCA_TABLE_CHUNK_SIZE, &status) || status) { mthca_free_icm(dev, table->icm[i], table->coherent); table->icm[i] = NULL; goto err; } /* * Add a reference to this ICM chunk so that it never * gets freed (since it contains reserved firmware objects). */ ++table->icm[i]->refcount; } return table; err: for (i = 0; i < num_icm; ++i) if (table->icm[i]) { mthca_UNMAP_ICM(dev, virt + i * MTHCA_TABLE_CHUNK_SIZE, MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE, &status); mthca_free_icm(dev, table->icm[i], table->coherent); } kfree(table); return NULL; } void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table) { int i; u8 status; for (i = 0; i < table->num_icm; ++i) if (table->icm[i]) { mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE, MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE, &status); mthca_free_icm(dev, table->icm[i], table->coherent); } kfree(table); } static u64 mthca_uarc_virt(struct mthca_dev *dev, struct mthca_uar *uar, int page) { return dev->uar_table.uarc_base + uar->index * dev->uar_table.uarc_size + page * MTHCA_ICM_PAGE_SIZE; } + #include #include #include #include #include int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, struct mthca_user_db_table *db_tab, int index, u64 uaddr) { #ifdef __linux__ struct page *pages[1]; int ret = 0; u8 status; int i; if (!mthca_is_memfree(dev)) return 0; if (index < 0 || index > dev->uar_table.uarc_size / 8) return -EINVAL; mutex_lock(&db_tab->mutex); i = index / MTHCA_DB_REC_PER_PAGE; if ((db_tab->page[i].refcount >= MTHCA_DB_REC_PER_PAGE) || (db_tab->page[i].uvirt && db_tab->page[i].uvirt != uaddr) || (uaddr & 4095)) { ret = -EINVAL; goto out; } if (db_tab->page[i].refcount) { ++db_tab->page[i].refcount; goto out; } ret = get_user_pages(current, current->mm, uaddr & PAGE_MASK, 1, 1, 0, pages, NULL); if (ret < 0) goto out; sg_set_page(&db_tab->page[i].mem, pages[0], MTHCA_ICM_PAGE_SIZE, uaddr & ~PAGE_MASK); ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); if (ret < 0) { put_page(pages[0]); goto out; } ret = mthca_MAP_ICM_page(dev, sg_dma_address(&db_tab->page[i].mem), mthca_uarc_virt(dev, uar, i), &status); if (!ret && status) ret = -EINVAL; if (ret) { pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); put_page(sg_page(&db_tab->page[i].mem)); goto out; } db_tab->page[i].uvirt = uaddr; db_tab->page[i].refcount = 1; out: mutex_unlock(&db_tab->mutex); return ret; #else struct proc *proc; vm_offset_t start; vm_paddr_t paddr; pmap_t pmap; vm_page_t m; int ret = 0; u8 status; int i; if (!mthca_is_memfree(dev)) return 0; if (index < 0 || index > dev->uar_table.uarc_size / 8) return -EINVAL; mutex_lock(&db_tab->mutex); i = index / MTHCA_DB_REC_PER_PAGE; start = 0; if ((db_tab->page[i].refcount >= MTHCA_DB_REC_PER_PAGE) || (db_tab->page[i].uvirt && db_tab->page[i].uvirt != uaddr) || (uaddr & 4095)) { ret = -EINVAL; goto out; } if (db_tab->page[i].refcount) { ++db_tab->page[i].refcount; goto out; } proc = curproc; pmap = vm_map_pmap(&proc->p_vmspace->vm_map); PROC_LOCK(proc); if (ptoa(pmap_wired_count(pmap) + 1) > lim_cur(proc, RLIMIT_MEMLOCK)) { PROC_UNLOCK(proc); ret = -ENOMEM; goto out; } PROC_UNLOCK(proc); if (cnt.v_wire_count + 1 > vm_page_max_wired) { ret = -EAGAIN; goto out; } start = uaddr & PAGE_MASK; ret = vm_map_wire(&proc->p_vmspace->vm_map, start, start + PAGE_SIZE, VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES | VM_MAP_WIRE_WRITE); if (ret != KERN_SUCCESS) { start = 0; ret = -ENOMEM; goto out; } paddr = pmap_extract(pmap, uaddr); if (paddr == 0) { ret = -EFAULT; goto out; } m = PHYS_TO_VM_PAGE(paddr); sg_set_page(&db_tab->page[i].mem, m, MTHCA_ICM_PAGE_SIZE, uaddr & ~PAGE_MASK); ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); if (ret < 0) goto out; ret = mthca_MAP_ICM_page(dev, sg_dma_address(&db_tab->page[i].mem), mthca_uarc_virt(dev, uar, i), &status); if (!ret && status) ret = -EINVAL; if (ret) { pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); goto out; } db_tab->page[i].uvirt = uaddr; db_tab->page[i].refcount = 1; out: if (ret < 0 && start) vm_map_unwire(&curthread->td_proc->p_vmspace->vm_map, start, start + PAGE_SIZE, VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); mutex_unlock(&db_tab->mutex); return ret; #endif } void mthca_unmap_user_db(struct mthca_dev *dev, struct mthca_uar *uar, struct mthca_user_db_table *db_tab, int index) { if (!mthca_is_memfree(dev)) return; /* * To make our bookkeeping simpler, we don't unmap DB * pages until we clean up the whole db table. */ mutex_lock(&db_tab->mutex); --db_tab->page[index / MTHCA_DB_REC_PER_PAGE].refcount; mutex_unlock(&db_tab->mutex); } struct mthca_user_db_table *mthca_init_user_db_tab(struct mthca_dev *dev) { struct mthca_user_db_table *db_tab; int npages; int i; if (!mthca_is_memfree(dev)) return NULL; npages = dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; db_tab = kmalloc(sizeof *db_tab + npages * sizeof *db_tab->page, GFP_KERNEL); if (!db_tab) return ERR_PTR(-ENOMEM); mutex_init(&db_tab->mutex); for (i = 0; i < npages; ++i) { db_tab->page[i].refcount = 0; db_tab->page[i].uvirt = 0; sg_init_table(&db_tab->page[i].mem, 1); } return db_tab; } void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar, struct mthca_user_db_table *db_tab) { int i; u8 status; if (!mthca_is_memfree(dev)) return; for (i = 0; i < dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; ++i) { if (db_tab->page[i].uvirt) { mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1, &status); pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); #ifdef __linux__ put_page(sg_page(&db_tab->page[i].mem)); #else vm_offset_t start; start = db_tab->page[i].uvirt & PAGE_MASK; vm_map_unwire(&curthread->td_proc->p_vmspace->vm_map, start, start + PAGE_SIZE, VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); #endif } } kfree(db_tab); } int mthca_alloc_db(struct mthca_dev *dev, enum mthca_db_type type, u32 qn, __be32 **db) { int group; int start, end, dir; int i, j; struct mthca_db_page *page; int ret = 0; u8 status; mutex_lock(&dev->db_tab->mutex); switch (type) { case MTHCA_DB_TYPE_CQ_ARM: case MTHCA_DB_TYPE_SQ: group = 0; start = 0; end = dev->db_tab->max_group1; dir = 1; break; case MTHCA_DB_TYPE_CQ_SET_CI: case MTHCA_DB_TYPE_RQ: case MTHCA_DB_TYPE_SRQ: group = 1; start = dev->db_tab->npages - 1; end = dev->db_tab->min_group2; dir = -1; break; default: ret = -EINVAL; goto out; } for (i = start; i != end; i += dir) if (dev->db_tab->page[i].db_rec && !bitmap_full(dev->db_tab->page[i].used, MTHCA_DB_REC_PER_PAGE)) { page = dev->db_tab->page + i; goto found; } for (i = start; i != end; i += dir) if (!dev->db_tab->page[i].db_rec) { page = dev->db_tab->page + i; goto alloc; } if (dev->db_tab->max_group1 >= dev->db_tab->min_group2 - 1) { ret = -ENOMEM; goto out; } if (group == 0) ++dev->db_tab->max_group1; else --dev->db_tab->min_group2; page = dev->db_tab->page + end; alloc: page->db_rec = dma_alloc_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, &page->mapping, GFP_KERNEL); if (!page->db_rec) { ret = -ENOMEM; goto out; } memset(page->db_rec, 0, MTHCA_ICM_PAGE_SIZE); ret = mthca_MAP_ICM_page(dev, page->mapping, mthca_uarc_virt(dev, &dev->driver_uar, i), &status); if (!ret && status) ret = -EINVAL; if (ret) { dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, page->db_rec, page->mapping); goto out; } bitmap_zero(page->used, MTHCA_DB_REC_PER_PAGE); found: j = find_first_zero_bit(page->used, MTHCA_DB_REC_PER_PAGE); set_bit(j, page->used); if (group == 1) j = MTHCA_DB_REC_PER_PAGE - 1 - j; ret = i * MTHCA_DB_REC_PER_PAGE + j; page->db_rec[j] = cpu_to_be64((qn << 8) | (type << 5)); *db = (__be32 *) &page->db_rec[j]; out: mutex_unlock(&dev->db_tab->mutex); return ret; } void mthca_free_db(struct mthca_dev *dev, int type, int db_index) { int i, j; struct mthca_db_page *page; u8 status; i = db_index / MTHCA_DB_REC_PER_PAGE; j = db_index % MTHCA_DB_REC_PER_PAGE; page = dev->db_tab->page + i; mutex_lock(&dev->db_tab->mutex); page->db_rec[j] = 0; if (i >= dev->db_tab->min_group2) j = MTHCA_DB_REC_PER_PAGE - 1 - j; clear_bit(j, page->used); if (bitmap_empty(page->used, MTHCA_DB_REC_PER_PAGE) && i >= dev->db_tab->max_group1 - 1) { mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1, &status); dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, page->db_rec, page->mapping); page->db_rec = NULL; if (i == dev->db_tab->max_group1) { --dev->db_tab->max_group1; /* XXX may be able to unmap more pages now */ } if (i == dev->db_tab->min_group2) ++dev->db_tab->min_group2; } mutex_unlock(&dev->db_tab->mutex); } int mthca_init_db_tab(struct mthca_dev *dev) { int i; if (!mthca_is_memfree(dev)) return 0; dev->db_tab = kmalloc(sizeof *dev->db_tab, GFP_KERNEL); if (!dev->db_tab) return -ENOMEM; mutex_init(&dev->db_tab->mutex); dev->db_tab->npages = dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; dev->db_tab->max_group1 = 0; dev->db_tab->min_group2 = dev->db_tab->npages - 1; dev->db_tab->page = kmalloc(dev->db_tab->npages * sizeof *dev->db_tab->page, GFP_KERNEL); if (!dev->db_tab->page) { kfree(dev->db_tab); return -ENOMEM; } for (i = 0; i < dev->db_tab->npages; ++i) dev->db_tab->page[i].db_rec = NULL; return 0; } void mthca_cleanup_db_tab(struct mthca_dev *dev) { int i; u8 status; if (!mthca_is_memfree(dev)) return; /* * Because we don't always free our UARC pages when they * become empty to make mthca_free_db() simpler we need to * make a sweep through the doorbell pages and free any * leftover pages now. */ for (i = 0; i < dev->db_tab->npages; ++i) { if (!dev->db_tab->page[i].db_rec) continue; if (!bitmap_empty(dev->db_tab->page[i].used, MTHCA_DB_REC_PER_PAGE)) mthca_warn(dev, "Kernel UARC page %d not empty\n", i); mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1, &status); dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, dev->db_tab->page[i].db_rec, dev->db_tab->page[i].mapping); } kfree(dev->db_tab->page); kfree(dev->db_tab); } Index: stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_provider.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_provider.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_provider.c (revision 273864) @@ -1,1427 +1,1428 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. * Copyright (c) 2005 Mellanox Technologies. All rights reserved. * Copyright (c) 2004 Voltaire, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #include +#include #include "mthca_dev.h" #include "mthca_cmd.h" #include "mthca_user.h" #include "mthca_memfree.h" static void init_query_mad(struct ib_smp *mad) { mad->base_version = 1; mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; mad->class_version = 1; mad->method = IB_MGMT_METHOD_GET; } static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *props) { struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; int err = -ENOMEM; struct mthca_dev *mdev = to_mdev(ibdev); u8 status; in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); if (!in_mad || !out_mad) goto out; memset(props, 0, sizeof *props); props->fw_ver = mdev->fw_ver; init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; err = mthca_MAD_IFC(mdev, 1, 1, 1, NULL, NULL, in_mad, out_mad, &status); if (err) goto out; if (status) { err = -EINVAL; goto out; } props->device_cap_flags = mdev->device_cap_flags; props->vendor_id = be32_to_cpup((__be32 *) (out_mad->data + 36)) & 0xffffff; props->vendor_part_id = be16_to_cpup((__be16 *) (out_mad->data + 30)); props->hw_ver = be32_to_cpup((__be32 *) (out_mad->data + 32)); memcpy(&props->sys_image_guid, out_mad->data + 4, 8); props->max_mr_size = ~0ull; props->page_size_cap = mdev->limits.page_size_cap; props->max_qp = mdev->limits.num_qps - mdev->limits.reserved_qps; props->max_qp_wr = mdev->limits.max_wqes; props->max_sge = mdev->limits.max_sg; props->max_cq = mdev->limits.num_cqs - mdev->limits.reserved_cqs; props->max_cqe = mdev->limits.max_cqes; props->max_mr = mdev->limits.num_mpts - mdev->limits.reserved_mrws; props->max_pd = mdev->limits.num_pds - mdev->limits.reserved_pds; props->max_qp_rd_atom = 1 << mdev->qp_table.rdb_shift; props->max_qp_init_rd_atom = mdev->limits.max_qp_init_rdma; props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; props->max_srq = mdev->limits.num_srqs - mdev->limits.reserved_srqs; props->max_srq_wr = mdev->limits.max_srq_wqes; props->max_srq_sge = mdev->limits.max_srq_sge; props->local_ca_ack_delay = mdev->limits.local_ca_ack_delay; props->atomic_cap = mdev->limits.flags & DEV_LIM_FLAG_ATOMIC ? IB_ATOMIC_HCA : IB_ATOMIC_NONE; props->max_pkeys = mdev->limits.pkey_table_len; props->max_mcast_grp = mdev->limits.num_mgms + mdev->limits.num_amgms; props->max_mcast_qp_attach = MTHCA_QP_PER_MGM; props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * props->max_mcast_grp; /* * If Sinai memory key optimization is being used, then only * the 8-bit key portion will change. For other HCAs, the * unused index bits will also be used for FMR remapping. */ if (mdev->mthca_flags & MTHCA_FLAG_SINAI_OPT) props->max_map_per_fmr = 255; else props->max_map_per_fmr = (1 << (32 - ilog2(mdev->limits.num_mpts))) - 1; err = 0; out: kfree(in_mad); kfree(out_mad); return err; } static int mthca_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) { struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; int err = -ENOMEM; u8 status; in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); if (!in_mad || !out_mad) goto out; memset(props, 0, sizeof *props); init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; in_mad->attr_mod = cpu_to_be32(port); err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad, &status); if (err) goto out; if (status) { err = -EINVAL; goto out; } props->lid = be16_to_cpup((__be16 *) (out_mad->data + 16)); props->lmc = out_mad->data[34] & 0x7; props->sm_lid = be16_to_cpup((__be16 *) (out_mad->data + 18)); props->sm_sl = out_mad->data[36] & 0xf; props->state = out_mad->data[32] & 0xf; props->phys_state = out_mad->data[33] >> 4; props->port_cap_flags = be32_to_cpup((__be32 *) (out_mad->data + 20)); props->gid_tbl_len = to_mdev(ibdev)->limits.gid_table_len; props->max_msg_sz = 0x80000000; props->pkey_tbl_len = to_mdev(ibdev)->limits.pkey_table_len; props->bad_pkey_cntr = be16_to_cpup((__be16 *) (out_mad->data + 46)); props->qkey_viol_cntr = be16_to_cpup((__be16 *) (out_mad->data + 48)); props->active_width = out_mad->data[31] & 0xf; props->active_speed = out_mad->data[35] >> 4; props->max_mtu = out_mad->data[41] & 0xf; props->active_mtu = out_mad->data[36] >> 4; props->subnet_timeout = out_mad->data[51] & 0x1f; props->max_vl_num = out_mad->data[37] >> 4; props->init_type_reply = out_mad->data[41] >> 4; out: kfree(in_mad); kfree(out_mad); return err; } static int mthca_modify_device(struct ib_device *ibdev, int mask, struct ib_device_modify *props) { if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) return -EOPNOTSUPP; if (mask & IB_DEVICE_MODIFY_NODE_DESC) { if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex)) return -ERESTARTSYS; memcpy(ibdev->node_desc, props->node_desc, 64); mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex); } return 0; } static int mthca_modify_port(struct ib_device *ibdev, u8 port, int port_modify_mask, struct ib_port_modify *props) { struct mthca_set_ib_param set_ib; struct ib_port_attr attr; int err; u8 status; if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex)) return -ERESTARTSYS; err = mthca_query_port(ibdev, port, &attr); if (err) goto out; set_ib.set_si_guid = 0; set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR); set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) & ~props->clr_port_cap_mask; err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port, &status); if (err) goto out; if (status) { err = -EINVAL; goto out; } out: mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex); return err; } static int mthca_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) { struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; int err = -ENOMEM; u8 status; in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); if (!in_mad || !out_mad) goto out; init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE; in_mad->attr_mod = cpu_to_be32(index / 32); err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad, &status); if (err) goto out; if (status) { err = -EINVAL; goto out; } *pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]); out: kfree(in_mad); kfree(out_mad); return err; } static int mthca_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid) { struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; int err = -ENOMEM; u8 status; in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); if (!in_mad || !out_mad) goto out; init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; in_mad->attr_mod = cpu_to_be32(port); err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad, &status); if (err) goto out; if (status) { err = -EINVAL; goto out; } memcpy(gid->raw, out_mad->data + 8, 8); init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; in_mad->attr_mod = cpu_to_be32(index / 8); err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad, &status); if (err) goto out; if (status) { err = -EINVAL; goto out; } memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8); out: kfree(in_mad); kfree(out_mad); return err; } static struct ib_ucontext *mthca_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata) { struct mthca_alloc_ucontext_resp uresp; struct mthca_ucontext *context; int err; if (!(to_mdev(ibdev)->active)) return ERR_PTR(-EAGAIN); memset(&uresp, 0, sizeof uresp); uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps; if (mthca_is_memfree(to_mdev(ibdev))) uresp.uarc_size = to_mdev(ibdev)->uar_table.uarc_size; else uresp.uarc_size = 0; context = kmalloc(sizeof *context, GFP_KERNEL); if (!context) return ERR_PTR(-ENOMEM); err = mthca_uar_alloc(to_mdev(ibdev), &context->uar); if (err) { kfree(context); return ERR_PTR(err); } context->db_tab = mthca_init_user_db_tab(to_mdev(ibdev)); if (IS_ERR(context->db_tab)) { err = PTR_ERR(context->db_tab); mthca_uar_free(to_mdev(ibdev), &context->uar); kfree(context); return ERR_PTR(err); } if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) { mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar, context->db_tab); mthca_uar_free(to_mdev(ibdev), &context->uar); kfree(context); return ERR_PTR(-EFAULT); } context->reg_mr_warned = 0; return &context->ibucontext; } static int mthca_dealloc_ucontext(struct ib_ucontext *context) { mthca_cleanup_user_db_tab(to_mdev(context->device), &to_mucontext(context)->uar, to_mucontext(context)->db_tab); mthca_uar_free(to_mdev(context->device), &to_mucontext(context)->uar); kfree(to_mucontext(context)); return 0; } static int mthca_mmap_uar(struct ib_ucontext *context, struct vm_area_struct *vma) { if (vma->vm_end - vma->vm_start != PAGE_SIZE) return -EINVAL; vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); if (io_remap_pfn_range(vma, vma->vm_start, to_mucontext(context)->uar.pfn, PAGE_SIZE, vma->vm_page_prot)) return -EAGAIN; return 0; } static struct ib_pd *mthca_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context, struct ib_udata *udata) { struct mthca_pd *pd; int err; pd = kmalloc(sizeof *pd, GFP_KERNEL); if (!pd) return ERR_PTR(-ENOMEM); err = mthca_pd_alloc(to_mdev(ibdev), !context, pd); if (err) { kfree(pd); return ERR_PTR(err); } if (context) { if (ib_copy_to_udata(udata, &pd->pd_num, sizeof (__u32))) { mthca_pd_free(to_mdev(ibdev), pd); kfree(pd); return ERR_PTR(-EFAULT); } } return &pd->ibpd; } static int mthca_dealloc_pd(struct ib_pd *pd) { mthca_pd_free(to_mdev(pd->device), to_mpd(pd)); kfree(pd); return 0; } static struct ib_ah *mthca_ah_create(struct ib_pd *pd, struct ib_ah_attr *ah_attr) { int err; struct mthca_ah *ah; ah = kmalloc(sizeof *ah, GFP_ATOMIC); if (!ah) return ERR_PTR(-ENOMEM); err = mthca_create_ah(to_mdev(pd->device), to_mpd(pd), ah_attr, ah); if (err) { kfree(ah); return ERR_PTR(err); } return &ah->ibah; } static int mthca_ah_destroy(struct ib_ah *ah) { mthca_destroy_ah(to_mdev(ah->device), to_mah(ah)); kfree(ah); return 0; } static struct ib_srq *mthca_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *init_attr, struct ib_udata *udata) { struct mthca_create_srq ucmd; struct mthca_ucontext *context = NULL; struct mthca_srq *srq; int err; srq = kmalloc(sizeof *srq, GFP_KERNEL); if (!srq) return ERR_PTR(-ENOMEM); if (pd->uobject) { context = to_mucontext(pd->uobject->context); if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { err = -EFAULT; goto err_free; } err = mthca_map_user_db(to_mdev(pd->device), &context->uar, context->db_tab, ucmd.db_index, ucmd.db_page); if (err) goto err_free; srq->mr.ibmr.lkey = ucmd.lkey; srq->db_index = ucmd.db_index; } err = mthca_alloc_srq(to_mdev(pd->device), to_mpd(pd), &init_attr->attr, srq); if (err && pd->uobject) mthca_unmap_user_db(to_mdev(pd->device), &context->uar, context->db_tab, ucmd.db_index); if (err) goto err_free; if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof (__u32))) { mthca_free_srq(to_mdev(pd->device), srq); err = -EFAULT; goto err_free; } return &srq->ibsrq; err_free: kfree(srq); return ERR_PTR(err); } static int mthca_destroy_srq(struct ib_srq *srq) { struct mthca_ucontext *context; if (srq->uobject) { context = to_mucontext(srq->uobject->context); mthca_unmap_user_db(to_mdev(srq->device), &context->uar, context->db_tab, to_msrq(srq)->db_index); } mthca_free_srq(to_mdev(srq->device), to_msrq(srq)); kfree(srq); return 0; } static struct ib_qp *mthca_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { struct mthca_create_qp ucmd; struct mthca_qp *qp; int err; if (init_attr->create_flags) return ERR_PTR(-EINVAL); switch (init_attr->qp_type) { case IB_QPT_RC: case IB_QPT_UC: case IB_QPT_UD: { struct mthca_ucontext *context; qp = kmalloc(sizeof *qp, GFP_KERNEL); if (!qp) return ERR_PTR(-ENOMEM); if (pd->uobject) { context = to_mucontext(pd->uobject->context); if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { kfree(qp); return ERR_PTR(-EFAULT); } err = mthca_map_user_db(to_mdev(pd->device), &context->uar, context->db_tab, ucmd.sq_db_index, ucmd.sq_db_page); if (err) { kfree(qp); return ERR_PTR(err); } err = mthca_map_user_db(to_mdev(pd->device), &context->uar, context->db_tab, ucmd.rq_db_index, ucmd.rq_db_page); if (err) { mthca_unmap_user_db(to_mdev(pd->device), &context->uar, context->db_tab, ucmd.sq_db_index); kfree(qp); return ERR_PTR(err); } qp->mr.ibmr.lkey = ucmd.lkey; qp->sq.db_index = ucmd.sq_db_index; qp->rq.db_index = ucmd.rq_db_index; } err = mthca_alloc_qp(to_mdev(pd->device), to_mpd(pd), to_mcq(init_attr->send_cq), to_mcq(init_attr->recv_cq), init_attr->qp_type, init_attr->sq_sig_type, &init_attr->cap, qp); if (err && pd->uobject) { context = to_mucontext(pd->uobject->context); mthca_unmap_user_db(to_mdev(pd->device), &context->uar, context->db_tab, ucmd.sq_db_index); mthca_unmap_user_db(to_mdev(pd->device), &context->uar, context->db_tab, ucmd.rq_db_index); } qp->ibqp.qp_num = qp->qpn; break; } case IB_QPT_SMI: case IB_QPT_GSI: { /* Don't allow userspace to create special QPs */ if (pd->uobject) return ERR_PTR(-EINVAL); qp = kmalloc(sizeof (struct mthca_sqp), GFP_KERNEL); if (!qp) return ERR_PTR(-ENOMEM); qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1; err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd), to_mcq(init_attr->send_cq), to_mcq(init_attr->recv_cq), init_attr->sq_sig_type, &init_attr->cap, qp->ibqp.qp_num, init_attr->port_num, to_msqp(qp)); break; } default: /* Don't support raw QPs */ return ERR_PTR(-ENOSYS); } if (err) { kfree(qp); return ERR_PTR(err); } init_attr->cap.max_send_wr = qp->sq.max; init_attr->cap.max_recv_wr = qp->rq.max; init_attr->cap.max_send_sge = qp->sq.max_gs; init_attr->cap.max_recv_sge = qp->rq.max_gs; init_attr->cap.max_inline_data = qp->max_inline_data; return &qp->ibqp; } static int mthca_destroy_qp(struct ib_qp *qp) { if (qp->uobject) { mthca_unmap_user_db(to_mdev(qp->device), &to_mucontext(qp->uobject->context)->uar, to_mucontext(qp->uobject->context)->db_tab, to_mqp(qp)->sq.db_index); mthca_unmap_user_db(to_mdev(qp->device), &to_mucontext(qp->uobject->context)->uar, to_mucontext(qp->uobject->context)->db_tab, to_mqp(qp)->rq.db_index); } mthca_free_qp(to_mdev(qp->device), to_mqp(qp)); kfree(qp); return 0; } static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, int entries, int comp_vector, struct ib_ucontext *context, struct ib_udata *udata) { struct mthca_create_cq ucmd; struct mthca_cq *cq; int nent; int err; if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes) return ERR_PTR(-EINVAL); if (context) { if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) return ERR_PTR(-EFAULT); err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar, to_mucontext(context)->db_tab, ucmd.set_db_index, ucmd.set_db_page); if (err) return ERR_PTR(err); err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar, to_mucontext(context)->db_tab, ucmd.arm_db_index, ucmd.arm_db_page); if (err) goto err_unmap_set; } cq = kmalloc(sizeof *cq, GFP_KERNEL); if (!cq) { err = -ENOMEM; goto err_unmap_arm; } if (context) { cq->buf.mr.ibmr.lkey = ucmd.lkey; cq->set_ci_db_index = ucmd.set_db_index; cq->arm_db_index = ucmd.arm_db_index; } for (nent = 1; nent <= entries; nent <<= 1) ; /* nothing */ err = mthca_init_cq(to_mdev(ibdev), nent, context ? to_mucontext(context) : NULL, context ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num, cq); if (err) goto err_free; if (context && ib_copy_to_udata(udata, &cq->cqn, sizeof (__u32))) { mthca_free_cq(to_mdev(ibdev), cq); goto err_free; } cq->resize_buf = NULL; return &cq->ibcq; err_free: kfree(cq); err_unmap_arm: if (context) mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar, to_mucontext(context)->db_tab, ucmd.arm_db_index); err_unmap_set: if (context) mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar, to_mucontext(context)->db_tab, ucmd.set_db_index); return ERR_PTR(err); } static int mthca_alloc_resize_buf(struct mthca_dev *dev, struct mthca_cq *cq, int entries) { int ret; spin_lock_irq(&cq->lock); if (cq->resize_buf) { ret = -EBUSY; goto unlock; } cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC); if (!cq->resize_buf) { ret = -ENOMEM; goto unlock; } cq->resize_buf->state = CQ_RESIZE_ALLOC; ret = 0; unlock: spin_unlock_irq(&cq->lock); if (ret) return ret; ret = mthca_alloc_cq_buf(dev, &cq->resize_buf->buf, entries); if (ret) { spin_lock_irq(&cq->lock); kfree(cq->resize_buf); cq->resize_buf = NULL; spin_unlock_irq(&cq->lock); return ret; } cq->resize_buf->cqe = entries - 1; spin_lock_irq(&cq->lock); cq->resize_buf->state = CQ_RESIZE_READY; spin_unlock_irq(&cq->lock); return 0; } static int mthca_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) { struct mthca_dev *dev = to_mdev(ibcq->device); struct mthca_cq *cq = to_mcq(ibcq); struct mthca_resize_cq ucmd; u32 lkey; u8 status; int ret; if (entries < 1 || entries > dev->limits.max_cqes) return -EINVAL; mutex_lock(&cq->mutex); entries = roundup_pow_of_two(entries + 1); if (entries == ibcq->cqe + 1) { ret = 0; goto out; } if (cq->is_kernel) { ret = mthca_alloc_resize_buf(dev, cq, entries); if (ret) goto out; lkey = cq->resize_buf->buf.mr.ibmr.lkey; } else { if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { ret = -EFAULT; goto out; } lkey = ucmd.lkey; } ret = mthca_RESIZE_CQ(dev, cq->cqn, lkey, ilog2(entries), &status); if (status) ret = -EINVAL; if (ret) { if (cq->resize_buf) { mthca_free_cq_buf(dev, &cq->resize_buf->buf, cq->resize_buf->cqe); kfree(cq->resize_buf); spin_lock_irq(&cq->lock); cq->resize_buf = NULL; spin_unlock_irq(&cq->lock); } goto out; } if (cq->is_kernel) { struct mthca_cq_buf tbuf; int tcqe; spin_lock_irq(&cq->lock); if (cq->resize_buf->state == CQ_RESIZE_READY) { mthca_cq_resize_copy_cqes(cq); tbuf = cq->buf; tcqe = cq->ibcq.cqe; cq->buf = cq->resize_buf->buf; cq->ibcq.cqe = cq->resize_buf->cqe; } else { tbuf = cq->resize_buf->buf; tcqe = cq->resize_buf->cqe; } kfree(cq->resize_buf); cq->resize_buf = NULL; spin_unlock_irq(&cq->lock); mthca_free_cq_buf(dev, &tbuf, tcqe); } else ibcq->cqe = entries - 1; out: mutex_unlock(&cq->mutex); return ret; } static int mthca_destroy_cq(struct ib_cq *cq) { if (cq->uobject) { mthca_unmap_user_db(to_mdev(cq->device), &to_mucontext(cq->uobject->context)->uar, to_mucontext(cq->uobject->context)->db_tab, to_mcq(cq)->arm_db_index); mthca_unmap_user_db(to_mdev(cq->device), &to_mucontext(cq->uobject->context)->uar, to_mucontext(cq->uobject->context)->db_tab, to_mcq(cq)->set_ci_db_index); } mthca_free_cq(to_mdev(cq->device), to_mcq(cq)); kfree(cq); return 0; } static inline u32 convert_access(int acc) { return (acc & IB_ACCESS_REMOTE_ATOMIC ? MTHCA_MPT_FLAG_ATOMIC : 0) | (acc & IB_ACCESS_REMOTE_WRITE ? MTHCA_MPT_FLAG_REMOTE_WRITE : 0) | (acc & IB_ACCESS_REMOTE_READ ? MTHCA_MPT_FLAG_REMOTE_READ : 0) | (acc & IB_ACCESS_LOCAL_WRITE ? MTHCA_MPT_FLAG_LOCAL_WRITE : 0) | MTHCA_MPT_FLAG_LOCAL_READ; } static struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, int acc) { struct mthca_mr *mr; int err; mr = kmalloc(sizeof *mr, GFP_KERNEL); if (!mr) return ERR_PTR(-ENOMEM); err = mthca_mr_alloc_notrans(to_mdev(pd->device), to_mpd(pd)->pd_num, convert_access(acc), mr); if (err) { kfree(mr); return ERR_PTR(err); } mr->umem = NULL; return &mr->ibmr; } static struct ib_mr *mthca_reg_phys_mr(struct ib_pd *pd, struct ib_phys_buf *buffer_list, int num_phys_buf, int acc, u64 *iova_start) { struct mthca_mr *mr; u64 *page_list; u64 total_size; unsigned long mask; int shift; int npages; int err; int i, j, n; mask = buffer_list[0].addr ^ *iova_start; total_size = 0; for (i = 0; i < num_phys_buf; ++i) { if (i != 0) mask |= buffer_list[i].addr; if (i != num_phys_buf - 1) mask |= buffer_list[i].addr + buffer_list[i].size; total_size += buffer_list[i].size; } if (mask & ~PAGE_MASK) return ERR_PTR(-EINVAL); shift = __ffs(mask | 1 << 31); buffer_list[0].size += buffer_list[0].addr & ((1ULL << shift) - 1); buffer_list[0].addr &= ~0ull << shift; mr = kmalloc(sizeof *mr, GFP_KERNEL); if (!mr) return ERR_PTR(-ENOMEM); npages = 0; for (i = 0; i < num_phys_buf; ++i) npages += (buffer_list[i].size + (1ULL << shift) - 1) >> shift; if (!npages) return &mr->ibmr; page_list = kmalloc(npages * sizeof *page_list, GFP_KERNEL); if (!page_list) { kfree(mr); return ERR_PTR(-ENOMEM); } n = 0; for (i = 0; i < num_phys_buf; ++i) for (j = 0; j < (buffer_list[i].size + (1ULL << shift) - 1) >> shift; ++j) page_list[n++] = buffer_list[i].addr + ((u64) j << shift); mthca_dbg(to_mdev(pd->device), "Registering memory at %llx (iova %llx) " "in PD %x; shift %d, npages %d.\n", (unsigned long long) buffer_list[0].addr, (unsigned long long) *iova_start, to_mpd(pd)->pd_num, shift, npages); err = mthca_mr_alloc_phys(to_mdev(pd->device), to_mpd(pd)->pd_num, page_list, shift, npages, *iova_start, total_size, convert_access(acc), mr); if (err) { kfree(page_list); kfree(mr); return ERR_PTR(err); } kfree(page_list); mr->umem = NULL; return &mr->ibmr; } static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, - u64 virt, int acc, struct ib_udata *udata) + u64 virt, int acc, struct ib_udata *udata, int mr_id) { struct mthca_dev *dev = to_mdev(pd->device); struct ib_umem_chunk *chunk; struct mthca_mr *mr; struct mthca_reg_mr ucmd; u64 *pages; int shift, n, len; int i, j, k; int err = 0; int write_mtt_size; if (udata->inlen - sizeof (struct ib_uverbs_cmd_hdr) < sizeof ucmd) { if (!to_mucontext(pd->uobject->context)->reg_mr_warned) { mthca_warn(dev, "Process '%s' did not pass in MR attrs.\n", curproc->p_comm); mthca_warn(dev, " Update libmthca to fix this.\n"); } ++to_mucontext(pd->uobject->context)->reg_mr_warned; ucmd.mr_attrs = 0; } else if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) return ERR_PTR(-EFAULT); mr = kmalloc(sizeof *mr, GFP_KERNEL); if (!mr) return ERR_PTR(-ENOMEM); mr->umem = ib_umem_get(pd->uobject->context, start, length, acc, ucmd.mr_attrs & MTHCA_MR_DMASYNC); if (IS_ERR(mr->umem)) { err = PTR_ERR(mr->umem); goto err; } shift = ffs(mr->umem->page_size) - 1; n = 0; list_for_each_entry(chunk, &mr->umem->chunk_list, list) n += chunk->nents; mr->mtt = mthca_alloc_mtt(dev, n); if (IS_ERR(mr->mtt)) { err = PTR_ERR(mr->mtt); goto err_umem; } pages = (u64 *) __get_free_page(GFP_KERNEL); if (!pages) { err = -ENOMEM; goto err_mtt; } i = n = 0; write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages)); list_for_each_entry(chunk, &mr->umem->chunk_list, list) for (j = 0; j < chunk->nmap; ++j) { len = sg_dma_len(&chunk->page_list[j]) >> shift; for (k = 0; k < len; ++k) { pages[i++] = sg_dma_address(&chunk->page_list[j]) + mr->umem->page_size * k; /* * Be friendly to write_mtt and pass it chunks * of appropriate size. */ if (i == write_mtt_size) { err = mthca_write_mtt(dev, mr->mtt, n, pages, i); if (err) goto mtt_done; n += i; i = 0; } } } if (i) err = mthca_write_mtt(dev, mr->mtt, n, pages, i); mtt_done: free_page((unsigned long) pages); if (err) goto err_mtt; err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, shift, virt, length, convert_access(acc), mr); if (err) goto err_mtt; return &mr->ibmr; err_mtt: mthca_free_mtt(dev, mr->mtt); err_umem: ib_umem_release(mr->umem); err: kfree(mr); return ERR_PTR(err); } static int mthca_dereg_mr(struct ib_mr *mr) { struct mthca_mr *mmr = to_mmr(mr); mthca_free_mr(to_mdev(mr->device), mmr); if (mmr->umem) ib_umem_release(mmr->umem); kfree(mmr); return 0; } static struct ib_fmr *mthca_alloc_fmr(struct ib_pd *pd, int mr_access_flags, struct ib_fmr_attr *fmr_attr) { struct mthca_fmr *fmr; int err; fmr = kmalloc(sizeof *fmr, GFP_KERNEL); if (!fmr) return ERR_PTR(-ENOMEM); memcpy(&fmr->attr, fmr_attr, sizeof *fmr_attr); err = mthca_fmr_alloc(to_mdev(pd->device), to_mpd(pd)->pd_num, convert_access(mr_access_flags), fmr); if (err) { kfree(fmr); return ERR_PTR(err); } return &fmr->ibmr; } static int mthca_dealloc_fmr(struct ib_fmr *fmr) { struct mthca_fmr *mfmr = to_mfmr(fmr); int err; err = mthca_free_fmr(to_mdev(fmr->device), mfmr); if (err) return err; kfree(mfmr); return 0; } static int mthca_unmap_fmr(struct list_head *fmr_list) { struct ib_fmr *fmr; int err; u8 status; struct mthca_dev *mdev = NULL; list_for_each_entry(fmr, fmr_list, list) { if (mdev && to_mdev(fmr->device) != mdev) return -EINVAL; mdev = to_mdev(fmr->device); } if (!mdev) return 0; if (mthca_is_memfree(mdev)) { list_for_each_entry(fmr, fmr_list, list) mthca_arbel_fmr_unmap(mdev, to_mfmr(fmr)); wmb(); } else list_for_each_entry(fmr, fmr_list, list) mthca_tavor_fmr_unmap(mdev, to_mfmr(fmr)); err = mthca_SYNC_TPT(mdev, &status); if (err) return err; if (status) return -EINVAL; return 0; } static ssize_t show_rev(struct device *device, struct device_attribute *attr, char *buf) { struct mthca_dev *dev = container_of(device, struct mthca_dev, ib_dev.dev); return sprintf(buf, "%x\n", dev->rev_id); } static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, char *buf) { struct mthca_dev *dev = container_of(device, struct mthca_dev, ib_dev.dev); return sprintf(buf, "%d.%d.%d\n", (int) (dev->fw_ver >> 32), (int) (dev->fw_ver >> 16) & 0xffff, (int) dev->fw_ver & 0xffff); } static ssize_t show_hca(struct device *device, struct device_attribute *attr, char *buf) { struct mthca_dev *dev = container_of(device, struct mthca_dev, ib_dev.dev); switch (dev->pdev->device) { case PCI_DEVICE_ID_MELLANOX_TAVOR: return sprintf(buf, "MT23108\n"); case PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT: return sprintf(buf, "MT25208 (MT23108 compat mode)\n"); case PCI_DEVICE_ID_MELLANOX_ARBEL: return sprintf(buf, "MT25208\n"); case PCI_DEVICE_ID_MELLANOX_SINAI: case PCI_DEVICE_ID_MELLANOX_SINAI_OLD: return sprintf(buf, "MT25204\n"); default: return sprintf(buf, "unknown\n"); } } static ssize_t show_board(struct device *device, struct device_attribute *attr, char *buf) { struct mthca_dev *dev = container_of(device, struct mthca_dev, ib_dev.dev); return sprintf(buf, "%.*s\n", MTHCA_BOARD_ID_LEN, dev->board_id); } static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL); static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL); static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL); static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL); static struct device_attribute *mthca_dev_attributes[] = { &dev_attr_hw_rev, &dev_attr_fw_ver, &dev_attr_hca_type, &dev_attr_board_id }; static int mthca_init_node_data(struct mthca_dev *dev) { struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; int err = -ENOMEM; u8 status; in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); if (!in_mad || !out_mad) goto out; init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_NODE_DESC; err = mthca_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad, &status); if (err) goto out; if (status) { err = -EINVAL; goto out; } memcpy(dev->ib_dev.node_desc, out_mad->data, 64); in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; err = mthca_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad, &status); if (err) goto out; if (status) { err = -EINVAL; goto out; } if (mthca_is_memfree(dev)) dev->rev_id = be32_to_cpup((__be32 *) (out_mad->data + 32)); memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8); out: kfree(in_mad); kfree(out_mad); return err; } int mthca_register_device(struct mthca_dev *dev) { int ret; int i; ret = mthca_init_node_data(dev); if (ret) return ret; strlcpy(dev->ib_dev.name, "mthca%d", IB_DEVICE_NAME_MAX); dev->ib_dev.owner = THIS_MODULE; dev->ib_dev.uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION; dev->ib_dev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | (1ull << IB_USER_VERBS_CMD_REG_MR) | (1ull << IB_USER_VERBS_CMD_DEREG_MR) | (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | (1ull << IB_USER_VERBS_CMD_CREATE_QP) | (1ull << IB_USER_VERBS_CMD_QUERY_QP) | (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) | (1ull << IB_USER_VERBS_CMD_DETACH_MCAST); dev->ib_dev.node_type = RDMA_NODE_IB_CA; dev->ib_dev.phys_port_cnt = dev->limits.num_ports; dev->ib_dev.num_comp_vectors = 1; dev->ib_dev.dma_device = &dev->pdev->dev; dev->ib_dev.query_device = mthca_query_device; dev->ib_dev.query_port = mthca_query_port; dev->ib_dev.modify_device = mthca_modify_device; dev->ib_dev.modify_port = mthca_modify_port; dev->ib_dev.query_pkey = mthca_query_pkey; dev->ib_dev.query_gid = mthca_query_gid; dev->ib_dev.alloc_ucontext = mthca_alloc_ucontext; dev->ib_dev.dealloc_ucontext = mthca_dealloc_ucontext; dev->ib_dev.mmap = mthca_mmap_uar; dev->ib_dev.alloc_pd = mthca_alloc_pd; dev->ib_dev.dealloc_pd = mthca_dealloc_pd; dev->ib_dev.create_ah = mthca_ah_create; dev->ib_dev.query_ah = mthca_ah_query; dev->ib_dev.destroy_ah = mthca_ah_destroy; if (dev->mthca_flags & MTHCA_FLAG_SRQ) { dev->ib_dev.create_srq = mthca_create_srq; dev->ib_dev.modify_srq = mthca_modify_srq; dev->ib_dev.query_srq = mthca_query_srq; dev->ib_dev.destroy_srq = mthca_destroy_srq; dev->ib_dev.uverbs_cmd_mask |= (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ); if (mthca_is_memfree(dev)) dev->ib_dev.post_srq_recv = mthca_arbel_post_srq_recv; else dev->ib_dev.post_srq_recv = mthca_tavor_post_srq_recv; } dev->ib_dev.create_qp = mthca_create_qp; dev->ib_dev.modify_qp = mthca_modify_qp; dev->ib_dev.query_qp = mthca_query_qp; dev->ib_dev.destroy_qp = mthca_destroy_qp; dev->ib_dev.create_cq = mthca_create_cq; dev->ib_dev.resize_cq = mthca_resize_cq; dev->ib_dev.destroy_cq = mthca_destroy_cq; dev->ib_dev.poll_cq = mthca_poll_cq; dev->ib_dev.get_dma_mr = mthca_get_dma_mr; dev->ib_dev.reg_phys_mr = mthca_reg_phys_mr; dev->ib_dev.reg_user_mr = mthca_reg_user_mr; dev->ib_dev.dereg_mr = mthca_dereg_mr; if (dev->mthca_flags & MTHCA_FLAG_FMR) { dev->ib_dev.alloc_fmr = mthca_alloc_fmr; dev->ib_dev.unmap_fmr = mthca_unmap_fmr; dev->ib_dev.dealloc_fmr = mthca_dealloc_fmr; if (mthca_is_memfree(dev)) dev->ib_dev.map_phys_fmr = mthca_arbel_map_phys_fmr; else dev->ib_dev.map_phys_fmr = mthca_tavor_map_phys_fmr; } dev->ib_dev.attach_mcast = mthca_multicast_attach; dev->ib_dev.detach_mcast = mthca_multicast_detach; dev->ib_dev.process_mad = mthca_process_mad; if (mthca_is_memfree(dev)) { dev->ib_dev.req_notify_cq = mthca_arbel_arm_cq; dev->ib_dev.post_send = mthca_arbel_post_send; dev->ib_dev.post_recv = mthca_arbel_post_receive; } else { dev->ib_dev.req_notify_cq = mthca_tavor_arm_cq; dev->ib_dev.post_send = mthca_tavor_post_send; dev->ib_dev.post_recv = mthca_tavor_post_receive; } mutex_init(&dev->cap_mask_mutex); - ret = ib_register_device(&dev->ib_dev); + ret = ib_register_device(&dev->ib_dev, NULL); if (ret) return ret; for (i = 0; i < ARRAY_SIZE(mthca_dev_attributes); ++i) { ret = device_create_file(&dev->ib_dev.dev, mthca_dev_attributes[i]); if (ret) { ib_unregister_device(&dev->ib_dev); return ret; } } mthca_start_catas_poll(dev); return 0; } void mthca_unregister_device(struct mthca_dev *dev) { mthca_stop_catas_poll(dev); ib_unregister_device(&dev->ib_dev); } Index: stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_reset.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_reset.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/hw/mthca/mthca_reset.c (revision 273864) @@ -1,298 +1,297 @@ /* * Copyright (c) 2004 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include #include #include #include #include #include "mthca_dev.h" #include "mthca_cmd.h" int mthca_reset(struct mthca_dev *mdev) { int i; int err = 0; u32 *hca_header = NULL; u32 *bridge_header = NULL; struct pci_dev *bridge = NULL; int bridge_pcix_cap = 0; int hca_pcie_cap = 0; int hca_pcix_cap = 0; u16 devctl; u16 linkctl; #define MTHCA_RESET_OFFSET 0xf0010 #define MTHCA_RESET_VALUE swab32(1) /* * Reset the chip. This is somewhat ugly because we have to * save off the PCI header before reset and then restore it * after the chip reboots. We skip config space offsets 22 * and 23 since those have a special meaning. * * To make matters worse, for Tavor (PCI-X HCA) we have to * find the associated bridge device and save off its PCI * header as well. */ if (!(mdev->mthca_flags & MTHCA_FLAG_PCIE)) { /* Look for the bridge -- its device ID will be 2 more than HCA's device ID. */ #ifdef __linux__ while ((bridge = pci_get_device(mdev->pdev->vendor, mdev->pdev->device + 2, bridge)) != NULL) { if (bridge->hdr_type == PCI_HEADER_TYPE_BRIDGE && bridge->subordinate == mdev->pdev->bus) { mthca_dbg(mdev, "Found bridge: %s\n", pci_name(bridge)); break; } } if (!bridge) { /* * Didn't find a bridge for a Tavor device -- * assume we're in no-bridge mode and hope for * the best. */ mthca_warn(mdev, "No bridge found for %s\n", pci_name(mdev->pdev)); } #else mthca_warn(mdev, "Reset on PCI-X is not supported.\n"); goto out; #endif } /* For Arbel do we need to save off the full 4K PCI Express header?? */ hca_header = kmalloc(256, GFP_KERNEL); if (!hca_header) { err = -ENOMEM; mthca_err(mdev, "Couldn't allocate memory to save HCA " "PCI header, aborting.\n"); goto out; } for (i = 0; i < 64; ++i) { if (i == 22 || i == 23) continue; if (pci_read_config_dword(mdev->pdev, i * 4, hca_header + i)) { err = -ENODEV; mthca_err(mdev, "Couldn't save HCA " "PCI header, aborting.\n"); goto out; } } hca_pcix_cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX); hca_pcie_cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_EXP); #ifdef __linux__ if (bridge) { bridge_header = kmalloc(256, GFP_KERNEL); if (!bridge_header) { err = -ENOMEM; mthca_err(mdev, "Couldn't allocate memory to save HCA " "bridge PCI header, aborting.\n"); goto out; } for (i = 0; i < 64; ++i) { if (i == 22 || i == 23) continue; if (pci_read_config_dword(bridge, i * 4, bridge_header + i)) { err = -ENODEV; mthca_err(mdev, "Couldn't save HCA bridge " "PCI header, aborting.\n"); goto out; } } bridge_pcix_cap = pci_find_capability(bridge, PCI_CAP_ID_PCIX); if (!bridge_pcix_cap) { err = -ENODEV; mthca_err(mdev, "Couldn't locate HCA bridge " "PCI-X capability, aborting.\n"); goto out; } } #endif /* actually hit reset */ { void __iomem *reset = ioremap(pci_resource_start(mdev->pdev, 0) + MTHCA_RESET_OFFSET, 4); if (!reset) { err = -ENOMEM; mthca_err(mdev, "Couldn't map HCA reset register, " "aborting.\n"); goto out; } writel(MTHCA_RESET_VALUE, reset); iounmap(reset); } /* Docs say to wait one second before accessing device */ msleep(1000); /* Now wait for PCI device to start responding again */ { u32 v; int c = 0; for (c = 0; c < 100; ++c) { if (pci_read_config_dword(bridge ? bridge : mdev->pdev, 0, &v)) { err = -ENODEV; mthca_err(mdev, "Couldn't access HCA after reset, " "aborting.\n"); goto out; } if (v != 0xffffffff) goto good; msleep(100); } err = -ENODEV; mthca_err(mdev, "PCI device did not come back after reset, " "aborting.\n"); goto out; } good: /* Now restore the PCI headers */ if (bridge) { if (pci_write_config_dword(bridge, bridge_pcix_cap + 0x8, bridge_header[(bridge_pcix_cap + 0x8) / 4])) { err = -ENODEV; mthca_err(mdev, "Couldn't restore HCA bridge Upstream " "split transaction control, aborting.\n"); goto out; } if (pci_write_config_dword(bridge, bridge_pcix_cap + 0xc, bridge_header[(bridge_pcix_cap + 0xc) / 4])) { err = -ENODEV; mthca_err(mdev, "Couldn't restore HCA bridge Downstream " "split transaction control, aborting.\n"); goto out; } /* * Bridge control register is at 0x3e, so we'll * naturally restore it last in this loop. */ for (i = 0; i < 16; ++i) { if (i * 4 == PCI_COMMAND) continue; if (pci_write_config_dword(bridge, i * 4, bridge_header[i])) { err = -ENODEV; mthca_err(mdev, "Couldn't restore HCA bridge reg %x, " "aborting.\n", i); goto out; } } if (pci_write_config_dword(bridge, PCI_COMMAND, bridge_header[PCI_COMMAND / 4])) { err = -ENODEV; mthca_err(mdev, "Couldn't restore HCA bridge COMMAND, " "aborting.\n"); goto out; } } if (hca_pcix_cap) { if (pci_write_config_dword(mdev->pdev, hca_pcix_cap, hca_header[hca_pcix_cap / 4])) { err = -ENODEV; mthca_err(mdev, "Couldn't restore HCA PCI-X " "command register, aborting.\n"); goto out; } } if (hca_pcie_cap) { devctl = hca_header[(hca_pcie_cap + PCI_EXP_DEVCTL) / 4]; if (pci_write_config_word(mdev->pdev, hca_pcie_cap + PCI_EXP_DEVCTL, devctl)) { err = -ENODEV; mthca_err(mdev, "Couldn't restore HCA PCI Express " "Device Control register, aborting.\n"); goto out; } linkctl = hca_header[(hca_pcie_cap + PCI_EXP_LNKCTL) / 4]; if (pci_write_config_word(mdev->pdev, hca_pcie_cap + PCI_EXP_LNKCTL, linkctl)) { err = -ENODEV; mthca_err(mdev, "Couldn't restore HCA PCI Express " "Link control register, aborting.\n"); goto out; } } for (i = 0; i < 16; ++i) { if (i * 4 == PCI_COMMAND) continue; if (pci_write_config_dword(mdev->pdev, i * 4, hca_header[i])) { err = -ENODEV; mthca_err(mdev, "Couldn't restore HCA reg %x, " "aborting.\n", i); goto out; } } if (pci_write_config_dword(mdev->pdev, PCI_COMMAND, hca_header[PCI_COMMAND / 4])) { err = -ENODEV; mthca_err(mdev, "Couldn't restore HCA COMMAND, " "aborting.\n"); goto out; } out: #ifdef __linux__ if (bridge) pci_dev_put(bridge); #endif kfree(bridge_header); kfree(hca_header); return err; } Index: stable/9/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib.h =================================================================== --- stable/9/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib.h (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib.h (revision 273864) @@ -1,755 +1,756 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2004 Voltaire, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef _IPOIB_H #define _IPOIB_H #include "opt_inet.h" #include "opt_inet6.h" #include "opt_ofed.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(INET) || defined(INET6) #include #include #include #include #endif #ifdef INET6 #include #endif #include #include #include #include #include #include #include #include #include /* constants */ #define INFINIBAND_ALEN 20 /* Octets in IPoIB HW addr */ #ifdef IPOIB_CM #define CONFIG_INFINIBAND_IPOIB_CM #endif #ifdef IPOIB_DEBUG #define CONFIG_INFINIBAND_IPOIB_DEBUG #define CONFIG_INFINIBAND_IPOIB_DEBUG_DATA #endif enum ipoib_flush_level { IPOIB_FLUSH_LIGHT, IPOIB_FLUSH_NORMAL, IPOIB_FLUSH_HEAVY }; enum { IPOIB_ENCAP_LEN = 4, IPOIB_HEADER_LEN = IPOIB_ENCAP_LEN + INFINIBAND_ALEN, IPOIB_UD_MAX_MTU = 4 * 1024, - IPOIB_UD_RX_SG = (IPOIB_UD_MAX_MTU / MJUMPAGESIZE), +// IPOIB_UD_RX_SG = (IPOIB_UD_MAX_MTU / MJUMPAGESIZE), + IPOIB_UD_RX_SG = 2, IPOIB_UD_TX_SG = (IPOIB_UD_MAX_MTU / MCLBYTES) + 2, IPOIB_CM_MAX_MTU = (64 * 1024), IPOIB_CM_TX_SG = (IPOIB_CM_MAX_MTU / MCLBYTES) + 2, IPOIB_CM_RX_SG = (IPOIB_CM_MAX_MTU / MJUMPAGESIZE), IPOIB_RX_RING_SIZE = 256, IPOIB_TX_RING_SIZE = 128, IPOIB_MAX_RX_SG = MAX(IPOIB_CM_RX_SG, IPOIB_UD_RX_SG), IPOIB_MAX_TX_SG = MAX(IPOIB_CM_TX_SG, IPOIB_UD_TX_SG), IPOIB_MAX_QUEUE_SIZE = 8192, IPOIB_MIN_QUEUE_SIZE = 2, IPOIB_CM_MAX_CONN_QP = 4096, IPOIB_NUM_WC = 4, IPOIB_MAX_PATH_REC_QUEUE = 3, IPOIB_MAX_MCAST_QUEUE = 3, IPOIB_FLAG_OPER_UP = 0, IPOIB_FLAG_INITIALIZED = 1, IPOIB_FLAG_ADMIN_UP = 2, IPOIB_PKEY_ASSIGNED = 3, IPOIB_PKEY_STOP = 4, IPOIB_FLAG_SUBINTERFACE = 5, IPOIB_MCAST_RUN = 6, IPOIB_STOP_REAPER = 7, IPOIB_FLAG_UMCAST = 10, IPOIB_FLAG_CSUM = 11, IPOIB_MAX_BACKOFF_SECONDS = 16, IPOIB_MCAST_FLAG_FOUND = 0, /* used in set_multicast_list */ IPOIB_MCAST_FLAG_SENDONLY = 1, IPOIB_MCAST_FLAG_BUSY = 2, /* joining or already joined */ IPOIB_MCAST_FLAG_ATTACHED = 3, IPOIB_MAX_LRO_DESCRIPTORS = 8, IPOIB_LRO_MAX_AGGR = 64, MAX_SEND_CQE = 16, IPOIB_CM_COPYBREAK = 256, }; #define IPOIB_OP_RECV (1ul << 31) #ifdef CONFIG_INFINIBAND_IPOIB_CM #define IPOIB_OP_CM (1ul << 30) #else #define IPOIB_OP_CM (0) #endif /* structs */ struct ipoib_header { u8 hwaddr[INFINIBAND_ALEN]; __be16 proto; u16 reserved; }; struct ipoib_pseudoheader { u8 hwaddr[INFINIBAND_ALEN]; }; /* Used for all multicast joins (broadcast, IPv4 mcast and IPv6 mcast) */ struct ipoib_mcast { struct ib_sa_mcmember_rec mcmember; struct ib_sa_multicast *mc; struct ipoib_ah *ah; struct rb_node rb_node; struct list_head list; unsigned long created; unsigned long backoff; unsigned long flags; unsigned char logcount; struct ifqueue pkt_queue; struct ipoib_dev_priv *priv; }; struct ipoib_cm_rx_buf { struct mbuf *mb; u64 mapping[IPOIB_CM_RX_SG]; }; struct ipoib_cm_tx_buf { struct mbuf *mb; u64 mapping[IPOIB_CM_TX_SG]; }; struct ipoib_rx_buf { struct mbuf *mb; u64 mapping[IPOIB_UD_RX_SG]; }; struct ipoib_tx_buf { struct mbuf *mb; u64 mapping[IPOIB_UD_TX_SG]; }; struct ib_cm_id; struct ipoib_cm_data { __be32 qpn; /* High byte MUST be ignored on receive */ __be32 mtu; }; /* * Quoting 10.3.1 Queue Pair and EE Context States: * * Note, for QPs that are associated with an SRQ, the Consumer should take the * QP through the Error State before invoking a Destroy QP or a Modify QP to the * Reset State. The Consumer may invoke the Destroy QP without first performing * a Modify QP to the Error State and waiting for the Affiliated Asynchronous * Last WQE Reached Event. However, if the Consumer does not wait for the * Affiliated Asynchronous Last WQE Reached Event, then WQE and Data Segment * leakage may occur. Therefore, it is good programming practice to tear down a * QP that is associated with an SRQ by using the following process: * * - Put the QP in the Error State * - Wait for the Affiliated Asynchronous Last WQE Reached Event; * - either: * drain the CQ by invoking the Poll CQ verb and either wait for CQ * to be empty or the number of Poll CQ operations has exceeded * CQ capacity size; * - or * post another WR that completes on the same CQ and wait for this * WR to return as a WC; * - and then invoke a Destroy QP or Reset QP. * * We use the second option and wait for a completion on the * same CQ before destroying QPs attached to our SRQ. */ enum ipoib_cm_state { IPOIB_CM_RX_LIVE, IPOIB_CM_RX_ERROR, /* Ignored by stale task */ IPOIB_CM_RX_FLUSH /* Last WQE Reached event observed */ }; struct ipoib_cm_rx { struct ib_cm_id *id; struct ib_qp *qp; struct ipoib_cm_rx_buf *rx_ring; struct list_head list; struct ipoib_dev_priv *priv; unsigned long jiffies; enum ipoib_cm_state state; int recv_count; }; struct ipoib_cm_tx { struct ib_cm_id *id; struct ib_qp *qp; struct list_head list; struct ipoib_dev_priv *priv; struct ipoib_path *path; struct ipoib_cm_tx_buf *tx_ring; unsigned tx_head; unsigned tx_tail; unsigned long flags; u32 mtu; /* remote specified mtu, with grh. */ }; struct ipoib_cm_dev_priv { struct ib_srq *srq; struct ipoib_cm_rx_buf *srq_ring; struct ib_cm_id *id; struct list_head passive_ids; /* state: LIVE */ struct list_head rx_error_list; /* state: ERROR */ struct list_head rx_flush_list; /* state: FLUSH, drain not started */ struct list_head rx_drain_list; /* state: FLUSH, drain started */ struct list_head rx_reap_list; /* state: FLUSH, drain done */ struct work_struct start_task; struct work_struct reap_task; struct work_struct mb_task; struct work_struct rx_reap_task; struct delayed_work stale_task; struct ifqueue mb_queue; struct list_head start_list; struct list_head reap_list; struct ib_sge rx_sge[IPOIB_CM_RX_SG]; struct ib_recv_wr rx_wr; int nonsrq_conn_qp; int max_cm_mtu; /* Actual buf size. */ int num_frags; }; struct ipoib_ethtool_st { u16 coalesce_usecs; u16 max_coalesced_frames; }; /* * Device private locking: network stack tx_lock protects members used * in TX fast path, lock protects everything else. lock nests inside * of tx_lock (ie tx_lock must be acquired first if needed). */ struct ipoib_dev_priv { spinlock_t lock; struct ifnet *dev; u8 broadcastaddr[INFINIBAND_ALEN]; unsigned long flags; struct mutex vlan_mutex; struct rb_root path_tree; struct list_head path_list; struct ipoib_mcast *broadcast; struct list_head multicast_list; struct rb_root multicast_tree; struct delayed_work pkey_poll_task; struct delayed_work mcast_task; struct work_struct carrier_on_task; struct work_struct flush_light; struct work_struct flush_normal; struct work_struct flush_heavy; struct work_struct restart_task; struct delayed_work ah_reap_task; struct ib_device *ca; u8 port; u16 pkey; u16 pkey_index; struct ib_pd *pd; struct ib_mr *mr; struct ib_cq *recv_cq; struct ib_cq *send_cq; struct ib_qp *qp; u32 qkey; union ib_gid local_gid; u16 local_lid; unsigned int admin_mtu; /* User selected MTU, no GRH. */ unsigned int mcast_mtu; /* Minus GRH bytes, from mcast group. */ unsigned int max_ib_mtu; /* Without header, actual buf size. */ struct ipoib_rx_buf *rx_ring; struct ipoib_tx_buf *tx_ring; unsigned tx_head; unsigned tx_tail; struct ib_sge tx_sge[IPOIB_MAX_TX_SG]; struct ib_send_wr tx_wr; unsigned tx_outstanding; struct ib_wc send_wc[MAX_SEND_CQE]; struct ib_recv_wr rx_wr; struct ib_sge rx_sge[IPOIB_MAX_RX_SG]; struct ib_wc ibwc[IPOIB_NUM_WC]; struct list_head dead_ahs; struct ib_event_handler event_handler; struct ifnet *parent; struct list_head child_intfs; struct list_head list; #ifdef CONFIG_INFINIBAND_IPOIB_CM struct ipoib_cm_dev_priv cm; #endif #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG struct list_head fs_list; struct dentry *mcg_dentry; struct dentry *path_dentry; #endif int hca_caps; struct ipoib_ethtool_st ethtool; struct timer_list poll_timer; }; struct ipoib_ah { struct ipoib_dev_priv *priv; struct ib_ah *ah; struct list_head list; struct kref ref; unsigned last_send; }; struct ipoib_path { struct ipoib_dev_priv *priv; struct rb_node rb_node; struct list_head list; #ifdef CONFIG_INFINIBAND_IPOIB_CM uint8_t hwaddr[INFINIBAND_ALEN]; struct ipoib_cm_tx *cm; #endif struct ipoib_ah *ah; struct ib_sa_path_rec pathrec; struct ifqueue queue; int query_id; struct ib_sa_query *query; struct completion done; int valid; }; /* UD Only transmits encap len but we want the two sizes to be symmetrical. */ #define IPOIB_UD_MTU(ib_mtu) (ib_mtu - IPOIB_ENCAP_LEN) #define IPOIB_CM_MTU(ib_mtu) (ib_mtu - 0x10) #define IPOIB_IS_MULTICAST(addr) ((addr)[4] == 0xff) extern struct workqueue_struct *ipoib_workqueue; #define IPOIB_MTAP_PROTO(_ifp, _m, _proto) \ do { \ if (bpf_peers_present((_ifp)->if_bpf)) { \ M_ASSERTVALID(_m); \ ipoib_mtap_proto((_ifp), (_m), (_proto)); \ } \ } while (0) /* functions */ void ipoib_mtap_proto(struct ifnet *ifp, struct mbuf *mb, uint16_t proto); void ipoib_ib_completion(struct ib_cq *cq, void *dev_ptr); void ipoib_send_comp_handler(struct ib_cq *cq, void *dev_ptr); struct ipoib_ah *ipoib_create_ah(struct ipoib_dev_priv *, struct ib_pd *pd, struct ib_ah_attr *attr); void ipoib_free_ah(struct kref *kref); static inline void ipoib_put_ah(struct ipoib_ah *ah) { kref_put(&ah->ref, ipoib_free_ah); } int ipoib_open(struct ipoib_dev_priv *priv); int ipoib_add_pkey_attr(struct ipoib_dev_priv *priv); int ipoib_add_umcast_attr(struct ipoib_dev_priv *priv); void ipoib_demux(struct ifnet *ifp, struct mbuf *m, u_short proto); void ipoib_send(struct ipoib_dev_priv *priv, struct mbuf *mb, struct ipoib_ah *address, u32 qpn); void ipoib_reap_ah(struct work_struct *work); void ipoib_mark_paths_invalid(struct ipoib_dev_priv *priv); void ipoib_flush_paths(struct ipoib_dev_priv *priv); struct ipoib_dev_priv *ipoib_intf_alloc(const char *format); int ipoib_ib_dev_init(struct ipoib_dev_priv *priv, struct ib_device *ca, int port); void ipoib_ib_dev_flush_light(struct work_struct *work); void ipoib_ib_dev_flush_normal(struct work_struct *work); void ipoib_ib_dev_flush_heavy(struct work_struct *work); void ipoib_pkey_event(struct work_struct *work); void ipoib_ib_dev_cleanup(struct ipoib_dev_priv *priv); int ipoib_ib_dev_open(struct ipoib_dev_priv *priv); int ipoib_ib_dev_up(struct ipoib_dev_priv *priv); int ipoib_ib_dev_down(struct ipoib_dev_priv *priv, int flush); int ipoib_ib_dev_stop(struct ipoib_dev_priv *priv, int flush); int ipoib_dev_init(struct ipoib_dev_priv *priv, struct ib_device *ca, int port); void ipoib_dev_cleanup(struct ipoib_dev_priv *priv); void ipoib_mcast_join_task(struct work_struct *work); void ipoib_mcast_carrier_on_task(struct work_struct *work); void ipoib_mcast_send(struct ipoib_dev_priv *priv, void *mgid, struct mbuf *mb); void ipoib_mcast_restart_task(struct work_struct *work); void ipoib_mcast_restart(struct ipoib_dev_priv *); int ipoib_mcast_start_thread(struct ipoib_dev_priv *priv); int ipoib_mcast_stop_thread(struct ipoib_dev_priv *priv, int flush); void ipoib_mcast_dev_down(struct ipoib_dev_priv *priv); void ipoib_mcast_dev_flush(struct ipoib_dev_priv *priv); void ipoib_path_free(struct ipoib_dev_priv *priv, struct ipoib_path *path); #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG struct ipoib_mcast_iter *ipoib_mcast_iter_init(struct ipoib_dev_priv *priv); int ipoib_mcast_iter_next(struct ipoib_mcast_iter *iter); void ipoib_mcast_iter_read(struct ipoib_mcast_iter *iter, union ib_gid *gid, unsigned long *created, unsigned int *queuelen, unsigned int *complete, unsigned int *send_only); struct ipoib_path_iter *ipoib_path_iter_init(struct ipoib_dev_priv *priv); int ipoib_path_iter_next(struct ipoib_path_iter *iter); void ipoib_path_iter_read(struct ipoib_path_iter *iter, struct ipoib_path *path); #endif int ipoib_change_mtu(struct ipoib_dev_priv *priv, int new_mtu); int ipoib_mcast_attach(struct ipoib_dev_priv *priv, u16 mlid, union ib_gid *mgid, int set_qkey); int ipoib_init_qp(struct ipoib_dev_priv *priv); int ipoib_transport_dev_init(struct ipoib_dev_priv *priv, struct ib_device *ca); void ipoib_transport_dev_cleanup(struct ipoib_dev_priv *priv); void ipoib_event(struct ib_event_handler *handler, struct ib_event *record); void ipoib_pkey_poll(struct work_struct *work); int ipoib_pkey_dev_delay_open(struct ipoib_dev_priv *priv); void ipoib_drain_cq(struct ipoib_dev_priv *priv); int ipoib_dma_map_tx(struct ib_device *ca, struct ipoib_tx_buf *tx_req, int max); void ipoib_dma_unmap_tx(struct ib_device *ca, struct ipoib_tx_buf *tx_req); int ipoib_poll_tx(struct ipoib_dev_priv *priv); void ipoib_dma_unmap_rx(struct ipoib_dev_priv *priv, struct ipoib_rx_buf *rx_req); void ipoib_dma_mb(struct ipoib_dev_priv *priv, struct mbuf *mb, unsigned int length); struct mbuf *ipoib_alloc_map_mb(struct ipoib_dev_priv *priv, struct ipoib_rx_buf *rx_req, int size); void ipoib_set_ethtool_ops(struct ifnet *dev); int ipoib_set_dev_features(struct ipoib_dev_priv *priv, struct ib_device *hca); #ifdef CONFIG_INFINIBAND_IPOIB_CM #define IPOIB_FLAGS_RC 0x80 #define IPOIB_FLAGS_UC 0x40 /* We don't support UC connections at the moment */ #define IPOIB_CM_SUPPORTED(ha) (ha[0] & (IPOIB_FLAGS_RC)) extern int ipoib_max_conn_qp; static inline int ipoib_cm_admin_enabled(struct ipoib_dev_priv *priv) { return IPOIB_CM_SUPPORTED(IF_LLADDR(priv->dev)); } static inline int ipoib_cm_enabled(struct ipoib_dev_priv *priv, uint8_t *hwaddr) { return IPOIB_CM_SUPPORTED(hwaddr); } static inline int ipoib_cm_up(struct ipoib_path *path) { return test_bit(IPOIB_FLAG_OPER_UP, &path->cm->flags); } static inline struct ipoib_cm_tx *ipoib_cm_get(struct ipoib_path *path) { return path->cm; } static inline void ipoib_cm_set(struct ipoib_path *path, struct ipoib_cm_tx *tx) { path->cm = tx; } static inline int ipoib_cm_has_srq(struct ipoib_dev_priv *priv) { return !!priv->cm.srq; } static inline unsigned int ipoib_cm_max_mtu(struct ipoib_dev_priv *priv) { return priv->cm.max_cm_mtu; } void ipoib_cm_send(struct ipoib_dev_priv *priv, struct mbuf *mb, struct ipoib_cm_tx *tx); int ipoib_cm_dev_open(struct ipoib_dev_priv *priv); void ipoib_cm_dev_stop(struct ipoib_dev_priv *priv); int ipoib_cm_dev_init(struct ipoib_dev_priv *priv); int ipoib_cm_add_mode_attr(struct ipoib_dev_priv *priv); void ipoib_cm_dev_cleanup(struct ipoib_dev_priv *priv); struct ipoib_cm_tx *ipoib_cm_create_tx(struct ipoib_dev_priv *priv, struct ipoib_path *path); void ipoib_cm_destroy_tx(struct ipoib_cm_tx *tx); void ipoib_cm_mb_too_long(struct ipoib_dev_priv *priv, struct mbuf *mb, unsigned int mtu); void ipoib_cm_handle_rx_wc(struct ipoib_dev_priv *priv, struct ib_wc *wc); void ipoib_cm_handle_tx_wc(struct ipoib_dev_priv *priv, struct ib_wc *wc); #else struct ipoib_cm_tx; #define ipoib_max_conn_qp 0 static inline int ipoib_cm_admin_enabled(struct ipoib_dev_priv *priv) { return 0; } static inline int ipoib_cm_enabled(struct ipoib_dev_priv *priv, uint8_t *hwaddr) { return 0; } static inline int ipoib_cm_up(struct ipoib_path *path) { return 0; } static inline struct ipoib_cm_tx *ipoib_cm_get(struct ipoib_path *path) { return NULL; } static inline void ipoib_cm_set(struct ipoib_path *path, struct ipoib_cm_tx *tx) { } static inline int ipoib_cm_has_srq(struct ipoib_dev_priv *priv) { return 0; } static inline unsigned int ipoib_cm_max_mtu(struct ipoib_dev_priv *priv) { return 0; } static inline void ipoib_cm_send(struct ipoib_dev_priv *priv, struct mbuf *mb, struct ipoib_cm_tx *tx) { return; } static inline int ipoib_cm_dev_open(struct ipoib_dev_priv *priv) { return 0; } static inline void ipoib_cm_dev_stop(struct ipoib_dev_priv *priv) { return; } static inline int ipoib_cm_dev_init(struct ipoib_dev_priv *priv) { return -ENOSYS; } static inline void ipoib_cm_dev_cleanup(struct ipoib_dev_priv *priv) { return; } static inline struct ipoib_cm_tx *ipoib_cm_create_tx(struct ipoib_dev_priv *priv, struct ipoib_path *path) { return NULL; } static inline void ipoib_cm_destroy_tx(struct ipoib_cm_tx *tx) { return; } static inline int ipoib_cm_add_mode_attr(struct ipoib_dev_priv *priv) { return 0; } static inline void ipoib_cm_mb_too_long(struct ipoib_dev_priv *priv, struct mbuf *mb, unsigned int mtu) { m_freem(mb); } static inline void ipoib_cm_handle_rx_wc(struct ipoib_dev_priv *priv, struct ib_wc *wc) { } static inline void ipoib_cm_handle_tx_wc(struct ipoib_dev_priv *priv, struct ib_wc *wc) { } #endif #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG void ipoib_create_debug_files(struct ipoib_dev_priv *priv); void ipoib_delete_debug_files(struct ipoib_dev_priv *priv); int ipoib_register_debugfs(void); void ipoib_unregister_debugfs(void); #else static inline void ipoib_create_debug_files(struct ipoib_dev_priv *priv) { } static inline void ipoib_delete_debug_files(struct ipoib_dev_priv *priv) { } static inline int ipoib_register_debugfs(void) { return 0; } static inline void ipoib_unregister_debugfs(void) { } #endif #define ipoib_printk(level, priv, format, arg...) \ printk(level "%s: " format, if_name(((struct ipoib_dev_priv *) priv)->dev), ## arg) #define ipoib_warn(priv, format, arg...) \ ipoib_printk(KERN_WARNING, priv, format , ## arg) extern int ipoib_sendq_size; extern int ipoib_recvq_size; extern struct ib_sa_client ipoib_sa_client; #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG extern int ipoib_debug_level; #define ipoib_dbg(priv, format, arg...) \ do { \ if (ipoib_debug_level > 0) \ ipoib_printk(KERN_DEBUG, priv, format , ## arg); \ } while (0) #define ipoib_dbg_mcast(priv, format, arg...) \ do { \ if (mcast_debug_level > 0) \ ipoib_printk(KERN_DEBUG, priv, format , ## arg); \ } while (0) #else /* CONFIG_INFINIBAND_IPOIB_DEBUG */ #define ipoib_dbg(priv, format, arg...) \ do { (void) (priv); } while (0) #define ipoib_dbg_mcast(priv, format, arg...) \ do { (void) (priv); } while (0) #endif /* CONFIG_INFINIBAND_IPOIB_DEBUG */ #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG_DATA #define ipoib_dbg_data(priv, format, arg...) \ do { \ if (data_debug_level > 0) \ ipoib_printk(KERN_DEBUG, priv, format , ## arg); \ } while (0) #else /* CONFIG_INFINIBAND_IPOIB_DEBUG_DATA */ #define ipoib_dbg_data(priv, format, arg...) \ do { (void) (priv); } while (0) #endif /* CONFIG_INFINIBAND_IPOIB_DEBUG_DATA */ #define IPOIB_QPN(ha) (be32_to_cpup((__be32 *) ha) & 0xffffff) #endif /* _IPOIB_H */ Index: stable/9/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c =================================================================== --- stable/9/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c (revision 273864) @@ -1,1541 +1,1557 @@ /* * Copyright (c) 2004 Topspin Communications. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2004 Voltaire, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "ipoib.h" static int ipoib_resolvemulti(struct ifnet *, struct sockaddr **, struct sockaddr *); #include -#include #include #include #include #include /* For ARPHRD_xxx */ #include #include #include MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("IP-over-InfiniBand net driver"); MODULE_LICENSE("Dual BSD/GPL"); int ipoib_sendq_size = IPOIB_TX_RING_SIZE; int ipoib_recvq_size = IPOIB_RX_RING_SIZE; module_param_named(send_queue_size, ipoib_sendq_size, int, 0444); MODULE_PARM_DESC(send_queue_size, "Number of descriptors in send queue"); module_param_named(recv_queue_size, ipoib_recvq_size, int, 0444); MODULE_PARM_DESC(recv_queue_size, "Number of descriptors in receive queue"); #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG int ipoib_debug_level = 1; module_param_named(debug_level, ipoib_debug_level, int, 0644); MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0"); #endif struct ipoib_path_iter { struct ipoib_dev_priv *priv; struct ipoib_path path; }; static const u8 ipv4_bcast_addr[] = { 0x00, 0xff, 0xff, 0xff, 0xff, 0x12, 0x40, 0x1b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; struct workqueue_struct *ipoib_workqueue; struct ib_sa_client ipoib_sa_client; static void ipoib_add_one(struct ib_device *device); static void ipoib_remove_one(struct ib_device *device); static void ipoib_start(struct ifnet *dev); static int ipoib_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, struct route *ro); static int ipoib_ioctl(struct ifnet *ifp, u_long command, caddr_t data); static void ipoib_input(struct ifnet *ifp, struct mbuf *m); #define IPOIB_MTAP(_ifp, _m) \ do { \ if (bpf_peers_present((_ifp)->if_bpf)) { \ M_ASSERTVALID(_m); \ ipoib_mtap_mb((_ifp), (_m)); \ } \ } while (0) /* * This is for clients that have an ipoib_header in the mbuf. */ static void ipoib_mtap_mb(struct ifnet *ifp, struct mbuf *mb) { struct ipoib_header *ih; struct ether_header eh; ih = mtod(mb, struct ipoib_header *); eh.ether_type = ih->proto; bcopy(ih->hwaddr, &eh.ether_dhost, ETHER_ADDR_LEN); bzero(&eh.ether_shost, ETHER_ADDR_LEN); mb->m_data += sizeof(struct ipoib_header); mb->m_len -= sizeof(struct ipoib_header); bpf_mtap2(ifp->if_bpf, &eh, sizeof(eh), mb); mb->m_data -= sizeof(struct ipoib_header); mb->m_len += sizeof(struct ipoib_header); } void ipoib_mtap_proto(struct ifnet *ifp, struct mbuf *mb, uint16_t proto) { struct ether_header eh; eh.ether_type = proto; bzero(&eh.ether_shost, ETHER_ADDR_LEN); bzero(&eh.ether_dhost, ETHER_ADDR_LEN); bpf_mtap2(ifp->if_bpf, &eh, sizeof(eh), mb); } static struct ib_client ipoib_client = { .name = "ipoib", .add = ipoib_add_one, .remove = ipoib_remove_one }; int ipoib_open(struct ipoib_dev_priv *priv) { struct ifnet *dev = priv->dev; ipoib_dbg(priv, "bringing up interface\n"); set_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags); if (ipoib_pkey_dev_delay_open(priv)) return 0; if (ipoib_ib_dev_open(priv)) goto err_disable; if (ipoib_ib_dev_up(priv)) goto err_stop; if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) { struct ipoib_dev_priv *cpriv; /* Bring up any child interfaces too */ mutex_lock(&priv->vlan_mutex); list_for_each_entry(cpriv, &priv->child_intfs, list) if ((cpriv->dev->if_drv_flags & IFF_DRV_RUNNING) == 0) ipoib_open(cpriv); mutex_unlock(&priv->vlan_mutex); } dev->if_drv_flags |= IFF_DRV_RUNNING; dev->if_drv_flags &= ~IFF_DRV_OACTIVE; return 0; err_stop: ipoib_ib_dev_stop(priv, 1); err_disable: clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags); return -EINVAL; } static void ipoib_init(void *arg) { struct ifnet *dev; struct ipoib_dev_priv *priv; priv = arg; dev = priv->dev; if ((dev->if_drv_flags & IFF_DRV_RUNNING) == 0) ipoib_open(priv); queue_work(ipoib_workqueue, &priv->flush_light); } static int ipoib_stop(struct ipoib_dev_priv *priv) { struct ifnet *dev = priv->dev; ipoib_dbg(priv, "stopping interface\n"); clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags); dev->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); ipoib_ib_dev_down(priv, 0); ipoib_ib_dev_stop(priv, 0); if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) { struct ipoib_dev_priv *cpriv; /* Bring down any child interfaces too */ mutex_lock(&priv->vlan_mutex); list_for_each_entry(cpriv, &priv->child_intfs, list) if ((cpriv->dev->if_drv_flags & IFF_DRV_RUNNING) != 0) ipoib_stop(cpriv); mutex_unlock(&priv->vlan_mutex); } return 0; } int ipoib_change_mtu(struct ipoib_dev_priv *priv, int new_mtu) { struct ifnet *dev = priv->dev; /* dev->if_mtu > 2K ==> connected mode */ if (ipoib_cm_admin_enabled(priv)) { if (new_mtu > IPOIB_CM_MTU(ipoib_cm_max_mtu(priv))) return -EINVAL; if (new_mtu > priv->mcast_mtu) ipoib_warn(priv, "mtu > %d will cause multicast packet drops.\n", priv->mcast_mtu); dev->if_mtu = new_mtu; return 0; } if (new_mtu > IPOIB_UD_MTU(priv->max_ib_mtu)) return -EINVAL; priv->admin_mtu = new_mtu; dev->if_mtu = min(priv->mcast_mtu, priv->admin_mtu); queue_work(ipoib_workqueue, &priv->flush_light); return 0; } static int ipoib_ioctl(struct ifnet *ifp, u_long command, caddr_t data) { struct ipoib_dev_priv *priv = ifp->if_softc; struct ifaddr *ifa = (struct ifaddr *) data; struct ifreq *ifr = (struct ifreq *) data; int error = 0; switch (command) { case SIOCSIFFLAGS: if (ifp->if_flags & IFF_UP) { if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) error = -ipoib_open(priv); } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) ipoib_stop(priv); break; case SIOCADDMULTI: case SIOCDELMULTI: if (ifp->if_drv_flags & IFF_DRV_RUNNING) queue_work(ipoib_workqueue, &priv->restart_task); break; case SIOCSIFADDR: ifp->if_flags |= IFF_UP; switch (ifa->ifa_addr->sa_family) { #ifdef INET case AF_INET: ifp->if_init(ifp->if_softc); /* before arpwhohas */ arp_ifinit(ifp, ifa); break; #endif default: ifp->if_init(ifp->if_softc); break; } break; case SIOCGIFADDR: { struct sockaddr *sa; sa = (struct sockaddr *) & ifr->ifr_data; bcopy(IF_LLADDR(ifp), (caddr_t) sa->sa_data, INFINIBAND_ALEN); } break; case SIOCSIFMTU: /* * Set the interface MTU. */ error = -ipoib_change_mtu(priv, ifr->ifr_mtu); break; default: error = EINVAL; break; } return (error); } static struct ipoib_path * __path_find(struct ipoib_dev_priv *priv, void *gid) { struct rb_node *n = priv->path_tree.rb_node; struct ipoib_path *path; int ret; while (n) { path = rb_entry(n, struct ipoib_path, rb_node); ret = memcmp(gid, path->pathrec.dgid.raw, sizeof (union ib_gid)); if (ret < 0) n = n->rb_left; else if (ret > 0) n = n->rb_right; else return path; } return NULL; } static int __path_add(struct ipoib_dev_priv *priv, struct ipoib_path *path) { struct rb_node **n = &priv->path_tree.rb_node; struct rb_node *pn = NULL; struct ipoib_path *tpath; int ret; while (*n) { pn = *n; tpath = rb_entry(pn, struct ipoib_path, rb_node); ret = memcmp(path->pathrec.dgid.raw, tpath->pathrec.dgid.raw, sizeof (union ib_gid)); if (ret < 0) n = &pn->rb_left; else if (ret > 0) n = &pn->rb_right; else return -EEXIST; } rb_link_node(&path->rb_node, pn, n); rb_insert_color(&path->rb_node, &priv->path_tree); list_add_tail(&path->list, &priv->path_list); return 0; } void ipoib_path_free(struct ipoib_dev_priv *priv, struct ipoib_path *path) { _IF_DRAIN(&path->queue); if (path->ah) ipoib_put_ah(path->ah); if (ipoib_cm_get(path)) ipoib_cm_destroy_tx(ipoib_cm_get(path)); kfree(path); } #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG struct ipoib_path_iter * ipoib_path_iter_init(struct ipoib_dev_priv *priv) { struct ipoib_path_iter *iter; iter = kmalloc(sizeof *iter, GFP_KERNEL); if (!iter) return NULL; iter->priv = priv; memset(iter->path.pathrec.dgid.raw, 0, 16); if (ipoib_path_iter_next(iter)) { kfree(iter); return NULL; } return iter; } int ipoib_path_iter_next(struct ipoib_path_iter *iter) { struct ipoib_dev_priv *priv = iter->priv; struct rb_node *n; struct ipoib_path *path; int ret = 1; spin_lock_irq(&priv->lock); n = rb_first(&priv->path_tree); while (n) { path = rb_entry(n, struct ipoib_path, rb_node); if (memcmp(iter->path.pathrec.dgid.raw, path->pathrec.dgid.raw, sizeof (union ib_gid)) < 0) { iter->path = *path; ret = 0; break; } n = rb_next(n); } spin_unlock_irq(&priv->lock); return ret; } void ipoib_path_iter_read(struct ipoib_path_iter *iter, struct ipoib_path *path) { *path = iter->path; } #endif /* CONFIG_INFINIBAND_IPOIB_DEBUG */ void ipoib_mark_paths_invalid(struct ipoib_dev_priv *priv) { struct ipoib_path *path, *tp; spin_lock_irq(&priv->lock); list_for_each_entry_safe(path, tp, &priv->path_list, list) { ipoib_dbg(priv, "mark path LID 0x%04x GID %16D invalid\n", be16_to_cpu(path->pathrec.dlid), path->pathrec.dgid.raw, ":"); path->valid = 0; } spin_unlock_irq(&priv->lock); } void ipoib_flush_paths(struct ipoib_dev_priv *priv) { struct ipoib_path *path, *tp; LIST_HEAD(remove_list); unsigned long flags; spin_lock_irqsave(&priv->lock, flags); list_splice_init(&priv->path_list, &remove_list); list_for_each_entry(path, &remove_list, list) rb_erase(&path->rb_node, &priv->path_tree); list_for_each_entry_safe(path, tp, &remove_list, list) { if (path->query) ib_sa_cancel_query(path->query_id, path->query); spin_unlock_irqrestore(&priv->lock, flags); wait_for_completion(&path->done); ipoib_path_free(priv, path); spin_lock_irqsave(&priv->lock, flags); } spin_unlock_irqrestore(&priv->lock, flags); } static void path_rec_completion(int status, struct ib_sa_path_rec *pathrec, void *path_ptr) { struct ipoib_path *path = path_ptr; struct ipoib_dev_priv *priv = path->priv; struct ifnet *dev = priv->dev; struct ipoib_ah *ah = NULL; struct ipoib_ah *old_ah = NULL; struct ifqueue mbqueue; struct mbuf *mb; unsigned long flags; if (!status) ipoib_dbg(priv, "PathRec LID 0x%04x for GID %16D\n", be16_to_cpu(pathrec->dlid), pathrec->dgid.raw, ":"); else ipoib_dbg(priv, "PathRec status %d for GID %16D\n", status, path->pathrec.dgid.raw, ":"); bzero(&mbqueue, sizeof(mbqueue)); if (!status) { struct ib_ah_attr av; if (!ib_init_ah_from_path(priv->ca, priv->port, pathrec, &av)) ah = ipoib_create_ah(priv, priv->pd, &av); } spin_lock_irqsave(&priv->lock, flags); if (ah) { path->pathrec = *pathrec; old_ah = path->ah; path->ah = ah; ipoib_dbg(priv, "created address handle %p for LID 0x%04x, SL %d\n", ah, be16_to_cpu(pathrec->dlid), pathrec->sl); for (;;) { _IF_DEQUEUE(&path->queue, mb); if (mb == NULL) break; _IF_ENQUEUE(&mbqueue, mb); } #ifdef CONFIG_INFINIBAND_IPOIB_CM if (ipoib_cm_enabled(priv, path->hwaddr) && !ipoib_cm_get(path)) ipoib_cm_set(path, ipoib_cm_create_tx(priv, path)); #endif path->valid = 1; } path->query = NULL; complete(&path->done); spin_unlock_irqrestore(&priv->lock, flags); if (old_ah) ipoib_put_ah(old_ah); for (;;) { _IF_DEQUEUE(&mbqueue, mb); if (mb == NULL) break; mb->m_pkthdr.rcvif = dev; if (dev->if_transmit(dev, mb)) ipoib_warn(priv, "dev_queue_xmit failed " "to requeue packet\n"); } } static struct ipoib_path * path_rec_create(struct ipoib_dev_priv *priv, uint8_t *hwaddr) { struct ipoib_path *path; if (!priv->broadcast) return NULL; path = kzalloc(sizeof *path, GFP_ATOMIC); if (!path) return NULL; path->priv = priv; bzero(&path->queue, sizeof(path->queue)); #ifdef CONFIG_INFINIBAND_IPOIB_CM memcpy(&path->hwaddr, hwaddr, INFINIBAND_ALEN); #endif memcpy(path->pathrec.dgid.raw, &hwaddr[4], sizeof (union ib_gid)); path->pathrec.sgid = priv->local_gid; path->pathrec.pkey = cpu_to_be16(priv->pkey); path->pathrec.numb_path = 1; path->pathrec.traffic_class = priv->broadcast->mcmember.traffic_class; return path; } static int path_rec_start(struct ipoib_dev_priv *priv, struct ipoib_path *path) { struct ifnet *dev = priv->dev; ib_sa_comp_mask comp_mask = IB_SA_PATH_REC_MTU_SELECTOR | IB_SA_PATH_REC_MTU; struct ib_sa_path_rec p_rec; p_rec = path->pathrec; p_rec.mtu_selector = IB_SA_GT; switch (roundup_pow_of_two(dev->if_mtu + IPOIB_ENCAP_LEN)) { case 512: p_rec.mtu = IB_MTU_256; break; case 1024: p_rec.mtu = IB_MTU_512; break; case 2048: p_rec.mtu = IB_MTU_1024; break; case 4096: p_rec.mtu = IB_MTU_2048; break; default: /* Wildcard everything */ comp_mask = 0; p_rec.mtu = 0; p_rec.mtu_selector = 0; } ipoib_dbg(priv, "Start path record lookup for %16D MTU > %d\n", p_rec.dgid.raw, ":", comp_mask ? ib_mtu_enum_to_int(p_rec.mtu) : 0); init_completion(&path->done); path->query_id = ib_sa_path_rec_get(&ipoib_sa_client, priv->ca, priv->port, &p_rec, comp_mask | IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID | IB_SA_PATH_REC_NUMB_PATH | IB_SA_PATH_REC_TRAFFIC_CLASS | IB_SA_PATH_REC_PKEY, 1000, GFP_ATOMIC, path_rec_completion, path, &path->query); if (path->query_id < 0) { ipoib_warn(priv, "ib_sa_path_rec_get failed: %d\n", path->query_id); path->query = NULL; complete(&path->done); return path->query_id; } return 0; } static void ipoib_unicast_send(struct mbuf *mb, struct ipoib_dev_priv *priv, struct ipoib_header *eh) { struct ipoib_path *path; path = __path_find(priv, eh->hwaddr + 4); if (!path || !path->valid) { int new_path = 0; if (!path) { path = path_rec_create(priv, eh->hwaddr); new_path = 1; } if (path) { _IF_ENQUEUE(&path->queue, mb); if (!path->query && path_rec_start(priv, path)) { spin_unlock_irqrestore(&priv->lock, flags); if (new_path) ipoib_path_free(priv, path); return; } else __path_add(priv, path); } else { ++priv->dev->if_oerrors; m_freem(mb); } return; } if (ipoib_cm_get(path) && ipoib_cm_up(path)) { ipoib_cm_send(priv, mb, ipoib_cm_get(path)); } else if (path->ah) { ipoib_send(priv, mb, path->ah, IPOIB_QPN(eh->hwaddr)); } else if ((path->query || !path_rec_start(priv, path)) && path->queue.ifq_len < IPOIB_MAX_PATH_REC_QUEUE) { _IF_ENQUEUE(&path->queue, mb); } else { ++priv->dev->if_oerrors; m_freem(mb); } } static int ipoib_send_one(struct ipoib_dev_priv *priv, struct mbuf *mb) { struct ipoib_header *eh; eh = mtod(mb, struct ipoib_header *); if (IPOIB_IS_MULTICAST(eh->hwaddr)) { /* Add in the P_Key for multicast*/ eh->hwaddr[8] = (priv->pkey >> 8) & 0xff; eh->hwaddr[9] = priv->pkey & 0xff; ipoib_mcast_send(priv, eh->hwaddr + 4, mb); } else ipoib_unicast_send(mb, priv, eh); return 0; } static void _ipoib_start(struct ifnet *dev, struct ipoib_dev_priv *priv) { struct mbuf *mb; if ((dev->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return; spin_lock(&priv->lock); while (!IFQ_DRV_IS_EMPTY(&dev->if_snd) && (dev->if_drv_flags & IFF_DRV_OACTIVE) == 0) { IFQ_DRV_DEQUEUE(&dev->if_snd, mb); if (mb == NULL) break; IPOIB_MTAP(dev, mb); ipoib_send_one(priv, mb); } spin_unlock(&priv->lock); } static void ipoib_start(struct ifnet *dev) { _ipoib_start(dev, dev->if_softc); } static void ipoib_vlan_start(struct ifnet *dev) { struct ipoib_dev_priv *priv; struct mbuf *mb; priv = VLAN_COOKIE(dev); if (priv != NULL) return _ipoib_start(dev, priv); while (!IFQ_DRV_IS_EMPTY(&dev->if_snd)) { IFQ_DRV_DEQUEUE(&dev->if_snd, mb); if (mb == NULL) break; m_freem(mb); dev->if_oerrors++; } } int ipoib_dev_init(struct ipoib_dev_priv *priv, struct ib_device *ca, int port) { /* Allocate RX/TX "rings" to hold queued mbs */ priv->rx_ring = kzalloc(ipoib_recvq_size * sizeof *priv->rx_ring, GFP_KERNEL); if (!priv->rx_ring) { printk(KERN_WARNING "%s: failed to allocate RX ring (%d entries)\n", ca->name, ipoib_recvq_size); goto out; } priv->tx_ring = kzalloc(ipoib_sendq_size * sizeof *priv->tx_ring, GFP_KERNEL); if (!priv->tx_ring) { printk(KERN_WARNING "%s: failed to allocate TX ring (%d entries)\n", ca->name, ipoib_sendq_size); goto out_rx_ring_cleanup; } memset(priv->tx_ring, 0, ipoib_sendq_size * sizeof *priv->tx_ring); /* priv->tx_head, tx_tail & tx_outstanding are already 0 */ if (ipoib_ib_dev_init(priv, ca, port)) goto out_tx_ring_cleanup; return 0; out_tx_ring_cleanup: kfree(priv->tx_ring); out_rx_ring_cleanup: kfree(priv->rx_ring); out: return -ENOMEM; } static void ipoib_detach(struct ipoib_dev_priv *priv) { struct ifnet *dev; dev = priv->dev; if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) { bpfdetach(dev); if_detach(dev); if_free(dev); } else VLAN_SETCOOKIE(priv->dev, NULL); free(priv, M_TEMP); } void ipoib_dev_cleanup(struct ipoib_dev_priv *priv) { struct ipoib_dev_priv *cpriv, *tcpriv; /* Delete any child interfaces first */ list_for_each_entry_safe(cpriv, tcpriv, &priv->child_intfs, list) { ipoib_dev_cleanup(cpriv); ipoib_detach(cpriv); } ipoib_ib_dev_cleanup(priv); kfree(priv->rx_ring); kfree(priv->tx_ring); priv->rx_ring = NULL; priv->tx_ring = NULL; } static volatile int ipoib_unit; static struct ipoib_dev_priv * ipoib_priv_alloc(void) { struct ipoib_dev_priv *priv; priv = malloc(sizeof(struct ipoib_dev_priv), M_TEMP, M_ZERO|M_WAITOK); spin_lock_init(&priv->lock); mutex_init(&priv->vlan_mutex); INIT_LIST_HEAD(&priv->path_list); INIT_LIST_HEAD(&priv->child_intfs); INIT_LIST_HEAD(&priv->dead_ahs); INIT_LIST_HEAD(&priv->multicast_list); INIT_DELAYED_WORK(&priv->pkey_poll_task, ipoib_pkey_poll); INIT_DELAYED_WORK(&priv->mcast_task, ipoib_mcast_join_task); INIT_WORK(&priv->carrier_on_task, ipoib_mcast_carrier_on_task); INIT_WORK(&priv->flush_light, ipoib_ib_dev_flush_light); INIT_WORK(&priv->flush_normal, ipoib_ib_dev_flush_normal); INIT_WORK(&priv->flush_heavy, ipoib_ib_dev_flush_heavy); INIT_WORK(&priv->restart_task, ipoib_mcast_restart_task); INIT_DELAYED_WORK(&priv->ah_reap_task, ipoib_reap_ah); memcpy(priv->broadcastaddr, ipv4_bcast_addr, INFINIBAND_ALEN); return (priv); } struct ipoib_dev_priv * ipoib_intf_alloc(const char *name) { struct ipoib_dev_priv *priv; struct sockaddr_dl *sdl; struct ifnet *dev; priv = ipoib_priv_alloc(); dev = priv->dev = if_alloc(IFT_INFINIBAND); if (!dev) { free(priv, M_TEMP); return NULL; } dev->if_softc = priv; if_initname(dev, name, atomic_fetchadd_int(&ipoib_unit, 1)); dev->if_flags = IFF_BROADCAST | IFF_MULTICAST; dev->if_addrlen = INFINIBAND_ALEN; dev->if_hdrlen = IPOIB_HEADER_LEN; if_attach(dev); dev->if_init = ipoib_init; dev->if_ioctl = ipoib_ioctl; dev->if_start = ipoib_start; dev->if_output = ipoib_output; dev->if_input = ipoib_input; dev->if_resolvemulti = ipoib_resolvemulti; dev->if_baudrate = IF_Gbps(10UL); dev->if_broadcastaddr = priv->broadcastaddr; dev->if_snd.ifq_maxlen = ipoib_sendq_size * 2; sdl = (struct sockaddr_dl *)dev->if_addr->ifa_addr; sdl->sdl_type = IFT_INFINIBAND; sdl->sdl_alen = dev->if_addrlen; priv->dev = dev; if_link_state_change(dev, LINK_STATE_DOWN); bpfattach(dev, DLT_EN10MB, ETHER_HDR_LEN); return dev->if_softc; } int ipoib_set_dev_features(struct ipoib_dev_priv *priv, struct ib_device *hca) { struct ib_device_attr *device_attr; int result = -ENOMEM; device_attr = kmalloc(sizeof *device_attr, GFP_KERNEL); if (!device_attr) { printk(KERN_WARNING "%s: allocation of %zu bytes failed\n", hca->name, sizeof *device_attr); return result; } result = ib_query_device(hca, device_attr); if (result) { printk(KERN_WARNING "%s: ib_query_device failed (ret = %d)\n", hca->name, result); kfree(device_attr); return result; } priv->hca_caps = device_attr->device_cap_flags; kfree(device_attr); priv->dev->if_hwassist = 0; priv->dev->if_capabilities = 0; #ifndef CONFIG_INFINIBAND_IPOIB_CM if (priv->hca_caps & IB_DEVICE_UD_IP_CSUM) { set_bit(IPOIB_FLAG_CSUM, &priv->flags); priv->dev->if_hwassist = CSUM_IP | CSUM_TCP | CSUM_UDP; priv->dev->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; } #if 0 if (priv->dev->features & NETIF_F_SG && priv->hca_caps & IB_DEVICE_UD_TSO) { priv->dev->if_capabilities |= IFCAP_TSO4; priv->dev->if_hwassist |= CSUM_TSO; } #endif #endif priv->dev->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_LINKSTATE; priv->dev->if_capenable = priv->dev->if_capabilities; return 0; } static struct ifnet * ipoib_add_port(const char *format, struct ib_device *hca, u8 port) { struct ipoib_dev_priv *priv; struct ib_port_attr attr; int result = -ENOMEM; priv = ipoib_intf_alloc(format); if (!priv) goto alloc_mem_failed; if (!ib_query_port(hca, port, &attr)) priv->max_ib_mtu = ib_mtu_enum_to_int(attr.max_mtu); else { printk(KERN_WARNING "%s: ib_query_port %d failed\n", hca->name, port); goto device_init_failed; } /* MTU will be reset when mcast join happens */ priv->dev->if_mtu = IPOIB_UD_MTU(priv->max_ib_mtu); priv->mcast_mtu = priv->admin_mtu = priv->dev->if_mtu; result = ib_query_pkey(hca, port, 0, &priv->pkey); if (result) { printk(KERN_WARNING "%s: ib_query_pkey port %d failed (ret = %d)\n", hca->name, port, result); goto device_init_failed; } if (ipoib_set_dev_features(priv, hca)) goto device_init_failed; /* * Set the full membership bit, so that we join the right * broadcast group, etc. */ priv->pkey |= 0x8000; priv->broadcastaddr[8] = priv->pkey >> 8; priv->broadcastaddr[9] = priv->pkey & 0xff; result = ib_query_gid(hca, port, 0, &priv->local_gid); if (result) { printk(KERN_WARNING "%s: ib_query_gid port %d failed (ret = %d)\n", hca->name, port, result); goto device_init_failed; } memcpy(IF_LLADDR(priv->dev) + 4, priv->local_gid.raw, sizeof (union ib_gid)); result = ipoib_dev_init(priv, hca, port); if (result < 0) { printk(KERN_WARNING "%s: failed to initialize port %d (ret = %d)\n", hca->name, port, result); goto device_init_failed; } if (ipoib_cm_admin_enabled(priv)) priv->dev->if_mtu = IPOIB_CM_MTU(ipoib_cm_max_mtu(priv)); INIT_IB_EVENT_HANDLER(&priv->event_handler, priv->ca, ipoib_event); result = ib_register_event_handler(&priv->event_handler); if (result < 0) { printk(KERN_WARNING "%s: ib_register_event_handler failed for " "port %d (ret = %d)\n", hca->name, port, result); goto event_failed; } if_printf(priv->dev, "Attached to %s port %d\n", hca->name, port); return priv->dev; event_failed: ipoib_dev_cleanup(priv); device_init_failed: ipoib_detach(priv); alloc_mem_failed: return ERR_PTR(result); } static void ipoib_add_one(struct ib_device *device) { struct list_head *dev_list; struct ifnet *dev; struct ipoib_dev_priv *priv; int s, e, p; if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB) return; dev_list = kmalloc(sizeof *dev_list, GFP_KERNEL); if (!dev_list) return; INIT_LIST_HEAD(dev_list); if (device->node_type == RDMA_NODE_IB_SWITCH) { s = 0; e = 0; } else { s = 1; e = device->phys_port_cnt; } for (p = s; p <= e; ++p) { if (rdma_port_get_link_layer(device, p) != IB_LINK_LAYER_INFINIBAND) continue; dev = ipoib_add_port("ib", device, p); if (!IS_ERR(dev)) { priv = dev->if_softc; list_add_tail(&priv->list, dev_list); } } ib_set_client_data(device, &ipoib_client, dev_list); } static void ipoib_remove_one(struct ib_device *device) { struct ipoib_dev_priv *priv, *tmp; struct list_head *dev_list; if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB) return; dev_list = ib_get_client_data(device, &ipoib_client); list_for_each_entry_safe(priv, tmp, dev_list, list) { if (rdma_port_get_link_layer(device, priv->port) != IB_LINK_LAYER_INFINIBAND) continue; ipoib_stop(priv); ib_unregister_event_handler(&priv->event_handler); /* dev_change_flags(priv->dev, priv->dev->flags & ~IFF_UP); */ flush_workqueue(ipoib_workqueue); ipoib_dev_cleanup(priv); ipoib_detach(priv); } kfree(dev_list); } static void ipoib_config_vlan(void *arg, struct ifnet *ifp, u_int16_t vtag) { struct ipoib_dev_priv *parent; struct ipoib_dev_priv *priv; struct ifnet *dev; uint16_t pkey; int error; if (ifp->if_type != IFT_INFINIBAND) return; dev = VLAN_DEVAT(ifp, vtag); if (dev == NULL) return; priv = NULL; error = 0; parent = ifp->if_softc; /* We only support 15 bits of pkey. */ if (vtag & 0x8000) return; pkey = vtag | 0x8000; /* Set full membership bit. */ if (pkey == parent->pkey) return; /* Check for dups */ mutex_lock(&parent->vlan_mutex); list_for_each_entry(priv, &parent->child_intfs, list) { if (priv->pkey == pkey) { priv = NULL; error = EBUSY; goto out; } } priv = ipoib_priv_alloc(); priv->dev = dev; priv->max_ib_mtu = parent->max_ib_mtu; priv->mcast_mtu = priv->admin_mtu = parent->dev->if_mtu; set_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags); error = ipoib_set_dev_features(priv, parent->ca); if (error) goto out; priv->pkey = pkey; priv->broadcastaddr[8] = pkey >> 8; priv->broadcastaddr[9] = pkey & 0xff; dev->if_broadcastaddr = priv->broadcastaddr; error = ipoib_dev_init(priv, parent->ca, parent->port); if (error) goto out; priv->parent = parent->dev; list_add_tail(&priv->list, &parent->child_intfs); VLAN_SETCOOKIE(dev, priv); dev->if_start = ipoib_vlan_start; dev->if_drv_flags &= ~IFF_DRV_RUNNING; dev->if_hdrlen = IPOIB_HEADER_LEN; if (ifp->if_drv_flags & IFF_DRV_RUNNING) ipoib_open(priv); mutex_unlock(&parent->vlan_mutex); return; out: mutex_unlock(&parent->vlan_mutex); if (priv) free(priv, M_TEMP); if (error) ipoib_warn(parent, "failed to initialize subinterface: device %s, port %d vtag 0x%X", parent->ca->name, parent->port, vtag); return; } static void ipoib_unconfig_vlan(void *arg, struct ifnet *ifp, u_int16_t vtag) { struct ipoib_dev_priv *parent; struct ipoib_dev_priv *priv; struct ifnet *dev; uint16_t pkey; if (ifp->if_type != IFT_INFINIBAND) return; dev = VLAN_DEVAT(ifp, vtag); if (dev) VLAN_SETCOOKIE(dev, NULL); pkey = vtag | 0x8000; parent = ifp->if_softc; mutex_lock(&parent->vlan_mutex); list_for_each_entry(priv, &parent->child_intfs, list) { if (priv->pkey == pkey) { ipoib_dev_cleanup(priv); list_del(&priv->list); break; } } mutex_unlock(&parent->vlan_mutex); } eventhandler_tag ipoib_vlan_attach; eventhandler_tag ipoib_vlan_detach; static int __init ipoib_init_module(void) { int ret; ipoib_recvq_size = roundup_pow_of_two(ipoib_recvq_size); ipoib_recvq_size = min(ipoib_recvq_size, IPOIB_MAX_QUEUE_SIZE); ipoib_recvq_size = max(ipoib_recvq_size, IPOIB_MIN_QUEUE_SIZE); ipoib_sendq_size = roundup_pow_of_two(ipoib_sendq_size); ipoib_sendq_size = min(ipoib_sendq_size, IPOIB_MAX_QUEUE_SIZE); ipoib_sendq_size = max(ipoib_sendq_size, max(2 * MAX_SEND_CQE, IPOIB_MIN_QUEUE_SIZE)); #ifdef CONFIG_INFINIBAND_IPOIB_CM ipoib_max_conn_qp = min(ipoib_max_conn_qp, IPOIB_CM_MAX_CONN_QP); #endif ipoib_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, ipoib_config_vlan, NULL, EVENTHANDLER_PRI_FIRST); ipoib_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, ipoib_unconfig_vlan, NULL, EVENTHANDLER_PRI_FIRST); /* * We create our own workqueue mainly because we want to be * able to flush it when devices are being removed. We can't * use schedule_work()/flush_scheduled_work() because both * unregister_netdev() and linkwatch_event take the rtnl lock, * so flush_scheduled_work() can deadlock during device * removal. */ ipoib_workqueue = create_singlethread_workqueue("ipoib"); if (!ipoib_workqueue) { ret = -ENOMEM; goto err_fs; } ib_sa_register_client(&ipoib_sa_client); ret = ib_register_client(&ipoib_client); if (ret) goto err_sa; return 0; err_sa: ib_sa_unregister_client(&ipoib_sa_client); destroy_workqueue(ipoib_workqueue); err_fs: return ret; } static void __exit ipoib_cleanup_module(void) { EVENTHANDLER_DEREGISTER(vlan_config, ipoib_vlan_attach); EVENTHANDLER_DEREGISTER(vlan_unconfig, ipoib_vlan_detach); ib_unregister_client(&ipoib_client); ib_sa_unregister_client(&ipoib_sa_client); destroy_workqueue(ipoib_workqueue); } /* * Infiniband output routine. */ static int ipoib_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, struct route *ro) { u_char edst[INFINIBAND_ALEN]; struct llentry *lle = NULL; struct rtentry *rt0 = NULL; struct ipoib_header *eh; int error = 0; short type; if (ro != NULL) { if (!(m->m_flags & (M_BCAST | M_MCAST))) lle = ro->ro_lle; rt0 = ro->ro_rt; } #ifdef MAC error = mac_ifnet_check_transmit(ifp, m); if (error) goto bad; #endif M_PROFILE(m); if (ifp->if_flags & IFF_MONITOR) { error = ENETDOWN; goto bad; } if (!((ifp->if_flags & IFF_UP) && (ifp->if_drv_flags & IFF_DRV_RUNNING))) { error = ENETDOWN; goto bad; } switch (dst->sa_family) { #ifdef INET case AF_INET: if (lle != NULL && (lle->la_flags & LLE_VALID)) memcpy(edst, &lle->ll_addr.mac8, sizeof(edst)); else if (m->m_flags & M_MCAST) ip_ib_mc_map(((struct sockaddr_in *)dst)->sin_addr.s_addr, ifp->if_broadcastaddr, edst); else error = arpresolve(ifp, rt0, m, dst, edst, &lle); if (error) return (error == EWOULDBLOCK ? 0 : error); type = htons(ETHERTYPE_IP); break; case AF_ARP: { struct arphdr *ah; ah = mtod(m, struct arphdr *); ah->ar_hrd = htons(ARPHRD_INFINIBAND); switch(ntohs(ah->ar_op)) { case ARPOP_REVREQUEST: case ARPOP_REVREPLY: type = htons(ETHERTYPE_REVARP); break; case ARPOP_REQUEST: case ARPOP_REPLY: default: type = htons(ETHERTYPE_ARP); break; } if (m->m_flags & M_BCAST) bcopy(ifp->if_broadcastaddr, edst, INFINIBAND_ALEN); else bcopy(ar_tha(ah), edst, INFINIBAND_ALEN); } break; #endif #ifdef INET6 case AF_INET6: if (lle != NULL && (lle->la_flags & LLE_VALID)) memcpy(edst, &lle->ll_addr.mac8, sizeof(edst)); else if (m->m_flags & M_MCAST) ipv6_ib_mc_map(&((struct sockaddr_in6 *)dst)->sin6_addr, ifp->if_broadcastaddr, edst); else error = nd6_storelladdr(ifp, m, dst, (u_char *)edst, &lle); if (error) return error; type = htons(ETHERTYPE_IPV6); break; #endif default: if_printf(ifp, "can't handle af%d\n", dst->sa_family); error = EAFNOSUPPORT; goto bad; } /* * Add local net header. If no space in first mbuf, * allocate another. */ M_PREPEND(m, IPOIB_HEADER_LEN, M_DONTWAIT); if (m == NULL) { error = ENOBUFS; goto bad; } eh = mtod(m, struct ipoib_header *); (void)memcpy(&eh->proto, &type, sizeof(eh->proto)); (void)memcpy(&eh->hwaddr, edst, sizeof (edst)); /* * Queue message on interface, update output statistics if * successful, and start output if interface not yet active. */ return ((ifp->if_transmit)(ifp, m)); bad: if (m != NULL) m_freem(m); return (error); } /* * Upper layer processing for a received Infiniband packet. */ void ipoib_demux(struct ifnet *ifp, struct mbuf *m, u_short proto) { int isr; #ifdef MAC /* * Tag the mbuf with an appropriate MAC label before any other * consumers can get to it. */ mac_ifnet_create_mbuf(ifp, m); #endif /* Allow monitor mode to claim this frame, after stats are updated. */ if (ifp->if_flags & IFF_MONITOR) { if_printf(ifp, "discard frame at IFF_MONITOR\n"); m_freem(m); return; } /* * Dispatch frame to upper layer. */ switch (proto) { #ifdef INET case ETHERTYPE_IP: isr = NETISR_IP; break; case ETHERTYPE_ARP: if (ifp->if_flags & IFF_NOARP) { /* Discard packet if ARP is disabled on interface */ m_freem(m); return; } isr = NETISR_ARP; break; #endif #ifdef INET6 case ETHERTYPE_IPV6: isr = NETISR_IPV6; break; #endif default: goto discard; } netisr_dispatch(isr, m); return; discard: m_freem(m); } /* * Process a received Infiniband packet. */ static void ipoib_input(struct ifnet *ifp, struct mbuf *m) { struct ipoib_header *eh; if ((ifp->if_flags & IFF_UP) == 0) { m_freem(m); return; } CURVNET_SET_QUIET(ifp->if_vnet); /* Let BPF have it before we strip the header. */ IPOIB_MTAP(ifp, m); eh = mtod(m, struct ipoib_header *); /* * Reset layer specific mbuf flags to avoid confusing upper layers. * Strip off Infiniband header. */ m->m_flags &= ~M_VLANTAG; m->m_flags &= ~(M_PROTOFLAGS); m_adj(m, IPOIB_HEADER_LEN); if (IPOIB_IS_MULTICAST(eh->hwaddr)) { if (memcmp(eh->hwaddr, ifp->if_broadcastaddr, ifp->if_addrlen) == 0) m->m_flags |= M_BCAST; else m->m_flags |= M_MCAST; ifp->if_imcasts++; } ipoib_demux(ifp, m, ntohs(eh->proto)); CURVNET_RESTORE(); } static int ipoib_resolvemulti(struct ifnet *ifp, struct sockaddr **llsa, struct sockaddr *sa) { struct sockaddr_dl *sdl; #ifdef INET struct sockaddr_in *sin; #endif #ifdef INET6 struct sockaddr_in6 *sin6; #endif u_char *e_addr; switch(sa->sa_family) { case AF_LINK: /* * No mapping needed. Just check that it's a valid MC address. */ sdl = (struct sockaddr_dl *)sa; e_addr = LLADDR(sdl); if (!IPOIB_IS_MULTICAST(e_addr)) return EADDRNOTAVAIL; *llsa = 0; return 0; #ifdef INET case AF_INET: sin = (struct sockaddr_in *)sa; if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) return EADDRNOTAVAIL; sdl = malloc(sizeof *sdl, M_IFMADDR, M_NOWAIT|M_ZERO); if (sdl == NULL) return ENOMEM; sdl->sdl_len = sizeof *sdl; sdl->sdl_family = AF_LINK; sdl->sdl_index = ifp->if_index; sdl->sdl_type = IFT_INFINIBAND; sdl->sdl_alen = INFINIBAND_ALEN; e_addr = LLADDR(sdl); ip_ib_mc_map(sin->sin_addr.s_addr, ifp->if_broadcastaddr, e_addr); *llsa = (struct sockaddr *)sdl; return 0; #endif #ifdef INET6 case AF_INET6: sin6 = (struct sockaddr_in6 *)sa; /* * An IP6 address of 0 means listen to all * of the multicast address used for IP6. * This has no meaning in ipoib. */ if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) return EADDRNOTAVAIL; if (!IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) return EADDRNOTAVAIL; sdl = malloc(sizeof *sdl, M_IFMADDR, M_NOWAIT|M_ZERO); if (sdl == NULL) return (ENOMEM); sdl->sdl_len = sizeof *sdl; sdl->sdl_family = AF_LINK; sdl->sdl_index = ifp->if_index; sdl->sdl_type = IFT_INFINIBAND; sdl->sdl_alen = INFINIBAND_ALEN; e_addr = LLADDR(sdl); ipv6_ib_mc_map(&sin6->sin6_addr, ifp->if_broadcastaddr, e_addr); *llsa = (struct sockaddr *)sdl; return 0; #endif default: return EAFNOSUPPORT; } } module_init(ipoib_init_module); module_exit(ipoib_cleanup_module); + +#undef MODULE_VERSION +#include +static int +ipoib_evhand(module_t mod, int event, void *arg) +{ + return (0); +} + +static moduledata_t ipoib_mod = { + .name = "ipoib", + .evhand = ipoib_evhand, +}; + +DECLARE_MODULE(ipoib, ipoib_mod, SI_SUB_SMP, SI_ORDER_ANY); +MODULE_DEPEND(ipoib, ibcore, 1, 1, 1); + Index: stable/9/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h =================================================================== --- stable/9/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h (revision 273863) +++ stable/9/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h (revision 273864) @@ -1,721 +1,722 @@ #ifndef _SDP_H_ #define _SDP_H_ #include "opt_ddb.h" #include "opt_inet.h" #include "opt_ofed.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SDP_DEBUG #define CONFIG_INFINIBAND_SDP_DEBUG #endif #include "sdp_dbg.h" #undef LIST_HEAD /* From sys/queue.h */ #define LIST_HEAD(name, type) \ struct name { \ struct type *lh_first; /* first element */ \ } /* Interval between sucessive polls in the Tx routine when polling is used instead of interrupts (in per-core Tx rings) - should be power of 2 */ #define SDP_TX_POLL_MODER 16 #define SDP_TX_POLL_TIMEOUT (HZ / 20) #define SDP_NAGLE_TIMEOUT (HZ / 10) #define SDP_SRCAVAIL_CANCEL_TIMEOUT (HZ * 5) #define SDP_SRCAVAIL_ADV_TIMEOUT (1 * HZ) #define SDP_SRCAVAIL_PAYLOAD_LEN 1 #define SDP_RESOLVE_TIMEOUT 1000 #define SDP_ROUTE_TIMEOUT 1000 #define SDP_RETRY_COUNT 5 #define SDP_KEEPALIVE_TIME (120 * 60 * HZ) #define SDP_FIN_WAIT_TIMEOUT (60 * HZ) /* like TCP_FIN_TIMEOUT */ #define SDP_TX_SIZE 0x40 #define SDP_RX_SIZE 0x40 #define SDP_FMR_SIZE (MIN(0x1000, PAGE_SIZE) / sizeof(u64)) #define SDP_FMR_POOL_SIZE 1024 #define SDP_FMR_DIRTY_SIZE ( SDP_FMR_POOL_SIZE / 4 ) #define SDP_MAX_RDMA_READ_LEN (PAGE_SIZE * (SDP_FMR_SIZE - 2)) /* mb inlined data len - rest will be rx'ed into frags */ #define SDP_HEAD_SIZE (sizeof(struct sdp_bsdh)) /* limit tx payload len, if the sink supports bigger buffers than the source * can handle. * or rx fragment size (limited by sge->length size) */ #define SDP_MAX_PACKET (1 << 16) #define SDP_MAX_PAYLOAD (SDP_MAX_PACKET - SDP_HEAD_SIZE) #define SDP_MAX_RECV_SGES (SDP_MAX_PACKET / MCLBYTES) #define SDP_MAX_SEND_SGES (SDP_MAX_PACKET / MCLBYTES) + 2 #define SDP_NUM_WC 4 #define SDP_DEF_ZCOPY_THRESH 64*1024 #define SDP_MIN_ZCOPY_THRESH PAGE_SIZE #define SDP_MAX_ZCOPY_THRESH 1048576 #define SDP_OP_RECV 0x800000000LL #define SDP_OP_SEND 0x400000000LL #define SDP_OP_RDMA 0x200000000LL #define SDP_OP_NOP 0x100000000LL /* how long (in jiffies) to block sender till tx completion*/ #define SDP_BZCOPY_POLL_TIMEOUT (HZ / 10) #define SDP_AUTO_CONF 0xffff #define AUTO_MOD_DELAY (HZ / 4) struct sdp_mb_cb { __u32 seq; /* Starting sequence number */ struct bzcopy_state *bz; struct rx_srcavail_state *rx_sa; struct tx_srcavail_state *tx_sa; }; #define M_PUSH M_PROTO1 /* Do a 'push'. */ #define M_URG M_PROTO2 /* Mark as urgent (oob). */ #define SDP_SKB_CB(__mb) ((struct sdp_mb_cb *)&((__mb)->cb[0])) #define BZCOPY_STATE(mb) (SDP_SKB_CB(mb)->bz) #define RX_SRCAVAIL_STATE(mb) (SDP_SKB_CB(mb)->rx_sa) #define TX_SRCAVAIL_STATE(mb) (SDP_SKB_CB(mb)->tx_sa) #ifndef MIN #define MIN(a, b) (a < b ? a : b) #endif #define ring_head(ring) (atomic_read(&(ring).head)) #define ring_tail(ring) (atomic_read(&(ring).tail)) #define ring_posted(ring) (ring_head(ring) - ring_tail(ring)) #define rx_ring_posted(ssk) ring_posted(ssk->rx_ring) #ifdef SDP_ZCOPY #define tx_ring_posted(ssk) (ring_posted(ssk->tx_ring) + \ (ssk->tx_ring.rdma_inflight ? ssk->tx_ring.rdma_inflight->busy : 0)) #else #define tx_ring_posted(ssk) ring_posted(ssk->tx_ring) #endif extern int sdp_zcopy_thresh; extern int rcvbuf_initial_size; extern struct workqueue_struct *rx_comp_wq; extern struct ib_client sdp_client; enum sdp_mid { SDP_MID_HELLO = 0x0, SDP_MID_HELLO_ACK = 0x1, SDP_MID_DISCONN = 0x2, SDP_MID_ABORT = 0x3, SDP_MID_SENDSM = 0x4, SDP_MID_RDMARDCOMPL = 0x6, SDP_MID_SRCAVAIL_CANCEL = 0x8, SDP_MID_CHRCVBUF = 0xB, SDP_MID_CHRCVBUF_ACK = 0xC, SDP_MID_SINKAVAIL = 0xFD, SDP_MID_SRCAVAIL = 0xFE, SDP_MID_DATA = 0xFF, }; enum sdp_flags { SDP_OOB_PRES = 1 << 0, SDP_OOB_PEND = 1 << 1, }; enum { SDP_MIN_TX_CREDITS = 2 }; enum { SDP_ERR_ERROR = -4, SDP_ERR_FAULT = -3, SDP_NEW_SEG = -2, SDP_DO_WAIT_MEM = -1 }; struct sdp_bsdh { u8 mid; u8 flags; __u16 bufs; __u32 len; __u32 mseq; __u32 mseq_ack; } __attribute__((__packed__)); union cma_ip_addr { struct in6_addr ip6; struct { __u32 pad[3]; __u32 addr; } ip4; } __attribute__((__packed__)); /* TODO: too much? Can I avoid having the src/dst and port here? */ struct sdp_hh { struct sdp_bsdh bsdh; u8 majv_minv; u8 ipv_cap; u8 rsvd1; u8 max_adverts; __u32 desremrcvsz; __u32 localrcvsz; __u16 port; __u16 rsvd2; union cma_ip_addr src_addr; union cma_ip_addr dst_addr; u8 rsvd3[IB_CM_REQ_PRIVATE_DATA_SIZE - sizeof(struct sdp_bsdh) - 48]; } __attribute__((__packed__)); struct sdp_hah { struct sdp_bsdh bsdh; u8 majv_minv; u8 ipv_cap; u8 rsvd1; u8 ext_max_adverts; __u32 actrcvsz; u8 rsvd2[IB_CM_REP_PRIVATE_DATA_SIZE - sizeof(struct sdp_bsdh) - 8]; } __attribute__((__packed__)); struct sdp_rrch { __u32 len; } __attribute__((__packed__)); struct sdp_srcah { __u32 len; __u32 rkey; __u64 vaddr; } __attribute__((__packed__)); struct sdp_buf { struct mbuf *mb; u64 mapping[SDP_MAX_SEND_SGES]; } __attribute__((__packed__)); struct sdp_chrecvbuf { u32 size; } __attribute__((__packed__)); /* Context used for synchronous zero copy bcopy (BZCOPY) */ struct bzcopy_state { unsigned char __user *u_base; int u_len; int left; int page_cnt; int cur_page; int cur_offset; int busy; struct sdp_sock *ssk; struct page **pages; }; enum rx_sa_flag { RX_SA_ABORTED = 2, }; enum tx_sa_flag { TX_SA_SENDSM = 0x01, TX_SA_CROSS_SEND = 0x02, TX_SA_INTRRUPTED = 0x04, TX_SA_TIMEDOUT = 0x08, TX_SA_ERROR = 0x10, }; struct rx_srcavail_state { /* Advertised buffer stuff */ u32 mseq; u32 used; u32 reported; u32 len; u32 rkey; u64 vaddr; /* Dest buff info */ struct ib_umem *umem; struct ib_pool_fmr *fmr; /* Utility */ u8 busy; enum rx_sa_flag flags; }; struct tx_srcavail_state { /* Data below 'busy' will be reset */ u8 busy; struct ib_umem *umem; struct ib_pool_fmr *fmr; u32 bytes_sent; u32 bytes_acked; enum tx_sa_flag abort_flags; u8 posted; u32 mseq; }; struct sdp_tx_ring { #ifdef SDP_ZCOPY struct rx_srcavail_state *rdma_inflight; #endif struct sdp_buf *buffer; atomic_t head; atomic_t tail; struct ib_cq *cq; atomic_t credits; #define tx_credits(ssk) (atomic_read(&ssk->tx_ring.credits)) struct callout timer; u16 poll_cnt; }; struct sdp_rx_ring { struct sdp_buf *buffer; atomic_t head; atomic_t tail; struct ib_cq *cq; int destroyed; struct rwlock destroyed_lock; }; struct sdp_device { struct ib_pd *pd; struct ib_mr *mr; struct ib_fmr_pool *fmr_pool; }; struct sdp_moderation { unsigned long last_moder_packets; unsigned long last_moder_tx_packets; unsigned long last_moder_bytes; unsigned long last_moder_jiffies; int last_moder_time; u16 rx_usecs; u16 rx_frames; u16 tx_usecs; u32 pkt_rate_low; u16 rx_usecs_low; u32 pkt_rate_high; u16 rx_usecs_high; u16 sample_interval; u16 adaptive_rx_coal; u32 msg_enable; int moder_cnt; int moder_time; }; /* These are flags fields. */ #define SDP_TIMEWAIT 0x0001 /* In ssk timewait state. */ #define SDP_DROPPED 0x0002 /* Socket has been dropped. */ #define SDP_SOCKREF 0x0004 /* Holding a sockref for close. */ #define SDP_NODELAY 0x0008 /* Disble nagle. */ #define SDP_NEEDFIN 0x0010 /* Send a fin on the next tx. */ #define SDP_DREQWAIT 0x0020 /* Waiting on DREQ. */ #define SDP_DESTROY 0x0040 /* Being destroyed. */ #define SDP_DISCON 0x0080 /* rdma_disconnect is owed. */ /* These are oobflags */ #define SDP_HADOOB 0x0001 /* Had OOB data. */ #define SDP_HAVEOOB 0x0002 /* Have OOB data. */ struct sdp_sock { LIST_ENTRY(sdp_sock) list; struct socket *socket; struct rdma_cm_id *id; struct ib_device *ib_device; struct sdp_device *sdp_dev; struct ib_qp *qp; struct ucred *cred; struct callout keep2msl; /* 2msl and keepalive timer. */ struct callout nagle_timer; /* timeout waiting for ack */ struct ib_ucontext context; in_port_t lport; in_addr_t laddr; in_port_t fport; in_addr_t faddr; int flags; int oobflags; /* protected by rx lock. */ int state; int softerror; int recv_bytes; /* Bytes per recv. buf including header */ int xmit_size_goal; char iobc; struct sdp_rx_ring rx_ring; struct sdp_tx_ring tx_ring; struct rwlock lock; struct mbuf *rx_ctl_q; struct mbuf *rx_ctl_tail; int qp_active; /* XXX Flag. */ int max_sge; struct work_struct rx_comp_work; #define rcv_nxt(ssk) atomic_read(&(ssk->rcv_nxt)) atomic_t rcv_nxt; /* SDP specific */ atomic_t mseq_ack; #define mseq_ack(ssk) (atomic_read(&ssk->mseq_ack)) unsigned max_bufs; /* Initial buffers offered by other side */ unsigned min_bufs; /* Low water mark to wake senders */ unsigned long nagle_last_unacked; /* mseq of lastest unacked packet */ atomic_t remote_credits; #define remote_credits(ssk) (atomic_read(&ssk->remote_credits)) int poll_cq; /* SDP slow start */ int recv_request_head; /* mark the rx_head when the resize request was recieved */ int recv_request; /* XXX flag if request to resize was recieved */ unsigned long tx_packets; unsigned long rx_packets; unsigned long tx_bytes; unsigned long rx_bytes; struct sdp_moderation auto_mod; struct task shutdown_task; #ifdef SDP_ZCOPY struct tx_srcavail_state *tx_sa; struct rx_srcavail_state *rx_sa; spinlock_t tx_sa_lock; struct delayed_work srcavail_cancel_work; int srcavail_cancel_mseq; /* ZCOPY data: -1:use global; 0:disable zcopy; >0: zcopy threshold */ int zcopy_thresh; #endif }; #define sdp_sk(so) ((struct sdp_sock *)(so->so_pcb)) #define SDP_RLOCK(ssk) rw_rlock(&(ssk)->lock) #define SDP_WLOCK(ssk) rw_wlock(&(ssk)->lock) #define SDP_RUNLOCK(ssk) rw_runlock(&(ssk)->lock) #define SDP_WUNLOCK(ssk) rw_wunlock(&(ssk)->lock) #define SDP_WLOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_WLOCKED) #define SDP_RLOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_RLOCKED) #define SDP_LOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_LOCKED) static inline void tx_sa_reset(struct tx_srcavail_state *tx_sa) { memset((void *)&tx_sa->busy, 0, sizeof(*tx_sa) - offsetof(typeof(*tx_sa), busy)); } static inline void rx_ring_unlock(struct sdp_rx_ring *rx_ring) { rw_runlock(&rx_ring->destroyed_lock); } static inline int rx_ring_trylock(struct sdp_rx_ring *rx_ring) { rw_rlock(&rx_ring->destroyed_lock); if (rx_ring->destroyed) { rx_ring_unlock(rx_ring); return 0; } return 1; } static inline void rx_ring_destroy_lock(struct sdp_rx_ring *rx_ring) { rw_wlock(&rx_ring->destroyed_lock); rx_ring->destroyed = 1; rw_wunlock(&rx_ring->destroyed_lock); } static inline void sdp_arm_rx_cq(struct sdp_sock *ssk) { sdp_prf(ssk->socket, NULL, "Arming RX cq"); sdp_dbg_data(ssk->socket, "Arming RX cq\n"); ib_req_notify_cq(ssk->rx_ring.cq, IB_CQ_NEXT_COMP); } static inline void sdp_arm_tx_cq(struct sdp_sock *ssk) { sdp_prf(ssk->socket, NULL, "Arming TX cq"); sdp_dbg_data(ssk->socket, "Arming TX cq. credits: %d, posted: %d\n", tx_credits(ssk), tx_ring_posted(ssk)); ib_req_notify_cq(ssk->tx_ring.cq, IB_CQ_NEXT_COMP); } /* return the min of: * - tx credits * - free slots in tx_ring (not including SDP_MIN_TX_CREDITS */ static inline int tx_slots_free(struct sdp_sock *ssk) { int min_free; min_free = MIN(tx_credits(ssk), SDP_TX_SIZE - tx_ring_posted(ssk)); if (min_free < SDP_MIN_TX_CREDITS) return 0; return min_free - SDP_MIN_TX_CREDITS; }; /* utilities */ static inline char *mid2str(int mid) { #define ENUM2STR(e) [e] = #e static char *mid2str[] = { ENUM2STR(SDP_MID_HELLO), ENUM2STR(SDP_MID_HELLO_ACK), ENUM2STR(SDP_MID_ABORT), ENUM2STR(SDP_MID_DISCONN), ENUM2STR(SDP_MID_SENDSM), ENUM2STR(SDP_MID_RDMARDCOMPL), ENUM2STR(SDP_MID_SRCAVAIL_CANCEL), ENUM2STR(SDP_MID_CHRCVBUF), ENUM2STR(SDP_MID_CHRCVBUF_ACK), ENUM2STR(SDP_MID_DATA), ENUM2STR(SDP_MID_SRCAVAIL), ENUM2STR(SDP_MID_SINKAVAIL), }; if (mid >= ARRAY_SIZE(mid2str)) return NULL; return mid2str[mid]; } static inline struct mbuf * sdp_alloc_mb(struct socket *sk, u8 mid, int size, int wait) { struct sdp_bsdh *h; struct mbuf *mb; MGETHDR(mb, wait, MT_DATA); if (mb == NULL) return (NULL); mb->m_pkthdr.len = mb->m_len = sizeof(struct sdp_bsdh); h = mtod(mb, struct sdp_bsdh *); h->mid = mid; return mb; } static inline struct mbuf * sdp_alloc_mb_data(struct socket *sk, int wait) { return sdp_alloc_mb(sk, SDP_MID_DATA, 0, wait); } static inline struct mbuf * sdp_alloc_mb_disconnect(struct socket *sk, int wait) { return sdp_alloc_mb(sk, SDP_MID_DISCONN, 0, wait); } static inline void * mb_put(struct mbuf *mb, int len) { uint8_t *data; data = mb->m_data; data += mb->m_len; mb->m_len += len; return (void *)data; } static inline struct mbuf * sdp_alloc_mb_chrcvbuf_ack(struct socket *sk, int size, int wait) { struct mbuf *mb; struct sdp_chrecvbuf *resp_size; mb = sdp_alloc_mb(sk, SDP_MID_CHRCVBUF_ACK, sizeof(*resp_size), wait); if (mb == NULL) return (NULL); resp_size = (struct sdp_chrecvbuf *)mb_put(mb, sizeof *resp_size); resp_size->size = htonl(size); return mb; } static inline struct mbuf * sdp_alloc_mb_srcavail(struct socket *sk, u32 len, u32 rkey, u64 vaddr, int wait) { struct mbuf *mb; struct sdp_srcah *srcah; mb = sdp_alloc_mb(sk, SDP_MID_SRCAVAIL, sizeof(*srcah), wait); if (mb == NULL) return (NULL); srcah = (struct sdp_srcah *)mb_put(mb, sizeof(*srcah)); srcah->len = htonl(len); srcah->rkey = htonl(rkey); srcah->vaddr = cpu_to_be64(vaddr); return mb; } static inline struct mbuf * sdp_alloc_mb_srcavail_cancel(struct socket *sk, int wait) { return sdp_alloc_mb(sk, SDP_MID_SRCAVAIL_CANCEL, 0, wait); } static inline struct mbuf * sdp_alloc_mb_rdmardcompl(struct socket *sk, u32 len, int wait) { struct mbuf *mb; struct sdp_rrch *rrch; mb = sdp_alloc_mb(sk, SDP_MID_RDMARDCOMPL, sizeof(*rrch), wait); if (mb == NULL) return (NULL); rrch = (struct sdp_rrch *)mb_put(mb, sizeof(*rrch)); rrch->len = htonl(len); return mb; } static inline struct mbuf * sdp_alloc_mb_sendsm(struct socket *sk, int wait) { return sdp_alloc_mb(sk, SDP_MID_SENDSM, 0, wait); } static inline int sdp_tx_ring_slots_left(struct sdp_sock *ssk) { return SDP_TX_SIZE - tx_ring_posted(ssk); } static inline int credit_update_needed(struct sdp_sock *ssk) { int c; c = remote_credits(ssk); if (likely(c > SDP_MIN_TX_CREDITS)) c += c/2; return unlikely(c < rx_ring_posted(ssk)) && likely(tx_credits(ssk) > 0) && likely(sdp_tx_ring_slots_left(ssk)); } #define SDPSTATS_COUNTER_INC(stat) #define SDPSTATS_COUNTER_ADD(stat, val) #define SDPSTATS_COUNTER_MID_INC(stat, mid) #define SDPSTATS_HIST_LINEAR(stat, size) #define SDPSTATS_HIST(stat, size) static inline void sdp_cleanup_sdp_buf(struct sdp_sock *ssk, struct sdp_buf *sbuf, enum dma_data_direction dir) { struct ib_device *dev; struct mbuf *mb; int i; dev = ssk->ib_device; for (i = 0, mb = sbuf->mb; mb != NULL; mb = mb->m_next, i++) ib_dma_unmap_single(dev, sbuf->mapping[i], mb->m_len, dir); } /* sdp_main.c */ void sdp_set_default_moderation(struct sdp_sock *ssk); void sdp_start_keepalive_timer(struct socket *sk); void sdp_urg(struct sdp_sock *ssk, struct mbuf *mb); void sdp_cancel_dreq_wait_timeout(struct sdp_sock *ssk); void sdp_abort(struct socket *sk); struct sdp_sock *sdp_notify(struct sdp_sock *ssk, int error); /* sdp_cma.c */ int sdp_cma_handler(struct rdma_cm_id *, struct rdma_cm_event *); /* sdp_tx.c */ int sdp_tx_ring_create(struct sdp_sock *ssk, struct ib_device *device); void sdp_tx_ring_destroy(struct sdp_sock *ssk); int sdp_xmit_poll(struct sdp_sock *ssk, int force); void sdp_post_send(struct sdp_sock *ssk, struct mbuf *mb); void sdp_post_sends(struct sdp_sock *ssk, int wait); void sdp_post_keepalive(struct sdp_sock *ssk); /* sdp_rx.c */ void sdp_rx_ring_init(struct sdp_sock *ssk); int sdp_rx_ring_create(struct sdp_sock *ssk, struct ib_device *device); void sdp_rx_ring_destroy(struct sdp_sock *ssk); int sdp_resize_buffers(struct sdp_sock *ssk, u32 new_size); int sdp_init_buffers(struct sdp_sock *ssk, u32 new_size); void sdp_do_posts(struct sdp_sock *ssk); void sdp_rx_comp_full(struct sdp_sock *ssk); /* sdp_zcopy.c */ +struct kiocb; int sdp_sendmsg_zcopy(struct kiocb *iocb, struct socket *sk, struct iovec *iov); int sdp_handle_srcavail(struct sdp_sock *ssk, struct sdp_srcah *srcah); void sdp_handle_sendsm(struct sdp_sock *ssk, u32 mseq_ack); void sdp_handle_rdma_read_compl(struct sdp_sock *ssk, u32 mseq_ack, u32 bytes_completed); int sdp_handle_rdma_read_cqe(struct sdp_sock *ssk); int sdp_rdma_to_iovec(struct socket *sk, struct iovec *iov, struct mbuf *mb, unsigned long *used); int sdp_post_rdma_rd_compl(struct sdp_sock *ssk, struct rx_srcavail_state *rx_sa); int sdp_post_sendsm(struct socket *sk); void srcavail_cancel_timeout(struct work_struct *work); void sdp_abort_srcavail(struct socket *sk); void sdp_abort_rdma_read(struct socket *sk); int sdp_process_rx(struct sdp_sock *ssk); #endif Index: stable/9/sys/ofed/drivers/net/mlx4/Makefile =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/Makefile (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/Makefile (revision 273864) @@ -1,9 +1,34 @@ -obj-$(CONFIG_MLX4_CORE) += mlx4_core.o +# $FreeBSD$ +#.PATH: ${.CURDIR}/../../ofed/drivers/net/mlx4:${.CURDIR}/../../ofed/include/linux +.PATH: ${.CURDIR}/../../../../../include/linux -mlx4_core-y := alloc.o catas.o cmd.o cq.o eq.o fw.o icm.o intf.o main.o mcg.o \ - mr.o pd.o port.o profile.o qp.o reset.o sense.o srq.o xrcd.o +.include -obj-$(CONFIG_MLX4_EN) += mlx4_en.o -mlx4_en-y := en_main.o en_tx.o en_rx.o en_ethtool.o en_port.o en_cq.o \ - en_resources.o en_netdev.o en_frag.o en_selftest.o +KMOD = mlx4 +SRCS = device_if.h bus_if.h pci_if.h vnode_if.h +SRCS+= alloc.c catas.c cmd.c cq.c eq.c fw.c icm.c intf.c main.c mcg.c mr.c linux_compat.c linux_radix.c +SRCS+= pd.c port.c profile.c qp.c reset.c sense.c srq.c resource_tracker.c sys_tune.c +SRCS+= opt_inet.h opt_inet6.h + + +#CFLAGS+= -I${.CURDIR}/../../ofed/drivers/net/mlx4 +#CFLAGS+= -I${.CURDIR}/../../ofed/include/ +CFLAGS+= -I${.CURDIR}/../../../../../include + +.if !defined(KERNBUILDDIR) +.if ${MK_INET_SUPPORT} != "no" +opt_inet.h: + @echo "#define INET 1" > ${.TARGET} +.endif + +.if ${MK_INET6_SUPPORT} != "no" +opt_inet6.h: + @echo "#define INET6 1" > ${.TARGET} +.endif +.endif + +.include + +CFLAGS+= -Wno-cast-qual -Wno-pointer-arith -fms-extensions + Index: stable/9/sys/ofed/drivers/net/mlx4/alloc.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/alloc.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/alloc.c (revision 273864) @@ -1,453 +1,455 @@ /* * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include -#include +#include #include #include #include "mlx4.h" u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) { u32 obj; spin_lock(&bitmap->lock); obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); if (obj >= bitmap->max) { bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) & bitmap->mask; obj = find_first_zero_bit(bitmap->table, bitmap->max); } if (obj < bitmap->max) { set_bit(obj, bitmap->table); bitmap->last = (obj + 1); if (bitmap->last == bitmap->max) bitmap->last = 0; obj |= bitmap->top; } else obj = -1; if (obj != -1) --bitmap->avail; spin_unlock(&bitmap->lock); return obj; } -void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj) +void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) { - mlx4_bitmap_free_range(bitmap, obj, 1); + mlx4_bitmap_free_range(bitmap, obj, 1, use_rr); } static unsigned long find_aligned_range(unsigned long *bitmap, u32 start, u32 nbits, - int len, int align) + int len, int align, u32 skip_mask) { unsigned long end, i; again: start = ALIGN(start, align); - while ((start < nbits) && test_bit(start, bitmap)) + while ((start < nbits) && (test_bit(start, bitmap) || + (start & skip_mask))) start += align; if (start >= nbits) return -1; end = start+len; if (end > nbits) return -1; for (i = start + 1; i < end; i++) { - if (test_bit(i, bitmap)) { + if (test_bit(i, bitmap) || ((u32)i & skip_mask)) { start = i + 1; goto again; } } return start; } -u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align) +u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, + int align, u32 skip_mask) { - u32 obj, i; + u32 obj; - if (likely(cnt == 1 && align == 1)) + if (likely(cnt == 1 && align == 1 && !skip_mask)) return mlx4_bitmap_alloc(bitmap); spin_lock(&bitmap->lock); obj = find_aligned_range(bitmap->table, bitmap->last, - bitmap->max, cnt, align); + bitmap->max, cnt, align, skip_mask); if (obj >= bitmap->max) { bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) & bitmap->mask; obj = find_aligned_range(bitmap->table, 0, bitmap->max, - cnt, align); + cnt, align, skip_mask); } if (obj < bitmap->max) { - for (i = 0; i < cnt; i++) - set_bit(obj + i, bitmap->table); + bitmap_set(bitmap->table, obj, cnt); if (obj == bitmap->last) { bitmap->last = (obj + cnt); if (bitmap->last >= bitmap->max) bitmap->last = 0; } obj |= bitmap->top; } else obj = -1; if (obj != -1) bitmap->avail -= cnt; spin_unlock(&bitmap->lock); return obj; } u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) { return bitmap->avail; } -void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt) +void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, + int use_rr) { - u32 i; - obj &= bitmap->max + bitmap->reserved_top - 1; spin_lock(&bitmap->lock); - for (i = 0; i < cnt; i++) - clear_bit(obj + i, bitmap->table); - bitmap->last = min(bitmap->last, obj); - bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) - & bitmap->mask; + if (!use_rr) { + bitmap->last = min(bitmap->last, obj); + bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) + & bitmap->mask; + } + bitmap_clear(bitmap->table, obj, cnt); bitmap->avail += cnt; spin_unlock(&bitmap->lock); } int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, u32 reserved_bot, u32 reserved_top) { - int i; + /* sanity check */ + if (num <= (u64)reserved_top + reserved_bot) + return -EINVAL; /* num must be a power of 2 */ if (num != roundup_pow_of_two(num)) return -EINVAL; + if (reserved_bot + reserved_top >= num) + return -EINVAL; + bitmap->last = 0; bitmap->top = 0; bitmap->max = num - reserved_top; bitmap->mask = mask; bitmap->reserved_top = reserved_top; bitmap->avail = num - reserved_top - reserved_bot; spin_lock_init(&bitmap->lock); bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * sizeof (long), GFP_KERNEL); if (!bitmap->table) return -ENOMEM; - for (i = 0; i < reserved_bot; ++i) - set_bit(i, bitmap->table); + bitmap_set(bitmap->table, 0, reserved_bot); return 0; } void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) { kfree(bitmap->table); } /* * Handling for queue buffers -- we allocate a bunch of memory and * register it in a memory region at HCA virtual address 0. If the * requested size is > max_direct, we split the allocation into * multiple pages, so we don't require too much contiguous memory. */ int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, struct mlx4_buf *buf) { dma_addr_t t; - buf->direct.buf = NULL; if (size <= max_direct) { buf->nbufs = 1; buf->npages = 1; buf->page_shift = get_order(size) + PAGE_SHIFT; buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, size, &t, GFP_KERNEL); if (!buf->direct.buf) return -ENOMEM; buf->direct.map = t; while (t & ((1 << buf->page_shift) - 1)) { --buf->page_shift; buf->npages *= 2; } memset(buf->direct.buf, 0, size); } else { int i; buf->direct.buf = NULL; - buf->direct.map = 0; buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; buf->npages = buf->nbufs; buf->page_shift = PAGE_SHIFT; - buf->page_list = kzalloc(buf->nbufs * sizeof *buf->page_list, + buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), GFP_KERNEL); if (!buf->page_list) return -ENOMEM; for (i = 0; i < buf->nbufs; ++i) { buf->page_list[i].buf = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, &t, GFP_KERNEL); if (!buf->page_list[i].buf) goto err_free; buf->page_list[i].map = t; memset(buf->page_list[i].buf, 0, PAGE_SIZE); } if (BITS_PER_LONG == 64) { struct page **pages; pages = kmalloc(sizeof *pages * buf->nbufs, GFP_KERNEL); if (!pages) goto err_free; for (i = 0; i < buf->nbufs; ++i) pages[i] = virt_to_page(buf->page_list[i].buf); buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); kfree(pages); if (!buf->direct.buf) goto err_free; } } return 0; err_free: mlx4_buf_free(dev, size, buf); return -ENOMEM; } EXPORT_SYMBOL_GPL(mlx4_buf_alloc); void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) { int i; if (buf->nbufs == 1) dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, buf->direct.map); else { if (BITS_PER_LONG == 64 && buf->direct.buf) vunmap(buf->direct.buf); for (i = 0; i < buf->nbufs; ++i) if (buf->page_list[i].buf) dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf->page_list[i].buf, buf->page_list[i].map); kfree(buf->page_list); } - buf->direct.buf = NULL; } EXPORT_SYMBOL_GPL(mlx4_buf_free); static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device) { struct mlx4_db_pgdir *pgdir; pgdir = kzalloc(sizeof *pgdir, GFP_KERNEL); if (!pgdir) return NULL; bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); pgdir->bits[0] = pgdir->order0; pgdir->bits[1] = pgdir->order1; pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, &pgdir->db_dma, GFP_KERNEL); if (!pgdir->db_page) { kfree(pgdir); return NULL; } return pgdir; } static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, struct mlx4_db *db, int order) { int o; int i; for (o = order; o <= 1; ++o) { i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); if (i < MLX4_DB_PER_PAGE >> o) goto found; } return -ENOMEM; found: clear_bit(i, pgdir->bits[o]); i <<= o; if (o > order) set_bit(i ^ 1, pgdir->bits[order]); db->u.pgdir = pgdir; db->index = i; db->db = pgdir->db_page + db->index; db->dma = pgdir->db_dma + db->index * 4; db->order = order; return 0; } int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_db_pgdir *pgdir; int ret = 0; mutex_lock(&priv->pgdir_mutex); list_for_each_entry(pgdir, &priv->pgdir_list, list) if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) goto out; pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev)); if (!pgdir) { ret = -ENOMEM; goto out; } list_add(&pgdir->list, &priv->pgdir_list); /* This should never fail -- we just allocated an empty page: */ WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); out: mutex_unlock(&priv->pgdir_mutex); return ret; } EXPORT_SYMBOL_GPL(mlx4_db_alloc); void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) { struct mlx4_priv *priv = mlx4_priv(dev); int o; int i; mutex_lock(&priv->pgdir_mutex); o = db->order; i = db->index; if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { clear_bit(i ^ 1, db->u.pgdir->order0); ++o; } i >>= o; set_bit(i, db->u.pgdir->bits[o]); if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, db->u.pgdir->db_page, db->u.pgdir->db_dma); list_del(&db->u.pgdir->list); kfree(db->u.pgdir); } mutex_unlock(&priv->pgdir_mutex); } EXPORT_SYMBOL_GPL(mlx4_db_free); int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, int size, int max_direct) { int err; err = mlx4_db_alloc(dev, &wqres->db, 1); if (err) return err; *wqres->db.db = 0; err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf); if (err) goto err_db; err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, &wqres->mtt); if (err) goto err_buf; err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf); if (err) goto err_mtt; return 0; err_mtt: mlx4_mtt_cleanup(dev, &wqres->mtt); err_buf: mlx4_buf_free(dev, size, &wqres->buf); err_db: mlx4_db_free(dev, &wqres->db); return err; } EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, int size) { mlx4_mtt_cleanup(dev, &wqres->mtt); mlx4_buf_free(dev, size, &wqres->buf); mlx4_db_free(dev, &wqres->db); } EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); Index: stable/9/sys/ofed/drivers/net/mlx4/catas.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/catas.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/catas.c (revision 273864) @@ -1,158 +1,173 @@ /* * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include +#include +#include + #include "mlx4.h" -#define MLX4_CATAS_POLL_INTERVAL (5 * HZ) +#define MLX4_CATAS_POLL_INTERVAL (5 * HZ) static DEFINE_SPINLOCK(catas_lock); static LIST_HEAD(catas_list); static struct work_struct catas_work; static int internal_err_reset = 1; module_param(internal_err_reset, int, 0644); MODULE_PARM_DESC(internal_err_reset, - "Reset device on internal errors if non-zero (default 1)"); + "Reset device on internal errors if non-zero" + " (default 1, in SRIOV mode default is 0)"); static void dump_err_buf(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int i; mlx4_err(dev, "Internal error detected:\n"); for (i = 0; i < priv->fw.catas_size; ++i) mlx4_err(dev, " buf[%02x]: %08x\n", i, swab32(readl(priv->catas_err.map + i))); } static void poll_catas(unsigned long dev_ptr) { struct mlx4_dev *dev = (struct mlx4_dev *) dev_ptr; struct mlx4_priv *priv = mlx4_priv(dev); if (readl(priv->catas_err.map)) { - dump_err_buf(dev); + /* If the device is off-line, we cannot try to recover it */ + if (pci_channel_offline(dev->pdev)) + mod_timer(&priv->catas_err.timer, + round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL)); + else { + dump_err_buf(dev); + mlx4_dispatch_event(dev, MLX4_DEV_EVENT_CATASTROPHIC_ERROR, 0); - mlx4_dispatch_event(dev, MLX4_DEV_EVENT_CATASTROPHIC_ERROR, 0); + if (internal_err_reset) { + spin_lock(&catas_lock); + list_add(&priv->catas_err.list, &catas_list); + spin_unlock(&catas_lock); - if (internal_err_reset) { - spin_lock(&catas_lock); - list_add(&priv->catas_err.list, &catas_list); - spin_unlock(&catas_lock); - - queue_work(mlx4_wq, &catas_work); + queue_work(mlx4_wq, &catas_work); + } } } else mod_timer(&priv->catas_err.timer, round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL)); } static void catas_reset(struct work_struct *work) { struct mlx4_priv *priv, *tmppriv; struct mlx4_dev *dev; LIST_HEAD(tlist); int ret; - if (!mutex_trylock(&drv_mutex)) - return; - spin_lock_irq(&catas_lock); list_splice_init(&catas_list, &tlist); spin_unlock_irq(&catas_lock); list_for_each_entry_safe(priv, tmppriv, &tlist, catas_err.list) { struct pci_dev *pdev = priv->dev.pdev; + /* If the device is off-line, we cannot reset it */ + if (pci_channel_offline(pdev)) + continue; + ret = mlx4_restart_one(priv->dev.pdev); /* 'priv' now is not valid */ if (ret) - printk(KERN_ERR "mlx4 %s: Reset failed (%d)\n", - pci_name(pdev), ret); + pr_err("mlx4 %s: Reset failed (%d)\n", + pci_name(pdev), ret); else { dev = pci_get_drvdata(pdev); mlx4_dbg(dev, "Reset succeeded\n"); } } - mutex_unlock(&drv_mutex); } void mlx4_start_catas_poll(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); - unsigned long addr; + phys_addr_t addr; + /*If we are in SRIOV the default of the module param must be 0*/ + if (mlx4_is_mfunc(dev)) + internal_err_reset = 0; + INIT_LIST_HEAD(&priv->catas_err.list); init_timer(&priv->catas_err.timer); priv->catas_err.map = NULL; addr = pci_resource_start(dev->pdev, priv->fw.catas_bar) + priv->fw.catas_offset; priv->catas_err.map = ioremap(addr, priv->fw.catas_size * 4); if (!priv->catas_err.map) { - mlx4_warn(dev, "Failed to map internal error buffer at 0x%lx\n", - addr); + mlx4_warn(dev, "Failed to map internal error buffer at 0x%llx\n", + (unsigned long long) addr); return; } priv->catas_err.timer.data = (unsigned long) dev; priv->catas_err.timer.function = poll_catas; priv->catas_err.timer.expires = round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL); add_timer(&priv->catas_err.timer); } void mlx4_stop_catas_poll(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); del_timer_sync(&priv->catas_err.timer); - if (priv->catas_err.map) + if (priv->catas_err.map) { iounmap(priv->catas_err.map); + priv->catas_err.map = NULL; + } spin_lock_irq(&catas_lock); - list_del(&priv->catas_err.list); + list_del_init(&priv->catas_err.list); spin_unlock_irq(&catas_lock); } void __init mlx4_catas_init(void) { INIT_WORK(&catas_work, catas_reset); } Index: stable/9/sys/ofed/drivers/net/mlx4/cmd.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/cmd.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/cmd.c (revision 273864) @@ -1,453 +1,2592 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include +#include +#include #include #include #include +#include +#include +#include #include +#include #include "mlx4.h" +#include "fw.h" #define CMD_POLL_TOKEN 0xffff +#define INBOX_MASK 0xffffffffffffff00ULL +#define CMD_CHAN_VER 1 +#define CMD_CHAN_IF_REV 1 + enum { /* command completed successfully: */ CMD_STAT_OK = 0x00, /* Internal error (such as a bus error) occurred while processing command: */ CMD_STAT_INTERNAL_ERR = 0x01, /* Operation/command not supported or opcode modifier not supported: */ CMD_STAT_BAD_OP = 0x02, /* Parameter not supported or parameter out of range: */ CMD_STAT_BAD_PARAM = 0x03, /* System not enabled or bad system state: */ CMD_STAT_BAD_SYS_STATE = 0x04, /* Attempt to access reserved or unallocaterd resource: */ CMD_STAT_BAD_RESOURCE = 0x05, /* Requested resource is currently executing a command, or is otherwise busy: */ CMD_STAT_RESOURCE_BUSY = 0x06, /* Required capability exceeds device limits: */ CMD_STAT_EXCEED_LIM = 0x08, /* Resource is not in the appropriate state or ownership: */ CMD_STAT_BAD_RES_STATE = 0x09, /* Index out of range: */ CMD_STAT_BAD_INDEX = 0x0a, /* FW image corrupted: */ CMD_STAT_BAD_NVMEM = 0x0b, /* Error in ICM mapping (e.g. not enough auxiliary ICM pages to execute command): */ CMD_STAT_ICM_ERROR = 0x0c, /* Attempt to modify a QP/EE which is not in the presumed state: */ CMD_STAT_BAD_QP_STATE = 0x10, /* Bad segment parameters (Address/Size): */ CMD_STAT_BAD_SEG_PARAM = 0x20, /* Memory Region has Memory Windows bound to: */ CMD_STAT_REG_BOUND = 0x21, /* HCA local attached memory not present: */ CMD_STAT_LAM_NOT_PRE = 0x22, /* Bad management packet (silently discarded): */ CMD_STAT_BAD_PKT = 0x30, /* More outstanding CQEs in CQ than new CQ size: */ CMD_STAT_BAD_SIZE = 0x40, /* Multi Function device support required: */ CMD_STAT_MULTI_FUNC_REQ = 0x50, }; enum { HCR_IN_PARAM_OFFSET = 0x00, HCR_IN_MODIFIER_OFFSET = 0x08, HCR_OUT_PARAM_OFFSET = 0x0c, HCR_TOKEN_OFFSET = 0x14, HCR_STATUS_OFFSET = 0x18, HCR_OPMOD_SHIFT = 12, HCR_T_BIT = 21, HCR_E_BIT = 22, HCR_GO_BIT = 23 }; enum { GO_BIT_TIMEOUT_MSECS = 10000 }; +enum mlx4_vlan_transition { + MLX4_VLAN_TRANSITION_VST_VST = 0, + MLX4_VLAN_TRANSITION_VST_VGT = 1, + MLX4_VLAN_TRANSITION_VGT_VST = 2, + MLX4_VLAN_TRANSITION_VGT_VGT = 3, +}; + + struct mlx4_cmd_context { struct completion done; int result; int next; u64 out_param; u16 token; u8 fw_status; }; +static int mlx4_master_process_vhcr(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr_cmd *in_vhcr); + static int mlx4_status_to_errno(u8 status) { static const int trans_table[] = { [CMD_STAT_INTERNAL_ERR] = -EIO, [CMD_STAT_BAD_OP] = -EPERM, [CMD_STAT_BAD_PARAM] = -EINVAL, [CMD_STAT_BAD_SYS_STATE] = -ENXIO, [CMD_STAT_BAD_RESOURCE] = -EBADF, [CMD_STAT_RESOURCE_BUSY] = -EBUSY, [CMD_STAT_EXCEED_LIM] = -ENOMEM, [CMD_STAT_BAD_RES_STATE] = -EBADF, [CMD_STAT_BAD_INDEX] = -EBADF, [CMD_STAT_BAD_NVMEM] = -EFAULT, [CMD_STAT_ICM_ERROR] = -ENFILE, [CMD_STAT_BAD_QP_STATE] = -EINVAL, [CMD_STAT_BAD_SEG_PARAM] = -EFAULT, [CMD_STAT_REG_BOUND] = -EBUSY, [CMD_STAT_LAM_NOT_PRE] = -EAGAIN, [CMD_STAT_BAD_PKT] = -EINVAL, [CMD_STAT_BAD_SIZE] = -ENOMEM, [CMD_STAT_MULTI_FUNC_REQ] = -EACCES, }; if (status >= ARRAY_SIZE(trans_table) || (status != CMD_STAT_OK && trans_table[status] == 0)) return -EIO; return trans_table[status]; } +static const char *cmd_to_str(u16 cmd) +{ + switch (cmd) { + case MLX4_CMD_SYS_EN: return "SYS_EN"; + case MLX4_CMD_SYS_DIS: return "SYS_DIS"; + case MLX4_CMD_MAP_FA: return "MAP_FA"; + case MLX4_CMD_UNMAP_FA: return "UNMAP_FA"; + case MLX4_CMD_RUN_FW: return "RUN_FW"; + case MLX4_CMD_MOD_STAT_CFG: return "MOD_STAT_CFG"; + case MLX4_CMD_QUERY_DEV_CAP: return "QUERY_DEV_CAP"; + case MLX4_CMD_QUERY_FW: return "QUERY_FW"; + case MLX4_CMD_ENABLE_LAM: return "ENABLE_LAM"; + case MLX4_CMD_DISABLE_LAM: return "DISABLE_LAM"; + case MLX4_CMD_QUERY_DDR: return "QUERY_DDR"; + case MLX4_CMD_QUERY_ADAPTER: return "QUERY_ADAPTER"; + case MLX4_CMD_INIT_HCA: return "INIT_HCA"; + case MLX4_CMD_CLOSE_HCA: return "CLOSE_HCA"; + case MLX4_CMD_INIT_PORT: return "INIT_PORT"; + case MLX4_CMD_CLOSE_PORT: return "CLOSE_PORT"; + case MLX4_CMD_QUERY_HCA: return "QUERY_HCA"; + case MLX4_CMD_QUERY_PORT: return "QUERY_PORT"; + case MLX4_CMD_SENSE_PORT: return "SENSE_PORT"; + case MLX4_CMD_HW_HEALTH_CHECK: return "HW_HEALTH_CHECK"; + case MLX4_CMD_SET_PORT: return "SET_PORT"; + case MLX4_CMD_SET_NODE: return "SET_NODE"; + case MLX4_CMD_QUERY_FUNC: return "QUERY_FUNC"; + case MLX4_CMD_MAP_ICM: return "MAP_ICM"; + case MLX4_CMD_UNMAP_ICM: return "UNMAP_ICM"; + case MLX4_CMD_MAP_ICM_AUX: return "MAP_ICM_AUX"; + case MLX4_CMD_UNMAP_ICM_AUX: return "UNMAP_ICM_AUX"; + case MLX4_CMD_SET_ICM_SIZE: return "SET_ICM_SIZE"; + /*master notify fw on finish for slave's flr*/ + case MLX4_CMD_INFORM_FLR_DONE: return "INFORM_FLR_DONE"; + case MLX4_CMD_GET_OP_REQ: return "GET_OP_REQ"; + + /* TPT commands */ + case MLX4_CMD_SW2HW_MPT: return "SW2HW_MPT"; + case MLX4_CMD_QUERY_MPT: return "QUERY_MPT"; + case MLX4_CMD_HW2SW_MPT: return "HW2SW_MPT"; + case MLX4_CMD_READ_MTT: return "READ_MTT"; + case MLX4_CMD_WRITE_MTT: return "WRITE_MTT"; + case MLX4_CMD_SYNC_TPT: return "SYNC_TPT"; + + /* EQ commands */ + case MLX4_CMD_MAP_EQ: return "MAP_EQ"; + case MLX4_CMD_SW2HW_EQ: return "SW2HW_EQ"; + case MLX4_CMD_HW2SW_EQ: return "HW2SW_EQ"; + case MLX4_CMD_QUERY_EQ: return "QUERY_EQ"; + + /* CQ commands */ + case MLX4_CMD_SW2HW_CQ: return "SW2HW_CQ"; + case MLX4_CMD_HW2SW_CQ: return "HW2SW_CQ"; + case MLX4_CMD_QUERY_CQ: return "QUERY_CQ:"; + case MLX4_CMD_MODIFY_CQ: return "MODIFY_CQ:"; + + /* SRQ commands */ + case MLX4_CMD_SW2HW_SRQ: return "SW2HW_SRQ"; + case MLX4_CMD_HW2SW_SRQ: return "HW2SW_SRQ"; + case MLX4_CMD_QUERY_SRQ: return "QUERY_SRQ"; + case MLX4_CMD_ARM_SRQ: return "ARM_SRQ"; + + /* QP/EE commands */ + case MLX4_CMD_RST2INIT_QP: return "RST2INIT_QP"; + case MLX4_CMD_INIT2RTR_QP: return "INIT2RTR_QP"; + case MLX4_CMD_RTR2RTS_QP: return "RTR2RTS_QP"; + case MLX4_CMD_RTS2RTS_QP: return "RTS2RTS_QP"; + case MLX4_CMD_SQERR2RTS_QP: return "SQERR2RTS_QP"; + case MLX4_CMD_2ERR_QP: return "2ERR_QP"; + case MLX4_CMD_RTS2SQD_QP: return "RTS2SQD_QP"; + case MLX4_CMD_SQD2SQD_QP: return "SQD2SQD_QP"; + case MLX4_CMD_SQD2RTS_QP: return "SQD2RTS_QP"; + case MLX4_CMD_2RST_QP: return "2RST_QP"; + case MLX4_CMD_QUERY_QP: return "QUERY_QP"; + case MLX4_CMD_INIT2INIT_QP: return "INIT2INIT_QP"; + case MLX4_CMD_SUSPEND_QP: return "SUSPEND_QP"; + case MLX4_CMD_UNSUSPEND_QP: return "UNSUSPEND_QP"; + /* special QP and management commands */ + case MLX4_CMD_CONF_SPECIAL_QP: return "CONF_SPECIAL_QP"; + case MLX4_CMD_MAD_IFC: return "MAD_IFC"; + + /* multicast commands */ + case MLX4_CMD_READ_MCG: return "READ_MCG"; + case MLX4_CMD_WRITE_MCG: return "WRITE_MCG"; + case MLX4_CMD_MGID_HASH: return "MGID_HASH"; + + /* miscellaneous commands */ + case MLX4_CMD_DIAG_RPRT: return "DIAG_RPRT"; + case MLX4_CMD_NOP: return "NOP"; + case MLX4_CMD_ACCESS_MEM: return "ACCESS_MEM"; + case MLX4_CMD_SET_VEP: return "SET_VEP"; + + /* Ethernet specific commands */ + case MLX4_CMD_SET_VLAN_FLTR: return "SET_VLAN_FLTR"; + case MLX4_CMD_SET_MCAST_FLTR: return "SET_MCAST_FLTR"; + case MLX4_CMD_DUMP_ETH_STATS: return "DUMP_ETH_STATS"; + + /* Communication channel commands */ + case MLX4_CMD_ARM_COMM_CHANNEL: return "ARM_COMM_CHANNEL"; + case MLX4_CMD_GEN_EQE: return "GEN_EQE"; + + /* virtual commands */ + case MLX4_CMD_ALLOC_RES: return "ALLOC_RES"; + case MLX4_CMD_FREE_RES: return "FREE_RES"; + case MLX4_CMD_MCAST_ATTACH: return "MCAST_ATTACH"; + case MLX4_CMD_UCAST_ATTACH: return "UCAST_ATTACH"; + case MLX4_CMD_PROMISC: return "PROMISC"; + case MLX4_CMD_QUERY_FUNC_CAP: return "QUERY_FUNC_CAP"; + case MLX4_CMD_QP_ATTACH: return "QP_ATTACH"; + + /* debug commands */ + case MLX4_CMD_QUERY_DEBUG_MSG: return "QUERY_DEBUG_MSG"; + case MLX4_CMD_SET_DEBUG_MSG: return "SET_DEBUG_MSG"; + + /* statistics commands */ + case MLX4_CMD_QUERY_IF_STAT: return "QUERY_IF_STAT"; + case MLX4_CMD_SET_IF_STAT: return "SET_IF_STAT"; + + /* register/delete flow steering network rules */ + case MLX4_QP_FLOW_STEERING_ATTACH: return "QP_FLOW_STEERING_ATTACH"; + case MLX4_QP_FLOW_STEERING_DETACH: return "QP_FLOW_STEERING_DETACH"; + case MLX4_FLOW_STEERING_IB_UC_QP_RANGE: return "FLOW_STEERING_IB_UC_QP_RANGE"; + default: return "OTHER"; + } +} + +static u8 mlx4_errno_to_status(int errno) +{ + switch (errno) { + case -EPERM: + return CMD_STAT_BAD_OP; + case -EINVAL: + return CMD_STAT_BAD_PARAM; + case -ENXIO: + return CMD_STAT_BAD_SYS_STATE; + case -EBUSY: + return CMD_STAT_RESOURCE_BUSY; + case -ENOMEM: + return CMD_STAT_EXCEED_LIM; + case -ENFILE: + return CMD_STAT_ICM_ERROR; + default: + return CMD_STAT_INTERNAL_ERR; + } +} + +static int comm_pending(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + u32 status = readl(&priv->mfunc.comm->slave_read); + + return (swab32(status) >> 31) != priv->cmd.comm_toggle; +} + +static void mlx4_comm_cmd_post(struct mlx4_dev *dev, u8 cmd, u16 param) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + u32 val; + + priv->cmd.comm_toggle ^= 1; + val = param | (cmd << 16) | (priv->cmd.comm_toggle << 31); + __raw_writel((__force u32) cpu_to_be32(val), + &priv->mfunc.comm->slave_write); + mmiowb(); +} + +static int mlx4_comm_cmd_poll(struct mlx4_dev *dev, u8 cmd, u16 param, + unsigned long timeout) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + unsigned long end; + int err = 0; + int ret_from_pending = 0; + + /* First, verify that the master reports correct status */ + if (comm_pending(dev)) { + mlx4_warn(dev, "Communication channel is not idle." + "my toggle is %d (cmd:0x%x)\n", + priv->cmd.comm_toggle, cmd); + return -EAGAIN; + } + + /* Write command */ + down(&priv->cmd.poll_sem); + mlx4_comm_cmd_post(dev, cmd, param); + + end = msecs_to_jiffies(timeout) + jiffies; + while (comm_pending(dev) && time_before(jiffies, end)) + cond_resched(); + ret_from_pending = comm_pending(dev); + if (ret_from_pending) { + /* check if the slave is trying to boot in the middle of + * FLR process. The only non-zero result in the RESET command + * is MLX4_DELAY_RESET_SLAVE*/ + if ((MLX4_COMM_CMD_RESET == cmd)) { + mlx4_warn(dev, "Got slave FLRed from Communication" + " channel (ret:0x%x)\n", ret_from_pending); + err = MLX4_DELAY_RESET_SLAVE; + } else { + mlx4_warn(dev, "Communication channel timed out\n"); + err = -ETIMEDOUT; + } + } + + up(&priv->cmd.poll_sem); + return err; +} + +static int mlx4_comm_cmd_wait(struct mlx4_dev *dev, u8 op, + u16 param, unsigned long timeout) +{ + struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; + struct mlx4_cmd_context *context; + unsigned long end; + int err = 0; + + down(&cmd->event_sem); + + end = msecs_to_jiffies(timeout) + jiffies; + while (comm_pending(dev) && time_before(jiffies, end)) + cond_resched(); + if (comm_pending(dev)) { + mlx4_warn(dev, "mlx4_comm_cmd_wait: Comm channel " + "is not idle. My toggle is %d (op: 0x%x)\n", + mlx4_priv(dev)->cmd.comm_toggle, op); + up(&cmd->event_sem); + return -EAGAIN; + } + + spin_lock(&cmd->context_lock); + BUG_ON(cmd->free_head < 0); + context = &cmd->context[cmd->free_head]; + context->token += cmd->token_mask + 1; + cmd->free_head = context->next; + spin_unlock(&cmd->context_lock); + + init_completion(&context->done); + + mlx4_comm_cmd_post(dev, op, param); + + /* In slave, wait unconditionally for completion */ + wait_for_completion(&context->done); + + err = context->result; + if (err && context->fw_status != CMD_STAT_MULTI_FUNC_REQ) { + mlx4_err(dev, "command 0x%x failed: fw status = 0x%x\n", + op, context->fw_status); + goto out; + } + +out: + /* wait for comm channel ready + * this is necessary for prevention the race + * when switching between event to polling mode + */ + end = msecs_to_jiffies(timeout) + jiffies; + while (comm_pending(dev) && time_before(jiffies, end)) + cond_resched(); + + spin_lock(&cmd->context_lock); + context->next = cmd->free_head; + cmd->free_head = context - cmd->context; + spin_unlock(&cmd->context_lock); + + up(&cmd->event_sem); + return err; +} + +int mlx4_comm_cmd(struct mlx4_dev *dev, u8 cmd, u16 param, + unsigned long timeout) +{ + if (mlx4_priv(dev)->cmd.use_events) + return mlx4_comm_cmd_wait(dev, cmd, param, timeout); + return mlx4_comm_cmd_poll(dev, cmd, param, timeout); +} + static int cmd_pending(struct mlx4_dev *dev) { - u32 status = readl(mlx4_priv(dev)->cmd.hcr + HCR_STATUS_OFFSET); + u32 status; + if (pci_channel_offline(dev->pdev)) + return -EIO; + + status = readl(mlx4_priv(dev)->cmd.hcr + HCR_STATUS_OFFSET); + return (status & swab32(1 << HCR_GO_BIT)) || (mlx4_priv(dev)->cmd.toggle == !!(status & swab32(1 << HCR_T_BIT))); } -static int mlx4_cmd_post(struct mlx4_dev *dev, u64 in_param, u64 out_param, - u32 in_modifier, u8 op_modifier, u16 op, u16 token, - int event) +static int get_status(struct mlx4_dev *dev, u32 *status, int *go_bit, + int *t_bit) { + if (pci_channel_offline(dev->pdev)) + return -EIO; + + *status = readl(mlx4_priv(dev)->cmd.hcr + HCR_STATUS_OFFSET); + *t_bit = !!(*status & swab32(1 << HCR_T_BIT)); + *go_bit = !!(*status & swab32(1 << HCR_GO_BIT)); + + return 0; +} + +static int mlx4_cmd_post(struct mlx4_dev *dev, struct timespec *ts1, + u64 in_param, u64 out_param, u32 in_modifier, + u8 op_modifier, u16 op, u16 token, int event) +{ struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; u32 __iomem *hcr = cmd->hcr; int ret = -EAGAIN; unsigned long end; + int err, go_bit = 0, t_bit = 0; + u32 status = 0; mutex_lock(&cmd->hcr_mutex); + if (pci_channel_offline(dev->pdev)) { + /* + * Device is going through error recovery + * and cannot accept commands. + */ + ret = -EIO; + goto out; + } + end = jiffies; if (event) end += msecs_to_jiffies(GO_BIT_TIMEOUT_MSECS); while (cmd_pending(dev)) { - if (time_after_eq(jiffies, end)) + if (pci_channel_offline(dev->pdev)) { + /* + * Device is going through error recovery + * and cannot accept commands. + */ + ret = -EIO; goto out; + } + + if (time_after_eq(jiffies, end)) { + mlx4_err(dev, "%s:cmd_pending failed\n", __func__); + goto out; + } cond_resched(); } /* * We use writel (instead of something like memcpy_toio) * because writes of less than 32 bits to the HCR don't work * (and some architectures such as ia64 implement memcpy_toio * in terms of writeb). */ __raw_writel((__force u32) cpu_to_be32(in_param >> 32), hcr + 0); __raw_writel((__force u32) cpu_to_be32(in_param & 0xfffffffful), hcr + 1); __raw_writel((__force u32) cpu_to_be32(in_modifier), hcr + 2); __raw_writel((__force u32) cpu_to_be32(out_param >> 32), hcr + 3); __raw_writel((__force u32) cpu_to_be32(out_param & 0xfffffffful), hcr + 4); __raw_writel((__force u32) cpu_to_be32(token << 16), hcr + 5); + if (ts1) + ktime_get_ts(ts1); + /* __raw_writel may not order writes. */ wmb(); __raw_writel((__force u32) cpu_to_be32((1 << HCR_GO_BIT) | (cmd->toggle << HCR_T_BIT) | (event ? (1 << HCR_E_BIT) : 0) | (op_modifier << HCR_OPMOD_SHIFT) | - op), hcr + 6); + op), hcr + 6); /* * Make sure that our HCR writes don't get mixed in with * writes from another CPU starting a FW command. */ mmiowb(); cmd->toggle = cmd->toggle ^ 1; ret = 0; out: + if (ret) { + err = get_status(dev, &status, &go_bit, &t_bit); + mlx4_warn(dev, "Could not post command %s (0x%x): ret=%d, " + "in_param=0x%llx, in_mod=0x%x, op_mod=0x%x, " + "get_status err=%d, status_reg=0x%x, go_bit=%d, " + "t_bit=%d, toggle=0x%x\n", cmd_to_str(op), op, ret, + (unsigned long long) in_param, in_modifier, op_modifier, err, status, + go_bit, t_bit, cmd->toggle); + } mutex_unlock(&cmd->hcr_mutex); return ret; } +static int mlx4_slave_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param, + int out_is_imm, u32 in_modifier, u8 op_modifier, + u16 op, unsigned long timeout) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_vhcr_cmd *vhcr = priv->mfunc.vhcr; + int ret; + + mutex_lock(&priv->cmd.slave_cmd_mutex); + + vhcr->in_param = cpu_to_be64(in_param); + vhcr->out_param = out_param ? cpu_to_be64(*out_param) : 0; + vhcr->in_modifier = cpu_to_be32(in_modifier); + vhcr->opcode = cpu_to_be16((((u16) op_modifier) << 12) | (op & 0xfff)); + vhcr->token = cpu_to_be16(CMD_POLL_TOKEN); + vhcr->status = 0; + vhcr->flags = !!(priv->cmd.use_events) << 6; + + if (mlx4_is_master(dev)) { + ret = mlx4_master_process_vhcr(dev, dev->caps.function, vhcr); + if (!ret) { + if (out_is_imm) { + if (out_param) + *out_param = + be64_to_cpu(vhcr->out_param); + else { + mlx4_err(dev, "response expected while" + "output mailbox is NULL for " + "command 0x%x\n", op); + vhcr->status = CMD_STAT_BAD_PARAM; + } + } + ret = mlx4_status_to_errno(vhcr->status); + } + } else { + ret = mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR_POST, 0, + MLX4_COMM_TIME + timeout); + if (!ret) { + if (out_is_imm) { + if (out_param) + *out_param = + be64_to_cpu(vhcr->out_param); + else { + mlx4_err(dev, "response expected while" + "output mailbox is NULL for " + "command 0x%x\n", op); + vhcr->status = CMD_STAT_BAD_PARAM; + } + } + ret = mlx4_status_to_errno(vhcr->status); + } else + mlx4_err(dev, "failed execution of VHCR_POST command" + "opcode %s (0x%x)\n", cmd_to_str(op), op); + } + + mutex_unlock(&priv->cmd.slave_cmd_mutex); + return ret; +} + static int mlx4_cmd_poll(struct mlx4_dev *dev, u64 in_param, u64 *out_param, int out_is_imm, u32 in_modifier, u8 op_modifier, u16 op, unsigned long timeout) { struct mlx4_priv *priv = mlx4_priv(dev); void __iomem *hcr = priv->cmd.hcr; int err = 0; unsigned long end; u32 stat; down(&priv->cmd.poll_sem); - err = mlx4_cmd_post(dev, in_param, out_param ? *out_param : 0, + if (pci_channel_offline(dev->pdev)) { + /* + * Device is going through error recovery + * and cannot accept commands. + */ + err = -EIO; + goto out; + } + + err = mlx4_cmd_post(dev, NULL, in_param, out_param ? *out_param : 0, in_modifier, op_modifier, op, CMD_POLL_TOKEN, 0); if (err) goto out; end = msecs_to_jiffies(timeout) + jiffies; - while (cmd_pending(dev) && time_before(jiffies, end)) + while (cmd_pending(dev) && time_before(jiffies, end)) { + if (pci_channel_offline(dev->pdev)) { + /* + * Device is going through error recovery + * and cannot accept commands. + */ + err = -EIO; + goto out; + } + cond_resched(); + } if (cmd_pending(dev)) { + mlx4_warn(dev, "command %s (0x%x) timed out (go bit not cleared)\n", + cmd_to_str(op), op); err = -ETIMEDOUT; goto out; } if (out_is_imm) *out_param = (u64) be32_to_cpu((__force __be32) __raw_readl(hcr + HCR_OUT_PARAM_OFFSET)) << 32 | (u64) be32_to_cpu((__force __be32) __raw_readl(hcr + HCR_OUT_PARAM_OFFSET + 4)); - stat = be32_to_cpu((__force __be32) __raw_readl(hcr + HCR_STATUS_OFFSET)) >> 24; + stat = be32_to_cpu((__force __be32) + __raw_readl(hcr + HCR_STATUS_OFFSET)) >> 24; err = mlx4_status_to_errno(stat); - if (err) { - if (op != MLX4_CMD_SET_NODE || stat != CMD_STAT_BAD_OP) - mlx4_err(dev, "command 0x%x failed: fw status = 0x%x\n", - op, stat); - } + if (err) + mlx4_err(dev, "command %s (0x%x) failed: fw status = 0x%x\n", + cmd_to_str(op), op, stat); out: up(&priv->cmd.poll_sem); return err; } void mlx4_cmd_event(struct mlx4_dev *dev, u16 token, u8 status, u64 out_param) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cmd_context *context = &priv->cmd.context[token & priv->cmd.token_mask]; /* previously timed out command completing at long last */ if (token != context->token) return; context->fw_status = status; context->result = mlx4_status_to_errno(status); context->out_param = out_param; complete(&context->done); } static int mlx4_cmd_wait(struct mlx4_dev *dev, u64 in_param, u64 *out_param, int out_is_imm, u32 in_modifier, u8 op_modifier, u16 op, unsigned long timeout) { struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; struct mlx4_cmd_context *context; int err = 0; + int go_bit = 0, t_bit = 0, stat_err; + u32 status = 0; + struct timespec ts1, ts2; + ktime_t t1, t2, delta; + s64 ds; + if (out_is_imm && !out_param) + return -EINVAL; + down(&cmd->event_sem); spin_lock(&cmd->context_lock); BUG_ON(cmd->free_head < 0); context = &cmd->context[cmd->free_head]; context->token += cmd->token_mask + 1; cmd->free_head = context->next; spin_unlock(&cmd->context_lock); init_completion(&context->done); - mlx4_cmd_post(dev, in_param, out_param ? *out_param : 0, - in_modifier, op_modifier, op, context->token, 1); + err = mlx4_cmd_post(dev, &ts1, in_param, out_param ? *out_param : 0, + in_modifier, op_modifier, op, context->token, 1); + if (err) + goto out; - if (!wait_for_completion_timeout(&context->done, msecs_to_jiffies(timeout))) { + if (!wait_for_completion_timeout(&context->done, + msecs_to_jiffies(timeout))) { + stat_err = get_status(dev, &status, &go_bit, &t_bit); + mlx4_warn(dev, "command %s (0x%x) timed out: in_param=0x%llx, " + "in_mod=0x%x, op_mod=0x%x, get_status err=%d, " + "status_reg=0x%x, go_bit=%d, t_bit=%d, toggle=0x%x\n" + , cmd_to_str(op), op, (unsigned long long) in_param, in_modifier, + op_modifier, stat_err, status, go_bit, t_bit, + mlx4_priv(dev)->cmd.toggle); err = -EBUSY; goto out; } + if (mlx4_debug_level & MLX4_DEBUG_MASK_CMD_TIME) { + ktime_get_ts(&ts2); + t1 = timespec_to_ktime(ts1); + t2 = timespec_to_ktime(ts2); + delta = ktime_sub(t2, t1); + ds = ktime_to_ns(delta); + pr_info("mlx4: fw exec time for %s is %lld nsec\n", cmd_to_str(op), (long long) ds); + } err = context->result; if (err) { - if (op != MLX4_CMD_SET_NODE || context->fw_status != CMD_STAT_BAD_OP) - mlx4_err(dev, "command 0x%x failed: fw status = 0x%x\n", - op, context->fw_status); + mlx4_err(dev, "command %s (0x%x) failed: in_param=0x%llx, " + "in_mod=0x%x, op_mod=0x%x, fw status = 0x%x\n", + cmd_to_str(op), op, (unsigned long long) in_param, in_modifier, + op_modifier, context->fw_status); goto out; } if (out_is_imm) *out_param = context->out_param; out: spin_lock(&cmd->context_lock); context->next = cmd->free_head; cmd->free_head = context - cmd->context; spin_unlock(&cmd->context_lock); up(&cmd->event_sem); return err; } int __mlx4_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param, int out_is_imm, u32 in_modifier, u8 op_modifier, - u16 op, unsigned long timeout) + u16 op, unsigned long timeout, int native) { - if (mlx4_priv(dev)->cmd.use_events && !cold) - return mlx4_cmd_wait(dev, in_param, out_param, out_is_imm, - in_modifier, op_modifier, op, timeout); - else - return mlx4_cmd_poll(dev, in_param, out_param, out_is_imm, - in_modifier, op_modifier, op, timeout); + if (pci_channel_offline(dev->pdev)) + return -EIO; + + if (!mlx4_is_mfunc(dev) || (native && mlx4_is_master(dev))) { + if (mlx4_priv(dev)->cmd.use_events) + return mlx4_cmd_wait(dev, in_param, out_param, + out_is_imm, in_modifier, + op_modifier, op, timeout); + else + return mlx4_cmd_poll(dev, in_param, out_param, + out_is_imm, in_modifier, + op_modifier, op, timeout); + } + return mlx4_slave_cmd(dev, in_param, out_param, out_is_imm, + in_modifier, op_modifier, op, timeout); } EXPORT_SYMBOL_GPL(__mlx4_cmd); + +static int mlx4_ARM_COMM_CHANNEL(struct mlx4_dev *dev) +{ + return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_ARM_COMM_CHANNEL, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); +} + +static int mlx4_ACCESS_MEM(struct mlx4_dev *dev, u64 master_addr, + int slave, u64 slave_addr, + int size, int is_read) +{ + u64 in_param; + u64 out_param; + + if ((slave_addr & 0xfff) | (master_addr & 0xfff) | + (slave & ~0x7f) | (size & 0xff)) { + mlx4_err(dev, "Bad access mem params - slave_addr:0x%llx " + "master_addr:0x%llx slave_id:%d size:%d\n", + (unsigned long long) slave_addr, (unsigned long long) master_addr, slave, size); + return -EINVAL; + } + + if (is_read) { + in_param = (u64) slave | slave_addr; + out_param = (u64) dev->caps.function | master_addr; + } else { + in_param = (u64) dev->caps.function | master_addr; + out_param = (u64) slave | slave_addr; + } + + return mlx4_cmd_imm(dev, in_param, &out_param, size, 0, + MLX4_CMD_ACCESS_MEM, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); +} + +static int query_pkey_block(struct mlx4_dev *dev, u8 port, u16 index, u16 *pkey, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox) +{ + struct ib_smp *in_mad = (struct ib_smp *)(inbox->buf); + struct ib_smp *out_mad = (struct ib_smp *)(outbox->buf); + int err; + int i; + + if (index & 0x1f) + return -EINVAL; + + in_mad->attr_mod = cpu_to_be32(index / 32); + + err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, port, 3, + MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_NATIVE); + if (err) + return err; + + for (i = 0; i < 32; ++i) + pkey[i] = be16_to_cpu(((__be16 *) out_mad->data)[i]); + + return err; +} + +static int get_full_pkey_table(struct mlx4_dev *dev, u8 port, u16 *table, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox) +{ + int i; + int err; + + for (i = 0; i < dev->caps.pkey_table_len[port]; i += 32) { + err = query_pkey_block(dev, port, i, table + i, inbox, outbox); + if (err) + return err; + } + + return 0; +} +#define PORT_CAPABILITY_LOCATION_IN_SMP 20 +#define PORT_STATE_OFFSET 32 + +static enum ib_port_state vf_port_state(struct mlx4_dev *dev, int port, int vf) +{ + if (mlx4_get_slave_port_state(dev, vf, port) == SLAVE_PORT_UP) + return IB_PORT_ACTIVE; + else + return IB_PORT_DOWN; +} + +static int mlx4_MAD_IFC_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct ib_smp *smp = inbox->buf; + u32 index; + u8 port; + u16 *table; + int err; + int vidx, pidx; + struct mlx4_priv *priv = mlx4_priv(dev); + struct ib_smp *outsmp = outbox->buf; + __be16 *outtab = (__be16 *)(outsmp->data); + __be32 slave_cap_mask; + __be64 slave_node_guid; + port = vhcr->in_modifier; + + if (smp->base_version == 1 && + smp->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED && + smp->class_version == 1) { + if (smp->method == IB_MGMT_METHOD_GET) { + if (smp->attr_id == IB_SMP_ATTR_PKEY_TABLE) { + index = be32_to_cpu(smp->attr_mod); + if (port < 1 || port > dev->caps.num_ports) + return -EINVAL; + table = kcalloc(dev->caps.pkey_table_len[port], sizeof *table, GFP_KERNEL); + if (!table) + return -ENOMEM; + /* need to get the full pkey table because the paravirtualized + * pkeys may be scattered among several pkey blocks. + */ + err = get_full_pkey_table(dev, port, table, inbox, outbox); + if (!err) { + for (vidx = index * 32; vidx < (index + 1) * 32; ++vidx) { + pidx = priv->virt2phys_pkey[slave][port - 1][vidx]; + outtab[vidx % 32] = cpu_to_be16(table[pidx]); + } + } + kfree(table); + return err; + } + if (smp->attr_id == IB_SMP_ATTR_PORT_INFO) { + /*get the slave specific caps:*/ + /*do the command */ + err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, + vhcr->in_modifier, vhcr->op_modifier, + vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); + /* modify the response for slaves */ + if (!err && slave != mlx4_master_func_num(dev)) { + u8 *state = outsmp->data + PORT_STATE_OFFSET; + + *state = (*state & 0xf0) | vf_port_state(dev, port, slave); + slave_cap_mask = priv->mfunc.master.slave_state[slave].ib_cap_mask[port]; + memcpy(outsmp->data + PORT_CAPABILITY_LOCATION_IN_SMP, &slave_cap_mask, 4); + } + return err; + } + if (smp->attr_id == IB_SMP_ATTR_GUID_INFO) { + /* compute slave's gid block */ + smp->attr_mod = cpu_to_be32(slave / 8); + /* execute cmd */ + err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, + vhcr->in_modifier, vhcr->op_modifier, + vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); + if (!err) { + /* if needed, move slave gid to index 0 */ + if (slave % 8) + memcpy(outsmp->data, + outsmp->data + (slave % 8) * 8, 8); + /* delete all other gids */ + memset(outsmp->data + 8, 0, 56); + } + return err; + } + if (smp->attr_id == IB_SMP_ATTR_NODE_INFO) { + err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, + vhcr->in_modifier, vhcr->op_modifier, + vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); + if (!err) { + slave_node_guid = mlx4_get_slave_node_guid(dev, slave); + memcpy(outsmp->data + 12, &slave_node_guid, 8); + } + return err; + } + } + } + if (slave != mlx4_master_func_num(dev) && + ((smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) || + (smp->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED && + smp->method == IB_MGMT_METHOD_SET))) { + mlx4_err(dev, "slave %d is trying to execute a Subnet MGMT MAD, " + "class 0x%x, method 0x%x for attr 0x%x. Rejecting\n", + slave, smp->method, smp->mgmt_class, + be16_to_cpu(smp->attr_id)); + return -EPERM; + } + /*default:*/ + return mlx4_cmd_box(dev, inbox->dma, outbox->dma, + vhcr->in_modifier, vhcr->op_modifier, + vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); +} + +static int MLX4_CMD_DIAG_RPRT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + return -EPERM; +} + +static int MLX4_CMD_UPDATE_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + return -EPERM; +} + +int mlx4_DMA_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + u64 in_param; + u64 out_param; + int err; + + in_param = cmd->has_inbox ? (u64) inbox->dma : vhcr->in_param; + out_param = cmd->has_outbox ? (u64) outbox->dma : vhcr->out_param; + if (cmd->encode_slave_id) { + in_param &= 0xffffffffffffff00ll; + in_param |= slave; + } + + err = __mlx4_cmd(dev, in_param, &out_param, cmd->out_is_imm, + vhcr->in_modifier, vhcr->op_modifier, vhcr->op, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + + if (cmd->out_is_imm) + vhcr->out_param = out_param; + + return err; +} + +static struct mlx4_cmd_info cmd_info[] = { + { + .opcode = MLX4_CMD_QUERY_FW, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_FW_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_HCA, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_QUERY_DEV_CAP, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_DEV_CAP_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_FUNC_CAP, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_FUNC_CAP_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_ADAPTER, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_INIT_PORT, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_INIT_PORT_wrapper + }, + { + .opcode = MLX4_CMD_CLOSE_PORT, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_CLOSE_PORT_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_PORT, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_PORT_wrapper + }, + { + .opcode = MLX4_CMD_SET_PORT, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_SET_PORT_wrapper + }, + { + .opcode = MLX4_CMD_MAP_EQ, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_MAP_EQ_wrapper + }, + { + .opcode = MLX4_CMD_SW2HW_EQ, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_SW2HW_EQ_wrapper + }, + { + .opcode = MLX4_CMD_HW_HEALTH_CHECK, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_DIAG_RPRT, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .skip_err_print = true, + .verify = NULL, + .wrapper = MLX4_CMD_DIAG_RPRT_wrapper + }, + { + .opcode = MLX4_CMD_NOP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_ALLOC_RES, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = true, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_ALLOC_RES_wrapper + }, + { + .opcode = MLX4_CMD_FREE_RES, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_FREE_RES_wrapper + }, + { + .opcode = MLX4_CMD_SW2HW_MPT, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_SW2HW_MPT_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_MPT, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_MPT_wrapper + }, + { + .opcode = MLX4_CMD_HW2SW_MPT, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_HW2SW_MPT_wrapper + }, + { + .opcode = MLX4_CMD_READ_MTT, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_WRITE_MTT, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_WRITE_MTT_wrapper + }, + { + .opcode = MLX4_CMD_SYNC_TPT, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_HW2SW_EQ, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_HW2SW_EQ_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_EQ, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_QUERY_EQ_wrapper + }, + { + .opcode = MLX4_CMD_SW2HW_CQ, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_SW2HW_CQ_wrapper + }, + { + .opcode = MLX4_CMD_HW2SW_CQ, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_HW2SW_CQ_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_CQ, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_CQ_wrapper + }, + { + .opcode = MLX4_CMD_MODIFY_CQ, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = true, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_MODIFY_CQ_wrapper + }, + { + .opcode = MLX4_CMD_SW2HW_SRQ, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_SW2HW_SRQ_wrapper + }, + { + .opcode = MLX4_CMD_HW2SW_SRQ, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_HW2SW_SRQ_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_SRQ, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_SRQ_wrapper + }, + { + .opcode = MLX4_CMD_ARM_SRQ, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_ARM_SRQ_wrapper + }, + { + .opcode = MLX4_CMD_RST2INIT_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = true, + .verify = NULL, + .wrapper = mlx4_RST2INIT_QP_wrapper + }, + { + .opcode = MLX4_CMD_INIT2INIT_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_INIT2INIT_QP_wrapper + }, + { + .opcode = MLX4_CMD_INIT2RTR_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_INIT2RTR_QP_wrapper + }, + { + .opcode = MLX4_CMD_RTR2RTS_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_RTR2RTS_QP_wrapper + }, + { + .opcode = MLX4_CMD_RTS2RTS_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_RTS2RTS_QP_wrapper + }, + { + .opcode = MLX4_CMD_SQERR2RTS_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_SQERR2RTS_QP_wrapper + }, + { + .opcode = MLX4_CMD_2ERR_QP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_GEN_QP_wrapper + }, + { + .opcode = MLX4_CMD_RTS2SQD_QP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_GEN_QP_wrapper + }, + { + .opcode = MLX4_CMD_SQD2SQD_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_SQD2SQD_QP_wrapper + }, + { + .opcode = MLX4_CMD_SQD2RTS_QP, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_SQD2RTS_QP_wrapper + }, + { + .opcode = MLX4_CMD_2RST_QP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_2RST_QP_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_QP, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_GEN_QP_wrapper + }, + { + .opcode = MLX4_CMD_SUSPEND_QP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_GEN_QP_wrapper + }, + { + .opcode = MLX4_CMD_UNSUSPEND_QP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_GEN_QP_wrapper + }, + { + .opcode = MLX4_CMD_UPDATE_QP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .skip_err_print = true, + .verify = NULL, + .wrapper = MLX4_CMD_UPDATE_QP_wrapper + }, + { + .opcode = MLX4_CMD_CONF_SPECIAL_QP, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, /* XXX verify: only demux can do this */ + .wrapper = NULL + }, + { + .opcode = MLX4_CMD_MAD_IFC, + .has_inbox = true, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_MAD_IFC_wrapper + }, + { + .opcode = MLX4_CMD_QUERY_IF_STAT, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QUERY_IF_STAT_wrapper + }, + /* Native multicast commands are not available for guests */ + { + .opcode = MLX4_CMD_QP_ATTACH, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QP_ATTACH_wrapper + }, + { + .opcode = MLX4_CMD_PROMISC, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_PROMISC_wrapper + }, + /* Ethernet specific commands */ + { + .opcode = MLX4_CMD_SET_VLAN_FLTR, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_SET_VLAN_FLTR_wrapper + }, + { + .opcode = MLX4_CMD_SET_MCAST_FLTR, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_SET_MCAST_FLTR_wrapper + }, + { + .opcode = MLX4_CMD_DUMP_ETH_STATS, + .has_inbox = false, + .has_outbox = true, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_DUMP_ETH_STATS_wrapper + }, + { + .opcode = MLX4_CMD_INFORM_FLR_DONE, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = NULL + }, + /* flow steering commands */ + { + .opcode = MLX4_QP_FLOW_STEERING_ATTACH, + .has_inbox = true, + .has_outbox = false, + .out_is_imm = true, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QP_FLOW_STEERING_ATTACH_wrapper + }, + { + .opcode = MLX4_QP_FLOW_STEERING_DETACH, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .verify = NULL, + .wrapper = mlx4_QP_FLOW_STEERING_DETACH_wrapper + }, + /* wol commands */ + { + .opcode = MLX4_CMD_MOD_STAT_CFG, + .has_inbox = false, + .has_outbox = false, + .out_is_imm = false, + .encode_slave_id = false, + .skip_err_print = true, + .verify = NULL, + .wrapper = mlx4_MOD_STAT_CFG_wrapper + }, +}; + +static int mlx4_master_process_vhcr(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr_cmd *in_vhcr) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cmd_info *cmd = NULL; + struct mlx4_vhcr_cmd *vhcr_cmd = in_vhcr ? in_vhcr : priv->mfunc.vhcr; + struct mlx4_vhcr *vhcr; + struct mlx4_cmd_mailbox *inbox = NULL; + struct mlx4_cmd_mailbox *outbox = NULL; + u64 in_param; + u64 out_param; + int ret = 0; + int i; + int err = 0; + + /* Create sw representation of Virtual HCR */ + vhcr = kzalloc(sizeof(struct mlx4_vhcr), GFP_KERNEL); + if (!vhcr) + return -ENOMEM; + + /* DMA in the vHCR */ + if (!in_vhcr) { + ret = mlx4_ACCESS_MEM(dev, priv->mfunc.vhcr_dma, slave, + priv->mfunc.master.slave_state[slave].vhcr_dma, + ALIGN(sizeof(struct mlx4_vhcr_cmd), + MLX4_ACCESS_MEM_ALIGN), 1); + if (ret) { + mlx4_err(dev, "%s:Failed reading vhcr" + "ret: 0x%x\n", __func__, ret); + kfree(vhcr); + return ret; + } + } + + /* Fill SW VHCR fields */ + vhcr->in_param = be64_to_cpu(vhcr_cmd->in_param); + vhcr->out_param = be64_to_cpu(vhcr_cmd->out_param); + vhcr->in_modifier = be32_to_cpu(vhcr_cmd->in_modifier); + vhcr->token = be16_to_cpu(vhcr_cmd->token); + vhcr->op = be16_to_cpu(vhcr_cmd->opcode) & 0xfff; + vhcr->op_modifier = (u8) (be16_to_cpu(vhcr_cmd->opcode) >> 12); + vhcr->e_bit = vhcr_cmd->flags & (1 << 6); + + /* Lookup command */ + for (i = 0; i < ARRAY_SIZE(cmd_info); ++i) { + if (vhcr->op == cmd_info[i].opcode) { + cmd = &cmd_info[i]; + break; + } + } + if (!cmd) { + mlx4_err(dev, "unparavirt command: %s (0x%x) accepted from slave:%d\n", + cmd_to_str(vhcr->op), vhcr->op, slave); + vhcr_cmd->status = CMD_STAT_BAD_PARAM; + goto out_status; + } + + /* Read inbox */ + if (cmd->has_inbox) { + vhcr->in_param &= INBOX_MASK; + inbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(inbox)) { + vhcr_cmd->status = CMD_STAT_BAD_SIZE; + inbox = NULL; + goto out_status; + } + + if (mlx4_ACCESS_MEM(dev, inbox->dma, slave, + vhcr->in_param, + MLX4_MAILBOX_SIZE, 1)) { + mlx4_err(dev, "%s: Failed reading inbox for cmd %s (0x%x)\n", + __func__, cmd_to_str(cmd->opcode), cmd->opcode); + vhcr_cmd->status = CMD_STAT_INTERNAL_ERR; + goto out_status; + } + } + + /* Apply permission and bound checks if applicable */ + if (cmd->verify && cmd->verify(dev, slave, vhcr, inbox)) { + mlx4_warn(dev, "Command %s (0x%x) from slave: %d failed protection " + "checks for resource_id: %d\n", cmd_to_str(vhcr->op), + vhcr->op, slave, vhcr->in_modifier); + vhcr_cmd->status = CMD_STAT_BAD_OP; + goto out_status; + } + + /* Allocate outbox */ + if (cmd->has_outbox) { + outbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(outbox)) { + vhcr_cmd->status = CMD_STAT_BAD_SIZE; + outbox = NULL; + goto out_status; + } + } + + /* Execute the command! */ + if (cmd->wrapper) { + err = cmd->wrapper(dev, slave, vhcr, inbox, outbox, + cmd); + if (cmd->out_is_imm) + vhcr_cmd->out_param = cpu_to_be64(vhcr->out_param); + } else { + in_param = cmd->has_inbox ? (u64) inbox->dma : + vhcr->in_param; + out_param = cmd->has_outbox ? (u64) outbox->dma : + vhcr->out_param; + err = __mlx4_cmd(dev, in_param, &out_param, + cmd->out_is_imm, vhcr->in_modifier, + vhcr->op_modifier, vhcr->op, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + + if (cmd->out_is_imm) { + vhcr->out_param = out_param; + vhcr_cmd->out_param = cpu_to_be64(vhcr->out_param); + } + } + + if (err) { + if (!cmd->skip_err_print) + mlx4_warn(dev, "vhcr command %s (0x%x) slave:%d " + "in_param 0x%llx in_mod=0x%x, op_mod=0x%x " + "failed with error:%d, status %d\n", + cmd_to_str(vhcr->op), vhcr->op, slave, + (unsigned long long) vhcr->in_param, vhcr->in_modifier, + vhcr->op_modifier, vhcr->errno, err); + vhcr_cmd->status = mlx4_errno_to_status(err); + goto out_status; + } + + + /* Write outbox if command completed successfully */ + if (cmd->has_outbox && !vhcr_cmd->status) { + ret = mlx4_ACCESS_MEM(dev, outbox->dma, slave, + vhcr->out_param, + MLX4_MAILBOX_SIZE, MLX4_CMD_WRAPPED); + if (ret) { + /* If we failed to write back the outbox after the + *command was successfully executed, we must fail this + * slave, as it is now in undefined state */ + mlx4_err(dev, "%s: Failed writing outbox\n", __func__); + goto out; + } + } + +out_status: + /* DMA back vhcr result */ + if (!in_vhcr) { + ret = mlx4_ACCESS_MEM(dev, priv->mfunc.vhcr_dma, slave, + priv->mfunc.master.slave_state[slave].vhcr_dma, + ALIGN(sizeof(struct mlx4_vhcr), + MLX4_ACCESS_MEM_ALIGN), + MLX4_CMD_WRAPPED); + if (ret) + mlx4_err(dev, "%s:Failed writing vhcr result\n", + __func__); + else if (vhcr->e_bit && + mlx4_GEN_EQE(dev, slave, &priv->mfunc.master.cmd_eqe)) + mlx4_warn(dev, "Failed to generate command completion " + "eqe for slave %d\n", slave); + } + +out: + kfree(vhcr); + mlx4_free_cmd_mailbox(dev, inbox); + mlx4_free_cmd_mailbox(dev, outbox); + return ret; +} + +static int mlx4_master_immediate_activate_vlan_qos(struct mlx4_priv *priv, + int slave, int port) +{ + struct mlx4_vport_oper_state *vp_oper; + struct mlx4_vport_state *vp_admin; + struct mlx4_vf_immed_vlan_work *work; + int err; + int admin_vlan_ix = NO_INDX; + + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; + + if (vp_oper->state.default_vlan == vp_admin->default_vlan && + vp_oper->state.default_qos == vp_admin->default_qos) + return 0; + + work = kzalloc(sizeof(*work), GFP_KERNEL); + if (!work) + return -ENOMEM; + + if (vp_oper->state.default_vlan != vp_admin->default_vlan) { + if (MLX4_VGT != vp_admin->default_vlan) { + err = __mlx4_register_vlan(&priv->dev, port, + vp_admin->default_vlan, + &admin_vlan_ix); + if (err) { + mlx4_warn((&priv->dev), + "No vlan resources slave %d, port %d\n", + slave, port); + return err; + } + } else { + admin_vlan_ix = NO_INDX; + } + work->flags |= MLX4_VF_IMMED_VLAN_FLAG_VLAN; + mlx4_dbg((&(priv->dev)), + "alloc vlan %d idx %d slave %d port %d\n", + (int)(vp_admin->default_vlan), + admin_vlan_ix, slave, port); + } + + /* save original vlan ix and vlan id */ + work->orig_vlan_id = vp_oper->state.default_vlan; + work->orig_vlan_ix = vp_oper->vlan_idx; + + /* handle new qos */ + if (vp_oper->state.default_qos != vp_admin->default_qos) + work->flags |= MLX4_VF_IMMED_VLAN_FLAG_QOS; + + if (work->flags & MLX4_VF_IMMED_VLAN_FLAG_VLAN) + vp_oper->vlan_idx = admin_vlan_ix; + + vp_oper->state.default_vlan = vp_admin->default_vlan; + vp_oper->state.default_qos = vp_admin->default_qos; + + /* iterate over QPs owned by this slave, using UPDATE_QP */ + work->port = port; + work->slave = slave; + work->qos = vp_oper->state.default_qos; + work->vlan_id = vp_oper->state.default_vlan; + work->vlan_ix = vp_oper->vlan_idx; + work->priv = priv; + INIT_WORK(&work->work, mlx4_vf_immed_vlan_work_handler); + queue_work(priv->mfunc.master.comm_wq, &work->work); + + return 0; +} + + +static int mlx4_master_activate_admin_state(struct mlx4_priv *priv, int slave) +{ + int port, err; + struct mlx4_vport_state *vp_admin; + struct mlx4_vport_oper_state *vp_oper; + + for (port = 1; port <= MLX4_MAX_PORTS; port++) { + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; + vp_oper->state = *vp_admin; + if (MLX4_VGT != vp_admin->default_vlan) { + err = __mlx4_register_vlan(&priv->dev, port, + vp_admin->default_vlan, &(vp_oper->vlan_idx)); + if (err) { + vp_oper->vlan_idx = NO_INDX; + mlx4_warn((&priv->dev), + "No vlan resorces slave %d, port %d\n", + slave, port); + return err; + } + mlx4_dbg((&(priv->dev)), "alloc vlan %d idx %d slave %d port %d\n", + (int)(vp_oper->state.default_vlan), + vp_oper->vlan_idx, slave, port); + } + if (vp_admin->spoofchk) { + vp_oper->mac_idx = __mlx4_register_mac(&priv->dev, + port, + vp_admin->mac); + if (0 > vp_oper->mac_idx) { + err = vp_oper->mac_idx; + vp_oper->mac_idx = NO_INDX; + mlx4_warn((&priv->dev), + "No mac resources slave %d, port %d\n", + slave, port); + return err; + } + mlx4_dbg((&(priv->dev)), "alloc mac %llx idx %d slave %d port %d\n", + (unsigned long long) vp_oper->state.mac, vp_oper->mac_idx, slave, port); + } + } + return 0; +} + +static void mlx4_master_deactivate_admin_state(struct mlx4_priv *priv, int slave) +{ + int port; + struct mlx4_vport_oper_state *vp_oper; + + for (port = 1; port <= MLX4_MAX_PORTS; port++) { + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + if (NO_INDX != vp_oper->vlan_idx) { + __mlx4_unregister_vlan(&priv->dev, + port, vp_oper->state.default_vlan); + vp_oper->vlan_idx = NO_INDX; + } + if (NO_INDX != vp_oper->mac_idx) { + __mlx4_unregister_mac(&priv->dev, port, vp_oper->state.mac); + vp_oper->mac_idx = NO_INDX; + } + } + return; +} + +static void mlx4_master_do_cmd(struct mlx4_dev *dev, int slave, u8 cmd, + u16 param, u8 toggle) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; + u32 reply; + u8 is_going_down = 0; + int i; + unsigned long flags; + + slave_state[slave].comm_toggle ^= 1; + reply = (u32) slave_state[slave].comm_toggle << 31; + if (toggle != slave_state[slave].comm_toggle) { + mlx4_warn(dev, "Incorrect toggle %d from slave %d. *** MASTER" + "STATE COMPROMISIED ***\n", toggle, slave); + goto reset_slave; + } + if (cmd == MLX4_COMM_CMD_RESET) { + mlx4_warn(dev, "Received reset from slave:%d\n", slave); + slave_state[slave].active = false; + slave_state[slave].old_vlan_api = false; + mlx4_master_deactivate_admin_state(priv, slave); + for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i) { + slave_state[slave].event_eq[i].eqn = -1; + slave_state[slave].event_eq[i].token = 0; + } + /*check if we are in the middle of FLR process, + if so return "retry" status to the slave*/ + if (MLX4_COMM_CMD_FLR == slave_state[slave].last_cmd) + goto inform_slave_state; + + mlx4_dispatch_event(dev, MLX4_DEV_EVENT_SLAVE_SHUTDOWN, slave); + + /* write the version in the event field */ + reply |= mlx4_comm_get_version(); + + goto reset_slave; + } + /*command from slave in the middle of FLR*/ + if (cmd != MLX4_COMM_CMD_RESET && + MLX4_COMM_CMD_FLR == slave_state[slave].last_cmd) { + mlx4_warn(dev, "slave:%d is Trying to run cmd (0x%x) " + "in the middle of FLR\n", slave, cmd); + return; + } + + switch (cmd) { + case MLX4_COMM_CMD_VHCR0: + if (slave_state[slave].last_cmd != MLX4_COMM_CMD_RESET) + goto reset_slave; + slave_state[slave].vhcr_dma = ((u64) param) << 48; + priv->mfunc.master.slave_state[slave].cookie = 0; + break; + case MLX4_COMM_CMD_VHCR1: + if (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR0) + goto reset_slave; + slave_state[slave].vhcr_dma |= ((u64) param) << 32; + break; + case MLX4_COMM_CMD_VHCR2: + if (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR1) + goto reset_slave; + slave_state[slave].vhcr_dma |= ((u64) param) << 16; + break; + case MLX4_COMM_CMD_VHCR_EN: + if (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR2) + goto reset_slave; + slave_state[slave].vhcr_dma |= param; + if (mlx4_master_activate_admin_state(priv, slave)) + goto reset_slave; + slave_state[slave].active = true; + mlx4_dispatch_event(dev, MLX4_DEV_EVENT_SLAVE_INIT, slave); + break; + case MLX4_COMM_CMD_VHCR_POST: + if ((slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR_EN) && + (slave_state[slave].last_cmd != MLX4_COMM_CMD_VHCR_POST)) + goto reset_slave; + + mutex_lock(&priv->cmd.slave_cmd_mutex); + if (mlx4_master_process_vhcr(dev, slave, NULL)) { + mlx4_err(dev, "Failed processing vhcr for slave: %d," + " resetting slave.\n", slave); + mutex_unlock(&priv->cmd.slave_cmd_mutex); + goto reset_slave; + } + mutex_unlock(&priv->cmd.slave_cmd_mutex); + break; + default: + mlx4_warn(dev, "Bad comm cmd: %d from slave: %d\n", cmd, slave); + goto reset_slave; + } + spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); + if (!slave_state[slave].is_slave_going_down) + slave_state[slave].last_cmd = cmd; + else + is_going_down = 1; + spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); + if (is_going_down) { + mlx4_warn(dev, "Slave is going down aborting command (%d)" + " executing from slave: %d\n", + cmd, slave); + return; + } + __raw_writel((__force u32) cpu_to_be32(reply), + &priv->mfunc.comm[slave].slave_read); + mmiowb(); + + return; + +reset_slave: + /* cleanup any slave resources */ + mlx4_delete_all_resources_for_slave(dev, slave); + spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); + if (!slave_state[slave].is_slave_going_down) + slave_state[slave].last_cmd = MLX4_COMM_CMD_RESET; + spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); + /*with slave in the middle of flr, no need to clean resources again.*/ +inform_slave_state: + __raw_writel((__force u32) cpu_to_be32(reply), + &priv->mfunc.comm[slave].slave_read); + wmb(); +} + +/* master command processing */ +void mlx4_master_comm_channel(struct work_struct *work) +{ + struct mlx4_mfunc_master_ctx *master = + container_of(work, + struct mlx4_mfunc_master_ctx, + comm_work); + struct mlx4_mfunc *mfunc = + container_of(master, struct mlx4_mfunc, master); + struct mlx4_priv *priv = + container_of(mfunc, struct mlx4_priv, mfunc); + struct mlx4_dev *dev = &priv->dev; + __be32 *bit_vec; + u32 comm_cmd; + u32 vec; + int i, j, slave; + int toggle; + int served = 0; + int reported = 0; + u32 slt; + + bit_vec = master->comm_arm_bit_vector; + for (i = 0; i < COMM_CHANNEL_BIT_ARRAY_SIZE; i++) { + vec = be32_to_cpu(bit_vec[i]); + for (j = 0; j < 32; j++) { + if (!(vec & (1 << j))) + continue; + ++reported; + slave = (i * 32) + j; + comm_cmd = swab32(readl( + &mfunc->comm[slave].slave_write)); + slt = swab32(readl(&mfunc->comm[slave].slave_read)) + >> 31; + toggle = comm_cmd >> 31; + if (toggle != slt) { + if (master->slave_state[slave].comm_toggle + != slt) { + mlx4_info(dev, "slave %d out of sync." + " read toggle %d, state toggle %d. " + "Resynching.\n", slave, slt, + master->slave_state[slave].comm_toggle); + master->slave_state[slave].comm_toggle = + slt; + } + mlx4_master_do_cmd(dev, slave, + comm_cmd >> 16 & 0xff, + comm_cmd & 0xffff, toggle); + ++served; + } else + mlx4_err(dev, "slave %d out of sync." + " read toggle %d, write toggle %d.\n", slave, slt, + toggle); + } + } + + if (reported && reported != served) + mlx4_warn(dev, "Got command event with bitmask from %d slaves" + " but %d were served\n", + reported, served); +} +/* master command processing */ +void mlx4_master_arm_comm_channel(struct work_struct *work) +{ + struct mlx4_mfunc_master_ctx *master = + container_of(work, + struct mlx4_mfunc_master_ctx, + arm_comm_work); + struct mlx4_mfunc *mfunc = + container_of(master, struct mlx4_mfunc, master); + struct mlx4_priv *priv = + container_of(mfunc, struct mlx4_priv, mfunc); + struct mlx4_dev *dev = &priv->dev; + + if (mlx4_ARM_COMM_CHANNEL(dev)) + mlx4_warn(dev, "Failed to arm comm channel events\n"); +} + +static int sync_toggles(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int wr_toggle; + int rd_toggle; + unsigned long end; + + wr_toggle = swab32(readl(&priv->mfunc.comm->slave_write)) >> 31; + end = jiffies + msecs_to_jiffies(5000); + + while (time_before(jiffies, end)) { + rd_toggle = swab32(readl(&priv->mfunc.comm->slave_read)) >> 31; + if (rd_toggle == wr_toggle) { + priv->cmd.comm_toggle = rd_toggle; + return 0; + } + + cond_resched(); + } + + /* + * we could reach here if for example the previous VM using this + * function misbehaved and left the channel with unsynced state. We + * should fix this here and give this VM a chance to use a properly + * synced channel + */ + mlx4_warn(dev, "recovering from previously mis-behaved VM\n"); + __raw_writel((__force u32) 0, &priv->mfunc.comm->slave_read); + __raw_writel((__force u32) 0, &priv->mfunc.comm->slave_write); + priv->cmd.comm_toggle = 0; + + return 0; +} + +int mlx4_multi_func_init(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *s_state; + int i, j, err, port; + + if (mlx4_is_master(dev)) + priv->mfunc.comm = + ioremap(pci_resource_start(dev->pdev, priv->fw.comm_bar) + + priv->fw.comm_base, MLX4_COMM_PAGESIZE); + else + priv->mfunc.comm = + ioremap(pci_resource_start(dev->pdev, 2) + + MLX4_SLAVE_COMM_BASE, MLX4_COMM_PAGESIZE); + if (!priv->mfunc.comm) { + mlx4_err(dev, "Couldn't map communication vector.\n"); + goto err_vhcr; + } + + if (mlx4_is_master(dev)) { + priv->mfunc.master.slave_state = + kzalloc(dev->num_slaves * + sizeof(struct mlx4_slave_state), GFP_KERNEL); + if (!priv->mfunc.master.slave_state) + goto err_comm; + + priv->mfunc.master.vf_admin = + kzalloc(dev->num_slaves * + sizeof(struct mlx4_vf_admin_state), GFP_KERNEL); + if (!priv->mfunc.master.vf_admin) + goto err_comm_admin; + + priv->mfunc.master.vf_oper = + kzalloc(dev->num_slaves * + sizeof(struct mlx4_vf_oper_state), GFP_KERNEL); + if (!priv->mfunc.master.vf_oper) + goto err_comm_oper; + + for (i = 0; i < dev->num_slaves; ++i) { + s_state = &priv->mfunc.master.slave_state[i]; + s_state->last_cmd = MLX4_COMM_CMD_RESET; + mutex_init(&priv->mfunc.master.gen_eqe_mutex[i]); + for (j = 0; j < MLX4_EVENT_TYPES_NUM; ++j) + s_state->event_eq[j].eqn = -1; + __raw_writel((__force u32) 0, + &priv->mfunc.comm[i].slave_write); + __raw_writel((__force u32) 0, + &priv->mfunc.comm[i].slave_read); + mmiowb(); + for (port = 1; port <= MLX4_MAX_PORTS; port++) { + s_state->vlan_filter[port] = + kzalloc(sizeof(struct mlx4_vlan_fltr), + GFP_KERNEL); + if (!s_state->vlan_filter[port]) { + if (--port) + kfree(s_state->vlan_filter[port]); + goto err_slaves; + } + INIT_LIST_HEAD(&s_state->mcast_filters[port]); + priv->mfunc.master.vf_admin[i].vport[port].default_vlan = MLX4_VGT; + priv->mfunc.master.vf_oper[i].vport[port].state.default_vlan = MLX4_VGT; + priv->mfunc.master.vf_oper[i].vport[port].vlan_idx = NO_INDX; + priv->mfunc.master.vf_oper[i].vport[port].mac_idx = NO_INDX; + } + spin_lock_init(&s_state->lock); + } + + memset(&priv->mfunc.master.cmd_eqe, 0, dev->caps.eqe_size); + priv->mfunc.master.cmd_eqe.type = MLX4_EVENT_TYPE_CMD; + INIT_WORK(&priv->mfunc.master.comm_work, + mlx4_master_comm_channel); + INIT_WORK(&priv->mfunc.master.arm_comm_work, + mlx4_master_arm_comm_channel); + INIT_WORK(&priv->mfunc.master.slave_event_work, + mlx4_gen_slave_eqe); + INIT_WORK(&priv->mfunc.master.slave_flr_event_work, + mlx4_master_handle_slave_flr); + spin_lock_init(&priv->mfunc.master.slave_state_lock); + spin_lock_init(&priv->mfunc.master.slave_eq.event_lock); + priv->mfunc.master.comm_wq = + create_singlethread_workqueue("mlx4_comm"); + if (!priv->mfunc.master.comm_wq) + goto err_slaves; + + if (mlx4_init_resource_tracker(dev)) + goto err_thread; + + err = mlx4_ARM_COMM_CHANNEL(dev); + if (err) { + mlx4_err(dev, " Failed to arm comm channel eq: %x\n", + err); + goto err_resource; + } + + } else { + err = sync_toggles(dev); + if (err) { + mlx4_err(dev, "Couldn't sync toggles\n"); + goto err_comm; + } + } + return 0; + +err_resource: + mlx4_free_resource_tracker(dev, RES_TR_FREE_ALL); +err_thread: + flush_workqueue(priv->mfunc.master.comm_wq); + destroy_workqueue(priv->mfunc.master.comm_wq); +err_slaves: + while (--i) { + for (port = 1; port <= MLX4_MAX_PORTS; port++) + kfree(priv->mfunc.master.slave_state[i].vlan_filter[port]); + } + kfree(priv->mfunc.master.vf_oper); +err_comm_oper: + kfree(priv->mfunc.master.vf_admin); +err_comm_admin: + kfree(priv->mfunc.master.slave_state); +err_comm: + iounmap(priv->mfunc.comm); +err_vhcr: + dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, + priv->mfunc.vhcr, + priv->mfunc.vhcr_dma); + priv->mfunc.vhcr = NULL; + return -ENOMEM; +} + int mlx4_cmd_init(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); mutex_init(&priv->cmd.hcr_mutex); + mutex_init(&priv->cmd.slave_cmd_mutex); sema_init(&priv->cmd.poll_sem, 1); priv->cmd.use_events = 0; priv->cmd.toggle = 1; - priv->cmd.hcr = ioremap(pci_resource_start(dev->pdev, 0) + MLX4_HCR_BASE, - MLX4_HCR_SIZE); - if (!priv->cmd.hcr) { - mlx4_err(dev, "Couldn't map command register."); - return -ENOMEM; + priv->cmd.hcr = NULL; + priv->mfunc.vhcr = NULL; + + if (!mlx4_is_slave(dev)) { + priv->cmd.hcr = ioremap(pci_resource_start(dev->pdev, 0) + + MLX4_HCR_BASE, MLX4_HCR_SIZE); + if (!priv->cmd.hcr) { + mlx4_err(dev, "Couldn't map command register.\n"); + return -ENOMEM; + } } + if (mlx4_is_mfunc(dev)) { + priv->mfunc.vhcr = dma_alloc_coherent(&(dev->pdev->dev), PAGE_SIZE, + &priv->mfunc.vhcr_dma, + GFP_KERNEL); + if (!priv->mfunc.vhcr) { + mlx4_err(dev, "Couldn't allocate VHCR.\n"); + goto err_hcr; + } + } + priv->cmd.pool = pci_pool_create("mlx4_cmd", dev->pdev, MLX4_MAILBOX_SIZE, MLX4_MAILBOX_SIZE, 0); - if (!priv->cmd.pool) { + if (!priv->cmd.pool) + goto err_vhcr; + + return 0; + +err_vhcr: + if (mlx4_is_mfunc(dev)) + dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, + priv->mfunc.vhcr, priv->mfunc.vhcr_dma); + priv->mfunc.vhcr = NULL; + +err_hcr: + if (!mlx4_is_slave(dev)) iounmap(priv->cmd.hcr); - return -ENOMEM; + return -ENOMEM; +} + +void mlx4_multi_func_cleanup(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int i, port; + + if (mlx4_is_master(dev)) { + flush_workqueue(priv->mfunc.master.comm_wq); + destroy_workqueue(priv->mfunc.master.comm_wq); + for (i = 0; i < dev->num_slaves; i++) { + for (port = 1; port <= MLX4_MAX_PORTS; port++) + kfree(priv->mfunc.master.slave_state[i].vlan_filter[port]); + } + kfree(priv->mfunc.master.slave_state); + kfree(priv->mfunc.master.vf_admin); + kfree(priv->mfunc.master.vf_oper); } - return 0; + iounmap(priv->mfunc.comm); } void mlx4_cmd_cleanup(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); pci_pool_destroy(priv->cmd.pool); - iounmap(priv->cmd.hcr); + + if (!mlx4_is_slave(dev)) + iounmap(priv->cmd.hcr); + if (mlx4_is_mfunc(dev)) + dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, + priv->mfunc.vhcr, priv->mfunc.vhcr_dma); + priv->mfunc.vhcr = NULL; } /* * Switch to using events to issue FW commands (can only be called * after event queue for command events has been initialized). */ int mlx4_cmd_use_events(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int i; + int err = 0; priv->cmd.context = kmalloc(priv->cmd.max_cmds * sizeof (struct mlx4_cmd_context), GFP_KERNEL); if (!priv->cmd.context) return -ENOMEM; for (i = 0; i < priv->cmd.max_cmds; ++i) { priv->cmd.context[i].token = i; priv->cmd.context[i].next = i + 1; } priv->cmd.context[priv->cmd.max_cmds - 1].next = -1; priv->cmd.free_head = 0; sema_init(&priv->cmd.event_sem, priv->cmd.max_cmds); spin_lock_init(&priv->cmd.context_lock); for (priv->cmd.token_mask = 1; priv->cmd.token_mask < priv->cmd.max_cmds; priv->cmd.token_mask <<= 1) ; /* nothing */ --priv->cmd.token_mask; + down(&priv->cmd.poll_sem); priv->cmd.use_events = 1; - down(&priv->cmd.poll_sem); - - return 0; + return err; } /* * Switch back to polling (used when shutting down the device) */ void mlx4_cmd_use_polling(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int i; priv->cmd.use_events = 0; for (i = 0; i < priv->cmd.max_cmds; ++i) down(&priv->cmd.event_sem); kfree(priv->cmd.context); up(&priv->cmd.poll_sem); } struct mlx4_cmd_mailbox *mlx4_alloc_cmd_mailbox(struct mlx4_dev *dev) { struct mlx4_cmd_mailbox *mailbox; mailbox = kmalloc(sizeof *mailbox, GFP_KERNEL); if (!mailbox) return ERR_PTR(-ENOMEM); mailbox->buf = pci_pool_alloc(mlx4_priv(dev)->cmd.pool, GFP_KERNEL, &mailbox->dma); if (!mailbox->buf) { kfree(mailbox); return ERR_PTR(-ENOMEM); } + memset(mailbox->buf, 0, MLX4_MAILBOX_SIZE); + return mailbox; } EXPORT_SYMBOL_GPL(mlx4_alloc_cmd_mailbox); -void mlx4_free_cmd_mailbox(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox) +void mlx4_free_cmd_mailbox(struct mlx4_dev *dev, + struct mlx4_cmd_mailbox *mailbox) { if (!mailbox) return; pci_pool_free(mlx4_priv(dev)->cmd.pool, mailbox->buf, mailbox->dma); kfree(mailbox); } EXPORT_SYMBOL_GPL(mlx4_free_cmd_mailbox); + +u32 mlx4_comm_get_version(void) +{ + return ((u32) CMD_CHAN_IF_REV << 8) | (u32) CMD_CHAN_VER; +} + +static int mlx4_get_slave_indx(struct mlx4_dev *dev, int vf) +{ + if ((vf < 0) || (vf >= dev->num_vfs)) { + mlx4_err(dev, "Bad vf number:%d (number of activated vf: %d)\n", vf, dev->num_vfs); + return -EINVAL; + } + return (vf+1); +} + +int mlx4_set_vf_mac(struct mlx4_dev *dev, int port, int vf, u8 *mac) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_vport_state *s_info; + int slave; + + if (!mlx4_is_master(dev)) + return -EPROTONOSUPPORT; + + slave = mlx4_get_slave_indx(dev, vf); + if (slave < 0) + return -EINVAL; + + s_info = &priv->mfunc.master.vf_admin[slave].vport[port]; + s_info->mac = mlx4_mac_to_u64(mac); + mlx4_info(dev, "default mac on vf %d port %d to %llX will take afect only after vf restart\n", + vf, port, (unsigned long long) s_info->mac); + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_set_vf_mac); + +int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, u8 qos) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_vport_oper_state *vf_oper; + struct mlx4_vport_state *vf_admin; + int slave; + + if ((!mlx4_is_master(dev)) || + !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_VLAN_CONTROL)) + return -EPROTONOSUPPORT; + + if ((vlan > 4095) || (qos > 7)) + return -EINVAL; + + slave = mlx4_get_slave_indx(dev, vf); + if (slave < 0) + return -EINVAL; + + vf_admin = &priv->mfunc.master.vf_admin[slave].vport[port]; + vf_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + + if ((0 == vlan) && (0 == qos)) + vf_admin->default_vlan = MLX4_VGT; + else + vf_admin->default_vlan = vlan; + vf_admin->default_qos = qos; + + if (priv->mfunc.master.slave_state[slave].active && + dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_UPDATE_QP) { + mlx4_info(dev, "updating vf %d port %d config params immediately\n", + vf, port); + mlx4_master_immediate_activate_vlan_qos(priv, slave, port); + } + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_set_vf_vlan); + + /* mlx4_get_slave_default_vlan - + * retrun true if VST ( default vlan) + * if VST will fill vlan & qos (if not NULL) */ +bool mlx4_get_slave_default_vlan(struct mlx4_dev *dev, int port, int slave, u16 *vlan, u8 *qos) +{ + struct mlx4_vport_oper_state *vp_oper; + struct mlx4_priv *priv; + + priv = mlx4_priv(dev); + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + + if (MLX4_VGT != vp_oper->state.default_vlan) { + if (vlan) + *vlan = vp_oper->state.default_vlan; + if (qos) + *qos = vp_oper->state.default_qos; + return true; + } + return false; +} +EXPORT_SYMBOL_GPL(mlx4_get_slave_default_vlan); + +int mlx4_set_vf_spoofchk(struct mlx4_dev *dev, int port, int vf, bool setting) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_vport_state *s_info; + int slave; + + if ((!mlx4_is_master(dev)) || + !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FSM)) + return -EPROTONOSUPPORT; + + slave = mlx4_get_slave_indx(dev, vf); + if (slave < 0) + return -EINVAL; + + s_info = &priv->mfunc.master.vf_admin[slave].vport[port]; + s_info->spoofchk = setting; + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_set_vf_spoofchk); + +int mlx4_set_vf_link_state(struct mlx4_dev *dev, int port, int vf, int link_state) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_vport_state *s_info; + struct mlx4_vport_oper_state *vp_oper; + int slave; + u8 link_stat_event; + + slave = mlx4_get_slave_indx(dev, vf); + if (slave < 0) + return -EINVAL; + + switch (link_state) { + case IFLA_VF_LINK_STATE_AUTO: + /* get link curent state */ + if (!priv->sense.do_sense_port[port]) + link_stat_event = MLX4_PORT_CHANGE_SUBTYPE_ACTIVE; + else + link_stat_event = MLX4_PORT_CHANGE_SUBTYPE_DOWN; + break; + + case IFLA_VF_LINK_STATE_ENABLE: + link_stat_event = MLX4_PORT_CHANGE_SUBTYPE_ACTIVE; + break; + + case IFLA_VF_LINK_STATE_DISABLE: + link_stat_event = MLX4_PORT_CHANGE_SUBTYPE_DOWN; + break; + + default: + mlx4_warn(dev, "unknown value for link_state %02x on slave %d port %d\n", + link_state, slave, port); + return -EINVAL; + }; + /* update the admin & oper state on the link state */ + s_info = &priv->mfunc.master.vf_admin[slave].vport[port]; + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + s_info->link_state = link_state; + vp_oper->state.link_state = link_state; + + /* send event */ + mlx4_gen_port_state_change_eqe(dev, slave, port, link_stat_event); + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_set_vf_link_state); + +int mlx4_get_vf_link_state(struct mlx4_dev *dev, int port, int vf) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_vport_state *s_info; + int slave; + + if (!mlx4_is_master(dev)) + return -EPROTONOSUPPORT; + + slave = mlx4_get_slave_indx(dev, vf); + if (slave < 0) + return -EINVAL; + + s_info = &priv->mfunc.master.vf_admin[slave].vport[port]; + + return s_info->link_state; +} +EXPORT_SYMBOL_GPL(mlx4_get_vf_link_state); + Index: stable/9/sys/ofed/drivers/net/mlx4/cq.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/cq.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/cq.c (revision 273864) @@ -1,338 +1,400 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2004 Voltaire, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include #include - +#include #include #include #include "mlx4.h" #include "icm.h" -struct mlx4_cq_context { - __be32 flags; - u16 reserved1[3]; - __be16 page_offset; - __be32 logsize_usrpage; - __be16 cq_period; - __be16 cq_max_count; - u8 reserved2[3]; - u8 comp_eqn; - u8 log_page_size; - u8 reserved3[2]; - u8 mtt_base_addr_h; - __be32 mtt_base_addr_l; - __be32 last_notified_index; - __be32 solicit_producer_index; - __be32 consumer_index; - __be32 producer_index; - u32 reserved4[2]; - __be64 db_rec_addr; -}; - #define MLX4_CQ_STATUS_OK ( 0 << 28) #define MLX4_CQ_STATUS_OVERFLOW ( 9 << 28) #define MLX4_CQ_STATUS_WRITE_FAIL (10 << 28) #define MLX4_CQ_FLAG_CC ( 1 << 18) #define MLX4_CQ_FLAG_OI ( 1 << 17) #define MLX4_CQ_STATE_ARMED ( 9 << 8) #define MLX4_CQ_STATE_ARMED_SOL ( 6 << 8) #define MLX4_EQ_STATE_FIRED (10 << 8) void mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn) { + struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table; struct mlx4_cq *cq; + read_lock(&cq_table->cq_table_lock); + cq = radix_tree_lookup(&mlx4_priv(dev)->cq_table.tree, cqn & (dev->caps.num_cqs - 1)); + if (cq) + atomic_inc(&cq->refcount); + + read_unlock(&cq_table->cq_table_lock); + if (!cq) { mlx4_dbg(dev, "Completion event for bogus CQ %08x\n", cqn); return; } ++cq->arm_sn; cq->comp(cq); + + if (atomic_dec_and_test(&cq->refcount)) + complete(&cq->free); } void mlx4_cq_event(struct mlx4_dev *dev, u32 cqn, int event_type) { struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table; struct mlx4_cq *cq; - spin_lock(&cq_table->lock); + read_lock(&cq_table->cq_table_lock); cq = radix_tree_lookup(&cq_table->tree, cqn & (dev->caps.num_cqs - 1)); if (cq) atomic_inc(&cq->refcount); - spin_unlock(&cq_table->lock); + read_unlock(&cq_table->cq_table_lock); if (!cq) { mlx4_warn(dev, "Async event for bogus CQ %08x\n", cqn); return; } cq->event(cq, event_type); if (atomic_dec_and_test(&cq->refcount)) complete(&cq->free); } static int mlx4_SW2HW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, int cq_num) { - return mlx4_cmd(dev, mailbox->dma, cq_num, 0, MLX4_CMD_SW2HW_CQ, - MLX4_CMD_TIME_CLASS_A); + return mlx4_cmd(dev, mailbox->dma, cq_num, 0, + MLX4_CMD_SW2HW_CQ, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); } static int mlx4_MODIFY_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, int cq_num, u32 opmod) { return mlx4_cmd(dev, mailbox->dma, cq_num, opmod, MLX4_CMD_MODIFY_CQ, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); } static int mlx4_HW2SW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, int cq_num) { - return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, cq_num, - mailbox ? 0 : 1, MLX4_CMD_HW2SW_CQ, - MLX4_CMD_TIME_CLASS_A); + return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, + cq_num, mailbox ? 0 : 1, MLX4_CMD_HW2SW_CQ, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); } int mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq, u16 count, u16 period) { struct mlx4_cmd_mailbox *mailbox; struct mlx4_cq_context *cq_context; int err; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); cq_context = mailbox->buf; memset(cq_context, 0, sizeof *cq_context); cq_context->cq_max_count = cpu_to_be16(count); cq_context->cq_period = cpu_to_be16(period); err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 1); mlx4_free_cmd_mailbox(dev, mailbox); return err; } EXPORT_SYMBOL_GPL(mlx4_cq_modify); int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq, int entries, struct mlx4_mtt *mtt) { struct mlx4_cmd_mailbox *mailbox; struct mlx4_cq_context *cq_context; u64 mtt_addr; int err; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); cq_context = mailbox->buf; memset(cq_context, 0, sizeof *cq_context); cq_context->logsize_usrpage = cpu_to_be32(ilog2(entries) << 24); cq_context->log_page_size = mtt->page_shift - 12; mtt_addr = mlx4_mtt_addr(dev, mtt); cq_context->mtt_base_addr_h = mtt_addr >> 32; cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 0); mlx4_free_cmd_mailbox(dev, mailbox); return err; } EXPORT_SYMBOL_GPL(mlx4_cq_resize); -static int mlx4_find_least_loaded_vector(struct mlx4_priv *priv) +int mlx4_cq_ignore_overrun(struct mlx4_dev *dev, struct mlx4_cq *cq) { - int i; - int index = 0; - int min = priv->eq_table.eq[0].load; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_cq_context *cq_context; + int err; - for (i = 1; i < priv->dev.caps.num_comp_vectors; i++) { - if (priv->eq_table.eq[i].load < min) { - index = i; - min = priv->eq_table.eq[i].load; + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + cq_context = mailbox->buf; + memset(cq_context, 0, sizeof *cq_context); + + cq_context->flags |= cpu_to_be32(MLX4_CQ_FLAG_OI); + + err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 3); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_cq_ignore_overrun); + +int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cq_table *cq_table = &priv->cq_table; + int err; + + *cqn = mlx4_bitmap_alloc(&cq_table->bitmap); + if (*cqn == -1) + return -ENOMEM; + + err = mlx4_table_get(dev, &cq_table->table, *cqn); + if (err) + goto err_out; + + err = mlx4_table_get(dev, &cq_table->cmpt_table, *cqn); + if (err) + goto err_put; + return 0; + +err_put: + mlx4_table_put(dev, &cq_table->table, *cqn); + +err_out: + mlx4_bitmap_free(&cq_table->bitmap, *cqn, MLX4_NO_RR); + return err; +} + +static int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn) +{ + u64 out_param; + int err; + + if (mlx4_is_mfunc(dev)) { + err = mlx4_cmd_imm(dev, 0, &out_param, RES_CQ, + RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + return err; + else { + *cqn = get_param_l(&out_param); + return 0; } } + return __mlx4_cq_alloc_icm(dev, cqn); +} - return index; +void __mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cq_table *cq_table = &priv->cq_table; + + mlx4_table_put(dev, &cq_table->cmpt_table, cqn); + mlx4_table_put(dev, &cq_table->table, cqn); + mlx4_bitmap_free(&cq_table->bitmap, cqn, MLX4_NO_RR); } -int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt, - struct mlx4_uar *uar, u64 db_rec, struct mlx4_cq *cq, - unsigned vector, int collapsed) +static void mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn) { + u64 in_param = 0; + int err; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, cqn); + err = mlx4_cmd(dev, in_param, RES_CQ, RES_OP_RESERVE_AND_MAP, + MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + mlx4_warn(dev, "Failed freeing cq:%d\n", cqn); + } else + __mlx4_cq_free_icm(dev, cqn); +} + +int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, + struct mlx4_mtt *mtt, struct mlx4_uar *uar, u64 db_rec, + struct mlx4_cq *cq, unsigned vector, int collapsed, + int timestamp_en) +{ struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cq_table *cq_table = &priv->cq_table; struct mlx4_cmd_mailbox *mailbox; struct mlx4_cq_context *cq_context; u64 mtt_addr; int err; - cq->vector = (vector == MLX4_LEAST_ATTACHED_VECTOR) ? - mlx4_find_least_loaded_vector(priv) : vector; - - if (cq->vector >= dev->caps.num_comp_vectors) + if (vector > dev->caps.num_comp_vectors + dev->caps.comp_pool) return -EINVAL; - cq->cqn = mlx4_bitmap_alloc(&cq_table->bitmap); - if (cq->cqn == -1) - return -ENOMEM; + cq->vector = vector; - err = mlx4_table_get(dev, &cq_table->table, cq->cqn); + err = mlx4_cq_alloc_icm(dev, &cq->cqn); if (err) - goto err_out; + return err; - err = mlx4_table_get(dev, &cq_table->cmpt_table, cq->cqn); - if (err) - goto err_put; - spin_lock_irq(&cq_table->lock); err = radix_tree_insert(&cq_table->tree, cq->cqn, cq); spin_unlock_irq(&cq_table->lock); if (err) - goto err_cmpt_put; + goto err_icm; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) { err = PTR_ERR(mailbox); goto err_radix; } cq_context = mailbox->buf; memset(cq_context, 0, sizeof *cq_context); cq_context->flags = cpu_to_be32(!!collapsed << 18); + if (timestamp_en) + cq_context->flags |= cpu_to_be32(1 << 19); + cq_context->logsize_usrpage = cpu_to_be32((ilog2(nent) << 24) | uar->index); - cq_context->comp_eqn = priv->eq_table.eq[cq->vector].eqn; + cq_context->comp_eqn = priv->eq_table.eq[vector].eqn; cq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; mtt_addr = mlx4_mtt_addr(dev, mtt); cq_context->mtt_base_addr_h = mtt_addr >> 32; cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); cq_context->db_rec_addr = cpu_to_be64(db_rec); err = mlx4_SW2HW_CQ(dev, mailbox, cq->cqn); mlx4_free_cmd_mailbox(dev, mailbox); if (err) goto err_radix; - priv->eq_table.eq[cq->vector].load++; cq->cons_index = 0; cq->arm_sn = 1; cq->uar = uar; atomic_set(&cq->refcount, 1); init_completion(&cq->free); + cq->eqn = priv->eq_table.eq[cq->vector].eqn; + cq->irq = priv->eq_table.eq[cq->vector].irq; + return 0; err_radix: spin_lock_irq(&cq_table->lock); radix_tree_delete(&cq_table->tree, cq->cqn); spin_unlock_irq(&cq_table->lock); -err_cmpt_put: - mlx4_table_put(dev, &cq_table->cmpt_table, cq->cqn); +err_icm: + mlx4_cq_free_icm(dev, cq->cqn); -err_put: - mlx4_table_put(dev, &cq_table->table, cq->cqn); - -err_out: - mlx4_bitmap_free(&cq_table->bitmap, cq->cqn); - return err; } EXPORT_SYMBOL_GPL(mlx4_cq_alloc); void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cq_table *cq_table = &priv->cq_table; int err; err = mlx4_HW2SW_CQ(dev, NULL, cq->cqn); if (err) mlx4_warn(dev, "HW2SW_CQ failed (%d) for CQN %06x\n", err, cq->cqn); synchronize_irq(priv->eq_table.eq[cq->vector].irq); - priv->eq_table.eq[cq->vector].load--; spin_lock_irq(&cq_table->lock); radix_tree_delete(&cq_table->tree, cq->cqn); spin_unlock_irq(&cq_table->lock); if (atomic_dec_and_test(&cq->refcount)) complete(&cq->free); wait_for_completion(&cq->free); - mlx4_table_put(dev, &cq_table->table, cq->cqn); - mlx4_bitmap_free(&cq_table->bitmap, cq->cqn); + mlx4_cq_free_icm(dev, cq->cqn); } EXPORT_SYMBOL_GPL(mlx4_cq_free); int mlx4_init_cq_table(struct mlx4_dev *dev) { struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table; int err; spin_lock_init(&cq_table->lock); + rwlock_init(&cq_table->cq_table_lock); INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC); + if (mlx4_is_slave(dev)) + return 0; err = mlx4_bitmap_init(&cq_table->bitmap, dev->caps.num_cqs, dev->caps.num_cqs - 1, dev->caps.reserved_cqs, 0); if (err) return err; return 0; } void mlx4_cleanup_cq_table(struct mlx4_dev *dev) { + if (mlx4_is_slave(dev)) + return; /* Nothing to do to clean up radix_tree */ mlx4_bitmap_cleanup(&mlx4_priv(dev)->cq_table.bitmap); } Index: stable/9/sys/ofed/drivers/net/mlx4/en_cq.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/en_cq.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/en_cq.c (revision 273864) @@ -1,160 +1,214 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ -#include "mlx4_en.h" - #include #include #include +#include "mlx4_en.h" + + static void mlx4_en_cq_event(struct mlx4_cq *cq, enum mlx4_event event) { return; } int mlx4_en_create_cq(struct mlx4_en_priv *priv, - struct mlx4_en_cq *cq, - int entries, int ring, enum cq_type mode) + struct mlx4_en_cq **pcq, + int entries, int ring, enum cq_type mode, + int node) { struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_cq *cq; int err; + cq = kzalloc_node(sizeof(struct mlx4_en_cq), GFP_KERNEL, node); + if (!cq) { + cq = kzalloc(sizeof(struct mlx4_en_cq), GFP_KERNEL); + if (!cq) { + en_err(priv, "Failed to allocate CW struture\n"); + return -ENOMEM; + } + } + cq->size = entries; + cq->buf_size = cq->size * mdev->dev->caps.cqe_size; + cq->tq = taskqueue_create_fast("mlx4_en_que", M_NOWAIT, - taskqueue_thread_enqueue, &cq->tq); - if (mode == RX) { - cq->buf_size = cq->size * sizeof(struct mlx4_cqe); - cq->vector = (ring + priv->port) % - mdev->dev->caps.num_comp_vectors; + taskqueue_thread_enqueue, &cq->tq); + if (mode == RX) { TASK_INIT(&cq->cq_task, 0, mlx4_en_rx_que, cq); taskqueue_start_threads(&cq->tq, 1, PI_NET, "%s rx cq", - if_name(priv->dev)); + if_name(priv->dev)); + } else { - cq->buf_size = sizeof(struct mlx4_cqe); - cq->vector = MLX4_LEAST_ATTACHED_VECTOR; TASK_INIT(&cq->cq_task, 0, mlx4_en_tx_que, cq); taskqueue_start_threads(&cq->tq, 1, PI_NET, "%s tx cq", - if_name(priv->dev)); + if_name(priv->dev)); } cq->ring = ring; cq->is_tx = mode; - mtx_init(&cq->lock.m, "mlx4 cq", NULL, MTX_DEF); + spin_lock_init(&cq->lock); err = mlx4_alloc_hwq_res(mdev->dev, &cq->wqres, cq->buf_size, 2 * PAGE_SIZE); if (err) - return err; + goto err_cq; err = mlx4_en_map_buffer(&cq->wqres.buf); if (err) - mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size); - else - cq->buf = (struct mlx4_cqe *) cq->wqres.buf.direct.buf; + goto err_res; + cq->buf = (struct mlx4_cqe *) cq->wqres.buf.direct.buf; + *pcq = cq; + + return 0; + +err_res: + mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size); +err_cq: + kfree(cq); return err; } -int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq) + +int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, + int cq_idx) { struct mlx4_en_dev *mdev = priv->mdev; - int err; + int err = 0; + char name[25]; + int timestamp_en = 0; cq->dev = mdev->pndev[priv->port]; cq->mcq.set_ci_db = cq->wqres.db.db; cq->mcq.arm_db = cq->wqres.db.db + 1; *cq->mcq.set_ci_db = 0; *cq->mcq.arm_db = 0; memset(cq->buf, 0, cq->buf_size); - if (!cq->is_tx) - cq->size = priv->rx_ring[cq->ring].actual_size; + if (cq->is_tx == RX) { + if (mdev->dev->caps.comp_pool) { + if (!cq->vector) { + sprintf(name, "%s-%d", if_name(priv->dev), + cq->ring); + /* Set IRQ for specific name (per ring) */ + if (mlx4_assign_eq(mdev->dev, name, &cq->vector)) { + cq->vector = (cq->ring + 1 + priv->port) + % mdev->dev->caps.num_comp_vectors; + mlx4_warn(mdev, "Failed Assigning an EQ to " + "%s ,Falling back to legacy EQ's\n", + name); + } + } + } else { + cq->vector = (cq->ring + 1 + priv->port) % + mdev->dev->caps.num_comp_vectors; + } + } else { + struct mlx4_en_cq *rx_cq; + /* + * For TX we use the same irq per + * ring we assigned for the RX + */ + cq_idx = cq_idx % priv->rx_ring_num; + rx_cq = priv->rx_cq[cq_idx]; + cq->vector = rx_cq->vector; + } - err = mlx4_cq_alloc(mdev->dev, cq->size, &cq->wqres.mtt, &mdev->priv_uar, - cq->wqres.db.dma, &cq->mcq, cq->vector, cq->is_tx); + if (!cq->is_tx) + cq->size = priv->rx_ring[cq->ring]->actual_size; + err = mlx4_cq_alloc(mdev->dev, cq->size, &cq->wqres.mtt, + &mdev->priv_uar, cq->wqres.db.dma, &cq->mcq, + cq->vector, 0, timestamp_en); if (err) return err; cq->mcq.comp = cq->is_tx ? mlx4_en_tx_irq : mlx4_en_rx_irq; cq->mcq.event = mlx4_en_cq_event; - if (cq->is_tx) { - init_timer(&cq->timer); - cq->timer.function = mlx4_en_poll_tx_cq; - cq->timer.data = (unsigned long) cq; - } + if (cq->is_tx) { + init_timer(&cq->timer); + cq->timer.function = mlx4_en_poll_tx_cq; + cq->timer.data = (unsigned long) cq; + } + return 0; } -void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq) +void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq) { struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_cq *cq = *pcq; taskqueue_drain(cq->tq, &cq->cq_task); taskqueue_free(cq->tq); mlx4_en_unmap_buffer(&cq->wqres.buf); mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size); - cq->buf_size = 0; - cq->buf = NULL; - mtx_destroy(&cq->lock.m); + if (priv->mdev->dev->caps.comp_pool && cq->vector) + mlx4_release_eq(priv->mdev->dev, cq->vector); + kfree(cq); + *pcq = NULL; } void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq) { - struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_dev *mdev = priv->mdev; - taskqueue_drain(cq->tq, &cq->cq_task); - if (cq->is_tx) - del_timer(&cq->timer); + taskqueue_drain(cq->tq, &cq->cq_task); + if (cq->is_tx) + del_timer(&cq->timer); - mlx4_cq_free(mdev->dev, &cq->mcq); + mlx4_cq_free(mdev->dev, &cq->mcq); } + /* Set rx cq moderation parameters */ int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq) { return mlx4_cq_modify(priv->mdev->dev, &cq->mcq, cq->moder_cnt, cq->moder_time); } int mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq) { mlx4_cq_arm(&cq->mcq, MLX4_CQ_DB_REQ_NOT, priv->mdev->uar_map, &priv->mdev->uar_lock); return 0; } Index: stable/9/sys/ofed/drivers/net/mlx4/en_ethtool.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/en_ethtool.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/en_ethtool.c (revision 273864) @@ -1,513 +1,1617 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ #include #include #include -#include +#include +#include +#include +#include #include "mlx4_en.h" #include "en_port.h" +#define EN_ETHTOOL_QP_ATTACH (1ull << 63) -static void mlx4_en_update_lro_stats(struct mlx4_en_priv *priv) -{ - int i; +union mlx4_ethtool_flow_union { + struct ethtool_tcpip4_spec tcp_ip4_spec; + struct ethtool_tcpip4_spec udp_ip4_spec; + struct ethtool_tcpip4_spec sctp_ip4_spec; + struct ethtool_ah_espip4_spec ah_ip4_spec; + struct ethtool_ah_espip4_spec esp_ip4_spec; + struct ethtool_usrip4_spec usr_ip4_spec; + struct ethhdr ether_spec; + __u8 hdata[52]; +}; - priv->port_stats.lro_aggregated = 0; - priv->port_stats.lro_flushed = 0; - priv->port_stats.lro_no_desc = 0; +struct mlx4_ethtool_flow_ext { + __u8 padding[2]; + unsigned char h_dest[ETH_ALEN]; + __be16 vlan_etype; + __be16 vlan_tci; + __be32 data[2]; +}; - for (i = 0; i < priv->rx_ring_num; i++) { - priv->port_stats.lro_aggregated += priv->rx_ring[i].lro.stats.aggregated; - priv->port_stats.lro_flushed += priv->rx_ring[i].lro.stats.flushed; - priv->port_stats.lro_no_desc += priv->rx_ring[i].lro.stats.no_desc; - } -} +struct mlx4_ethtool_rx_flow_spec { + __u32 flow_type; + union mlx4_ethtool_flow_union h_u; + struct mlx4_ethtool_flow_ext h_ext; + union mlx4_ethtool_flow_union m_u; + struct mlx4_ethtool_flow_ext m_ext; + __u64 ring_cookie; + __u32 location; +}; +struct mlx4_ethtool_rxnfc { + __u32 cmd; + __u32 flow_type; + __u64 data; + struct mlx4_ethtool_rx_flow_spec fs; + __u32 rule_cnt; + __u32 rule_locs[0]; +}; + +#ifndef FLOW_MAC_EXT +#define FLOW_MAC_EXT 0x40000000 +#endif + static void mlx4_en_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_dev *mdev = priv->mdev; - sprintf(drvinfo->driver, DRV_NAME " (%s)", mdev->dev->board_id); - strncpy(drvinfo->version, DRV_VERSION " (" DRV_RELDATE ")", 32); - sprintf(drvinfo->fw_version, "%d.%d.%d", + strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); + strlcpy(drvinfo->version, DRV_VERSION " (" DRV_RELDATE ")", + sizeof(drvinfo->version)); + snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), + "%d.%d.%d", (u16) (mdev->dev->caps.fw_ver >> 32), (u16) ((mdev->dev->caps.fw_ver >> 16) & 0xffff), (u16) (mdev->dev->caps.fw_ver & 0xffff)); - strncpy(drvinfo->bus_info, pci_name(mdev->dev->pdev), 32); + strlcpy(drvinfo->bus_info, pci_name(mdev->dev->pdev), + sizeof(drvinfo->bus_info)); drvinfo->n_stats = 0; drvinfo->regdump_len = 0; drvinfo->eedump_len = 0; } -static u32 mlx4_en_get_tso(struct net_device *dev) -{ - return (dev->features & NETIF_F_TSO) != 0; -} +static const char main_strings[][ETH_GSTRING_LEN] = { + /* packet statistics */ + "rx_packets", + "rx_bytes", + "rx_multicast_packets", + "rx_broadcast_packets", + "rx_errors", + "rx_dropped", + "rx_length_errors", + "rx_over_errors", + "rx_crc_errors", + "rx_jabbers", + "rx_in_range_length_error", + "rx_out_range_length_error", + "rx_lt_64_bytes_packets", + "rx_127_bytes_packets", + "rx_255_bytes_packets", + "rx_511_bytes_packets", + "rx_1023_bytes_packets", + "rx_1518_bytes_packets", + "rx_1522_bytes_packets", + "rx_1548_bytes_packets", + "rx_gt_1548_bytes_packets", + "tx_packets", + "tx_bytes", + "tx_multicast_packets", + "tx_broadcast_packets", + "tx_errors", + "tx_dropped", + "tx_lt_64_bytes_packets", + "tx_127_bytes_packets", + "tx_255_bytes_packets", + "tx_511_bytes_packets", + "tx_1023_bytes_packets", + "tx_1518_bytes_packets", + "tx_1522_bytes_packets", + "tx_1548_bytes_packets", + "tx_gt_1548_bytes_packets", + "rx_prio_0_packets", "rx_prio_0_bytes", + "rx_prio_1_packets", "rx_prio_1_bytes", + "rx_prio_2_packets", "rx_prio_2_bytes", + "rx_prio_3_packets", "rx_prio_3_bytes", + "rx_prio_4_packets", "rx_prio_4_bytes", + "rx_prio_5_packets", "rx_prio_5_bytes", + "rx_prio_6_packets", "rx_prio_6_bytes", + "rx_prio_7_packets", "rx_prio_7_bytes", + "rx_novlan_packets", "rx_novlan_bytes", + "tx_prio_0_packets", "tx_prio_0_bytes", + "tx_prio_1_packets", "tx_prio_1_bytes", + "tx_prio_2_packets", "tx_prio_2_bytes", + "tx_prio_3_packets", "tx_prio_3_bytes", + "tx_prio_4_packets", "tx_prio_4_bytes", + "tx_prio_5_packets", "tx_prio_5_bytes", + "tx_prio_6_packets", "tx_prio_6_bytes", + "tx_prio_7_packets", "tx_prio_7_bytes", + "tx_novlan_packets", "tx_novlan_bytes", -static int mlx4_en_set_tso(struct net_device *dev, u32 data) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); + /* flow control statistics */ + "rx_pause_prio_0", "rx_pause_duration_prio_0", + "rx_pause_transition_prio_0", "tx_pause_prio_0", + "tx_pause_duration_prio_0", "tx_pause_transition_prio_0", + "rx_pause_prio_1", "rx_pause_duration_prio_1", + "rx_pause_transition_prio_1", "tx_pause_prio_1", + "tx_pause_duration_prio_1", "tx_pause_transition_prio_1", + "rx_pause_prio_2", "rx_pause_duration_prio_2", + "rx_pause_transition_prio_2", "tx_pause_prio_2", + "tx_pause_duration_prio_2", "tx_pause_transition_prio_2", + "rx_pause_prio_3", "rx_pause_duration_prio_3", + "rx_pause_transition_prio_3", "tx_pause_prio_3", + "tx_pause_duration_prio_3", "tx_pause_transition_prio_3", + "rx_pause_prio_4", "rx_pause_duration_prio_4", + "rx_pause_transition_prio_4", "tx_pause_prio_4", + "tx_pause_duration_prio_4", "tx_pause_transition_prio_4", + "rx_pause_prio_5", "rx_pause_duration_prio_5", + "rx_pause_transition_prio_5", "tx_pause_prio_5", + "tx_pause_duration_prio_5", "tx_pause_transition_prio_5", + "rx_pause_prio_6", "rx_pause_duration_prio_6", + "rx_pause_transition_prio_6", "tx_pause_prio_6", + "tx_pause_duration_prio_6", "tx_pause_transition_prio_6", + "rx_pause_prio_7", "rx_pause_duration_prio_7", + "rx_pause_transition_prio_7", "tx_pause_prio_7", + "tx_pause_duration_prio_7", "tx_pause_transition_prio_7", - if (data) { - if (!priv->mdev->LSO_support) - return -EPERM; - dev->features |= (NETIF_F_TSO | NETIF_F_TSO6); -#ifdef HAVE_NETDEV_VLAN_FEATURES - dev->vlan_features |= (NETIF_F_TSO | NETIF_F_TSO6); -#else - if (priv->vlgrp) { - int i; - struct net_device *vdev; - for (i = 0; i < VLAN_GROUP_ARRAY_LEN; i++) { - vdev = vlan_group_get_device(priv->vlgrp, i); - if (vdev) { - vdev->features |= (NETIF_F_TSO | NETIF_F_TSO6); - vlan_group_set_device(priv->vlgrp, i, vdev); - } - } - } -#endif - } else { - dev->features &= ~(NETIF_F_TSO | NETIF_F_TSO6); -#ifdef HAVE_NETDEV_VLAN_FEATURES - dev->vlan_features &= ~(NETIF_F_TSO | NETIF_F_TSO6); -#else - if (priv->vlgrp) { - int i; - struct net_device *vdev; - for (i = 0; i < VLAN_GROUP_ARRAY_LEN; i++) { - vdev = vlan_group_get_device(priv->vlgrp, i); - if (vdev) { - vdev->features &= ~(NETIF_F_TSO | NETIF_F_TSO6); - vlan_group_set_device(priv->vlgrp, i, vdev); - } - } - } -#endif - } - return 0; -} + /* VF statistics */ + "rx_packets", + "rx_bytes", + "rx_multicast_packets", + "rx_broadcast_packets", + "rx_errors", + "rx_dropped", + "tx_packets", + "tx_bytes", + "tx_multicast_packets", + "tx_broadcast_packets", + "tx_errors", -static u32 mlx4_en_get_rx_csum(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - return priv->rx_csum; -} + /* VPort statistics */ + "vport_rx_unicast_packets", + "vport_rx_unicast_bytes", + "vport_rx_multicast_packets", + "vport_rx_multicast_bytes", + "vport_rx_broadcast_packets", + "vport_rx_broadcast_bytes", + "vport_rx_dropped", + "vport_rx_errors", + "vport_tx_unicast_packets", + "vport_tx_unicast_bytes", + "vport_tx_multicast_packets", + "vport_tx_multicast_bytes", + "vport_tx_broadcast_packets", + "vport_tx_broadcast_bytes", + "vport_tx_errors", -static int mlx4_en_set_rx_csum(struct net_device *dev, u32 data) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - priv->rx_csum = (data != 0); - return 0; -} - -static const char main_strings[][ETH_GSTRING_LEN] = { - "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", - "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", - "rx_length_errors", "rx_over_errors", "rx_crc_errors", - "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", - "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors", - "tx_heartbeat_errors", "tx_window_errors", - /* port statistics */ - "lro_aggregated", "lro_flushed", "lro_no_desc", "tso_packets", - "queue_stopped", "wake_queue", "tx_timeout", "rx_alloc_failed", + "tx_tso_packets", + "tx_queue_stopped", "tx_wake_queue", "tx_timeout", "rx_alloc_failed", "rx_csum_good", "rx_csum_none", "tx_chksum_offload", - - /* packet statistics */ - "broadcast", "rx_prio_0", "rx_prio_1", "rx_prio_2", "rx_prio_3", - "rx_prio_4", "rx_prio_5", "rx_prio_6", "rx_prio_7", "tx_prio_0", - "tx_prio_1", "tx_prio_2", "tx_prio_3", "tx_prio_4", "tx_prio_5", - "tx_prio_6", "tx_prio_7", }; -#define NUM_MAIN_STATS 21 -#define NUM_ALL_STATS (NUM_MAIN_STATS + NUM_PORT_STATS + NUM_PKT_STATS + NUM_PERF_STATS) static const char mlx4_en_test_names[][ETH_GSTRING_LEN]= { - "Interupt Test", + "Interrupt Test", "Link Test", "Speed Test", "Register Test", "Loopback Test", }; static u32 mlx4_en_get_msglevel(struct net_device *dev) { return ((struct mlx4_en_priv *) netdev_priv(dev))->msg_enable; } static void mlx4_en_set_msglevel(struct net_device *dev, u32 val) { ((struct mlx4_en_priv *) netdev_priv(dev))->msg_enable = val; } static void mlx4_en_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) { - wol->supported = 0; - wol->wolopts = 0; + struct mlx4_en_priv *priv = netdev_priv(netdev); + int err = 0; + u64 config = 0; + u64 mask; - return; + if ((priv->port < 1) || (priv->port > 2)) { + en_err(priv, "Failed to get WoL information\n"); + return; + } + + mask = (priv->port == 1) ? MLX4_DEV_CAP_FLAG_WOL_PORT1 : + MLX4_DEV_CAP_FLAG_WOL_PORT2; + + if (!(priv->mdev->dev->caps.flags & mask)) { + wol->supported = 0; + wol->wolopts = 0; + return; + } + + err = mlx4_wol_read(priv->mdev->dev, &config, priv->port); + if (err) { + en_err(priv, "Failed to get WoL information\n"); + return; + } + + if (config & MLX4_EN_WOL_MAGIC) + wol->supported = WAKE_MAGIC; + else + wol->supported = 0; + + if (config & MLX4_EN_WOL_ENABLED) + wol->wolopts = WAKE_MAGIC; + else + wol->wolopts = 0; } -static int mlx4_en_get_sset_count(struct net_device *dev, int sset) +static int mlx4_en_set_wol(struct net_device *netdev, + struct ethtool_wolinfo *wol) { + struct mlx4_en_priv *priv = netdev_priv(netdev); + u64 config = 0; + int err = 0; + u64 mask; + + if ((priv->port < 1) || (priv->port > 2)) + return -EOPNOTSUPP; + + mask = (priv->port == 1) ? MLX4_DEV_CAP_FLAG_WOL_PORT1 : + MLX4_DEV_CAP_FLAG_WOL_PORT2; + + if (!(priv->mdev->dev->caps.flags & mask)) + return -EOPNOTSUPP; + + if (wol->supported & ~WAKE_MAGIC) + return -EINVAL; + + err = mlx4_wol_read(priv->mdev->dev, &config, priv->port); + if (err) { + en_err(priv, "Failed to get WoL info, unable to modify\n"); + return err; + } + + if (wol->wolopts & WAKE_MAGIC) { + config |= MLX4_EN_WOL_DO_MODIFY | MLX4_EN_WOL_ENABLED | + MLX4_EN_WOL_MAGIC; + } else { + config &= ~(MLX4_EN_WOL_ENABLED | MLX4_EN_WOL_MAGIC); + config |= MLX4_EN_WOL_DO_MODIFY; + } + + err = mlx4_wol_write(priv->mdev->dev, config, priv->port); + if (err) + en_err(priv, "Failed to set WoL information\n"); + + return err; +} + +struct bitmap_sim_iterator { + bool advance_array; + unsigned long *stats_bitmap; + unsigned int count; + unsigned int j; +}; + +static inline void bitmap_sim_iterator_init(struct bitmap_sim_iterator *h, + unsigned long *stats_bitmap, + int count) +{ + h->j = 0; + h->advance_array = !bitmap_empty(stats_bitmap, count); + h->count = h->advance_array ? bitmap_weight(stats_bitmap, count) + : count; + h->stats_bitmap = stats_bitmap; +} + +static inline int bitmap_sim_iterator_test(struct bitmap_sim_iterator *h) +{ + return !h->advance_array ? 1 : test_bit(h->j, h->stats_bitmap); +} + +static inline int bitmap_sim_iterator_inc(struct bitmap_sim_iterator *h) +{ + return h->j++; +} + +static inline unsigned int bitmap_sim_iterator_count( + struct bitmap_sim_iterator *h) +{ + return h->count; +} + +int mlx4_en_get_sset_count(struct net_device *dev, int sset) +{ struct mlx4_en_priv *priv = netdev_priv(dev); + struct bitmap_sim_iterator it; + int num_of_stats = NUM_ALL_STATS - + ((priv->mdev->dev->caps.flags2 & + MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) ? 0 : NUM_FLOW_STATS); + + bitmap_sim_iterator_init(&it, priv->stats_bitmap, num_of_stats); + switch (sset) { case ETH_SS_STATS: - return NUM_ALL_STATS + - (priv->tx_ring_num + priv->rx_ring_num) * 2; + return bitmap_sim_iterator_count(&it) + + (priv->tx_ring_num * 2) + +#ifdef LL_EXTENDED_STATS + (priv->rx_ring_num * 5); +#else + (priv->rx_ring_num * 2); +#endif case ETH_SS_TEST: - return MLX4_EN_NUM_SELF_TEST - !(priv->mdev->dev->caps.loopback_support) * 2; + return MLX4_EN_NUM_SELF_TEST - !(priv->mdev->dev->caps.flags + & MLX4_DEV_CAP_FLAG_UC_LOOPBACK) * 2; default: return -EOPNOTSUPP; } } -static void mlx4_en_get_ethtool_stats(struct net_device *dev, - struct ethtool_stats *stats, uint64_t *data) +void mlx4_en_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, u64 *data) { struct mlx4_en_priv *priv = netdev_priv(dev); int index = 0; int i; + struct bitmap_sim_iterator it; + int num_of_stats = NUM_ALL_STATS - + ((priv->mdev->dev->caps.flags2 & + MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) ? 0 : NUM_FLOW_STATS); + + bitmap_sim_iterator_init(&it, priv->stats_bitmap, num_of_stats); + + if (!data || !priv->port_up) + return; + spin_lock_bh(&priv->stats_lock); - mlx4_en_update_lro_stats(priv); + for (i = 0; i < NUM_PKT_STATS; i++, + bitmap_sim_iterator_inc(&it)) + if (bitmap_sim_iterator_test(&it)) + data[index++] = + ((unsigned long *)&priv->pkstats)[i]; + for (i = 0; i < NUM_FLOW_STATS; i++, + bitmap_sim_iterator_inc(&it)) + if (priv->mdev->dev->caps.flags2 & + MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) + if (bitmap_sim_iterator_test(&it)) + data[index++] = + ((u64 *)&priv->flowstats)[i]; + for (i = 0; i < NUM_VF_STATS; i++, + bitmap_sim_iterator_inc(&it)) + if (bitmap_sim_iterator_test(&it)) + data[index++] = + ((unsigned long *)&priv->vf_stats)[i]; + for (i = 0; i < NUM_VPORT_STATS; i++, + bitmap_sim_iterator_inc(&it)) + if (bitmap_sim_iterator_test(&it)) + data[index++] = + ((unsigned long *)&priv->vport_stats)[i]; + for (i = 0; i < NUM_PORT_STATS; i++, + bitmap_sim_iterator_inc(&it)) + if (bitmap_sim_iterator_test(&it)) + data[index++] = + ((unsigned long *)&priv->port_stats)[i]; - for (i = 0; i < NUM_MAIN_STATS; i++) - data[index++] = ((unsigned long *) &priv->stats)[i]; - for (i = 0; i < NUM_PORT_STATS; i++) - data[index++] = ((unsigned long *) &priv->port_stats)[i]; for (i = 0; i < priv->tx_ring_num; i++) { - data[index++] = priv->tx_ring[i].packets; - data[index++] = priv->tx_ring[i].bytes; + data[index++] = priv->tx_ring[i]->packets; + data[index++] = priv->tx_ring[i]->bytes; } for (i = 0; i < priv->rx_ring_num; i++) { - data[index++] = priv->rx_ring[i].packets; - data[index++] = priv->rx_ring[i].bytes; + data[index++] = priv->rx_ring[i]->packets; + data[index++] = priv->rx_ring[i]->bytes; +#ifdef LL_EXTENDED_STATS + data[index++] = priv->rx_ring[i]->yields; + data[index++] = priv->rx_ring[i]->misses; + data[index++] = priv->rx_ring[i]->cleaned; +#endif } - for (i = 0; i < NUM_PKT_STATS; i++) - data[index++] = ((unsigned long *) &priv->pkstats)[i]; spin_unlock_bh(&priv->stats_lock); } +void mlx4_en_restore_ethtool_stats(struct mlx4_en_priv *priv, u64 *data) +{ + int index = 0; + int i; + struct bitmap_sim_iterator it; + + int num_of_stats = NUM_ALL_STATS - + ((priv->mdev->dev->caps.flags2 & + MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) ? 0 : NUM_FLOW_STATS); + + bitmap_sim_iterator_init(&it, priv->stats_bitmap, num_of_stats); + + if (!data || !priv->port_up) + return; + + spin_lock_bh(&priv->stats_lock); + + for (i = 0; i < NUM_PKT_STATS; i++, + bitmap_sim_iterator_inc(&it)) + if (bitmap_sim_iterator_test(&it)) + ((unsigned long *)&priv->pkstats)[i] = + data[index++]; + for (i = 0; i < NUM_FLOW_STATS; i++, + bitmap_sim_iterator_inc(&it)) + if (priv->mdev->dev->caps.flags2 & + MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) + if (bitmap_sim_iterator_test(&it)) + ((u64 *)&priv->flowstats)[i] = + data[index++]; + for (i = 0; i < NUM_VF_STATS; i++, + bitmap_sim_iterator_inc(&it)) + if (bitmap_sim_iterator_test(&it)) + ((unsigned long *)&priv->vf_stats)[i] = + data[index++]; + for (i = 0; i < NUM_VPORT_STATS; i++, + bitmap_sim_iterator_inc(&it)) + if (bitmap_sim_iterator_test(&it)) + ((unsigned long *)&priv->vport_stats)[i] = + data[index++]; + for (i = 0; i < NUM_PORT_STATS; i++, + bitmap_sim_iterator_inc(&it)) + if (bitmap_sim_iterator_test(&it)) + ((unsigned long *)&priv->port_stats)[i] = + data[index++]; + + for (i = 0; i < priv->tx_ring_num; i++) { + priv->tx_ring[i]->packets = data[index++]; + priv->tx_ring[i]->bytes = data[index++]; + } + for (i = 0; i < priv->rx_ring_num; i++) { + priv->rx_ring[i]->packets = data[index++]; + priv->rx_ring[i]->bytes = data[index++]; + } + spin_unlock_bh(&priv->stats_lock); +} + static void mlx4_en_self_test(struct net_device *dev, struct ethtool_test *etest, u64 *buf) { mlx4_en_ex_selftest(dev, &etest->flags, buf); } static void mlx4_en_get_strings(struct net_device *dev, uint32_t stringset, uint8_t *data) { struct mlx4_en_priv *priv = netdev_priv(dev); int index = 0; - int i; + int i, k; + struct bitmap_sim_iterator it; + int num_of_stats = NUM_ALL_STATS - + ((priv->mdev->dev->caps.flags2 & + MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) ? 0 : NUM_FLOW_STATS); + + bitmap_sim_iterator_init(&it, priv->stats_bitmap, num_of_stats); + switch (stringset) { case ETH_SS_TEST: for (i = 0; i < MLX4_EN_NUM_SELF_TEST - 2; i++) strcpy(data + i * ETH_GSTRING_LEN, mlx4_en_test_names[i]); - if (priv->mdev->dev->caps.loopback_support) + if (priv->mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UC_LOOPBACK) for (; i < MLX4_EN_NUM_SELF_TEST; i++) strcpy(data + i * ETH_GSTRING_LEN, mlx4_en_test_names[i]); break; case ETH_SS_STATS: /* Add main counters */ - for (i = 0; i < NUM_MAIN_STATS; i++) - strcpy(data + (index++) * ETH_GSTRING_LEN, main_strings[i]); - for (i = 0; i< NUM_PORT_STATS; i++) - strcpy(data + (index++) * ETH_GSTRING_LEN, - main_strings[i + NUM_MAIN_STATS]); + for (i = 0; i < NUM_PKT_STATS; i++, + bitmap_sim_iterator_inc(&it)) + if (bitmap_sim_iterator_test(&it)) + strcpy(data + (index++) * ETH_GSTRING_LEN, + main_strings[i]); + + for (k = 0; k < NUM_FLOW_STATS; k++, + bitmap_sim_iterator_inc(&it)) + if (priv->mdev->dev->caps.flags2 & + MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) + if (bitmap_sim_iterator_test(&it)) + strcpy(data + (index++) * + ETH_GSTRING_LEN, + main_strings[i + k]); + + for (; (i + k) < num_of_stats; i++, + bitmap_sim_iterator_inc(&it)) + if (bitmap_sim_iterator_test(&it)) + strcpy(data + (index++) * ETH_GSTRING_LEN, + main_strings[i + k]); + for (i = 0; i < priv->tx_ring_num; i++) { sprintf(data + (index++) * ETH_GSTRING_LEN, "tx%d_packets", i); sprintf(data + (index++) * ETH_GSTRING_LEN, "tx%d_bytes", i); } for (i = 0; i < priv->rx_ring_num; i++) { sprintf(data + (index++) * ETH_GSTRING_LEN, "rx%d_packets", i); sprintf(data + (index++) * ETH_GSTRING_LEN, "rx%d_bytes", i); +#ifdef LL_EXTENDED_STATS + sprintf(data + (index++) * ETH_GSTRING_LEN, + "rx%d_napi_yield", i); + sprintf(data + (index++) * ETH_GSTRING_LEN, + "rx%d_misses", i); + sprintf(data + (index++) * ETH_GSTRING_LEN, + "rx%d_cleaned", i); +#endif } - for (i = 0; i< NUM_PKT_STATS; i++) - strcpy(data + (index++) * ETH_GSTRING_LEN, - main_strings[i + NUM_MAIN_STATS + NUM_PORT_STATS]); break; } } +static u32 mlx4_en_autoneg_get(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + u32 autoneg = AUTONEG_DISABLE; + + if ((mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETH_BACKPL_AN_REP) && + priv->port_state.autoneg) { + autoneg = AUTONEG_ENABLE; + } + + return autoneg; +} + static int mlx4_en_get_settings(struct net_device *dev, struct ethtool_cmd *cmd) { struct mlx4_en_priv *priv = netdev_priv(dev); int trans_type; - cmd->autoneg = AUTONEG_DISABLE; - cmd->supported = SUPPORTED_10000baseT_Full; - cmd->advertising = ADVERTISED_10000baseT_Full; + /* SUPPORTED_1000baseT_Half isn't supported */ + cmd->supported = SUPPORTED_1000baseT_Full + |SUPPORTED_10000baseT_Full; + cmd->advertising = ADVERTISED_1000baseT_Full + |ADVERTISED_10000baseT_Full; + + cmd->supported |= SUPPORTED_1000baseKX_Full + |SUPPORTED_10000baseKX4_Full + |SUPPORTED_10000baseKR_Full + |SUPPORTED_10000baseR_FEC + |SUPPORTED_40000baseKR4_Full + |SUPPORTED_40000baseCR4_Full + |SUPPORTED_40000baseSR4_Full + |SUPPORTED_40000baseLR4_Full; + + /* ADVERTISED_1000baseT_Half isn't advertised */ + cmd->advertising |= ADVERTISED_1000baseKX_Full + |ADVERTISED_10000baseKX4_Full + |ADVERTISED_10000baseKR_Full + |ADVERTISED_10000baseR_FEC + |ADVERTISED_40000baseKR4_Full + |ADVERTISED_40000baseCR4_Full + |ADVERTISED_40000baseSR4_Full + |ADVERTISED_40000baseLR4_Full; + if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) return -ENOMEM; + cmd->autoneg = mlx4_en_autoneg_get(dev); + if (cmd->autoneg == AUTONEG_ENABLE) { + cmd->supported |= SUPPORTED_Autoneg; + cmd->advertising |= ADVERTISED_Autoneg; + } + trans_type = priv->port_state.transciver; if (netif_carrier_ok(dev)) { - cmd->speed = priv->port_state.link_speed; + ethtool_cmd_speed_set(cmd, priv->port_state.link_speed); cmd->duplex = DUPLEX_FULL; } else { - cmd->speed = -1; + ethtool_cmd_speed_set(cmd, -1); cmd->duplex = -1; } if (trans_type > 0 && trans_type <= 0xC) { cmd->port = PORT_FIBRE; cmd->transceiver = XCVR_EXTERNAL; cmd->supported |= SUPPORTED_FIBRE; cmd->advertising |= ADVERTISED_FIBRE; } else if (trans_type == 0x80 || trans_type == 0) { cmd->port = PORT_TP; cmd->transceiver = XCVR_INTERNAL; cmd->supported |= SUPPORTED_TP; cmd->advertising |= ADVERTISED_TP; } else { cmd->port = -1; cmd->transceiver = -1; } return 0; } +static const char *mlx4_en_duplex_to_string(int duplex) +{ + switch (duplex) { + case DUPLEX_FULL: + return "FULL"; + case DUPLEX_HALF: + return "HALF"; + default: + break; + } + return "UNKNOWN"; +} + static int mlx4_en_set_settings(struct net_device *dev, struct ethtool_cmd *cmd) { - if ((cmd->autoneg == AUTONEG_ENABLE) || - (cmd->speed != SPEED_10000) || (cmd->duplex != DUPLEX_FULL)) - return -EINVAL; + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_port_state *port_state = &priv->port_state; - /* Nothing to change */ + if ((cmd->autoneg != port_state->autoneg) || + (ethtool_cmd_speed(cmd) != port_state->link_speed) || + (cmd->duplex != DUPLEX_FULL)) { + en_info(priv, "Changing port state properties (auto-negotiation" + " , speed/duplex) is not supported. Current:" + " auto-negotiation=%d speed/duplex=%d/%s\n", + port_state->autoneg, port_state->link_speed, + mlx4_en_duplex_to_string(DUPLEX_FULL)); + return -EOPNOTSUPP; + } + + /* User provided same port state properties that are currently set. + * Nothing to change + */ return 0; } static int mlx4_en_get_coalesce(struct net_device *dev, struct ethtool_coalesce *coal) { struct mlx4_en_priv *priv = netdev_priv(dev); - coal->tx_coalesce_usecs = 0; - coal->tx_max_coalesced_frames = 0; + coal->tx_coalesce_usecs = priv->tx_usecs; + coal->tx_max_coalesced_frames = priv->tx_frames; coal->rx_coalesce_usecs = priv->rx_usecs; coal->rx_max_coalesced_frames = priv->rx_frames; coal->pkt_rate_low = priv->pkt_rate_low; coal->rx_coalesce_usecs_low = priv->rx_usecs_low; coal->pkt_rate_high = priv->pkt_rate_high; coal->rx_coalesce_usecs_high = priv->rx_usecs_high; coal->rate_sample_interval = priv->sample_interval; coal->use_adaptive_rx_coalesce = priv->adaptive_rx_coal; return 0; } static int mlx4_en_set_coalesce(struct net_device *dev, struct ethtool_coalesce *coal) { struct mlx4_en_priv *priv = netdev_priv(dev); int err, i; priv->rx_frames = (coal->rx_max_coalesced_frames == MLX4_EN_AUTO_CONF) ? MLX4_EN_RX_COAL_TARGET / priv->dev->mtu + 1 : coal->rx_max_coalesced_frames; priv->rx_usecs = (coal->rx_coalesce_usecs == MLX4_EN_AUTO_CONF) ? MLX4_EN_RX_COAL_TIME : coal->rx_coalesce_usecs; + /* Setting TX coalescing parameters */ + if (coal->tx_coalesce_usecs != priv->tx_usecs || + coal->tx_max_coalesced_frames != priv->tx_frames) { + priv->tx_usecs = coal->tx_coalesce_usecs; + priv->tx_frames = coal->tx_max_coalesced_frames; + if (priv->port_up) { + for (i = 0; i < priv->tx_ring_num; i++) { + priv->tx_cq[i]->moder_cnt = priv->tx_frames; + priv->tx_cq[i]->moder_time = priv->tx_usecs; + if (mlx4_en_set_cq_moder(priv, priv->tx_cq[i])) + en_warn(priv, "Failed changing moderation for TX cq %d\n", i); + } + } + } + /* Set adaptive coalescing params */ priv->pkt_rate_low = coal->pkt_rate_low; priv->rx_usecs_low = coal->rx_coalesce_usecs_low; priv->pkt_rate_high = coal->pkt_rate_high; priv->rx_usecs_high = coal->rx_coalesce_usecs_high; priv->sample_interval = coal->rate_sample_interval; priv->adaptive_rx_coal = coal->use_adaptive_rx_coalesce; - priv->last_moder_time = MLX4_EN_AUTO_CONF; if (priv->adaptive_rx_coal) return 0; - for (i = 0; i < priv->rx_ring_num; i++) { - priv->rx_cq[i].moder_cnt = priv->rx_frames; - priv->rx_cq[i].moder_time = priv->rx_usecs; - err = mlx4_en_set_cq_moder(priv, &priv->rx_cq[i]); - if (err) - return err; + if (priv->port_up) { + for (i = 0; i < priv->rx_ring_num; i++) { + priv->rx_cq[i]->moder_cnt = priv->rx_frames; + priv->rx_cq[i]->moder_time = priv->rx_usecs; + priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; + err = mlx4_en_set_cq_moder(priv, priv->rx_cq[i]); + if (err) + return err; + } } + return 0; } static int mlx4_en_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *pause) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_dev *mdev = priv->mdev; int err; + if (pause->autoneg) + return -EOPNOTSUPP; + priv->prof->tx_pause = pause->tx_pause != 0; priv->prof->rx_pause = pause->rx_pause != 0; err = mlx4_SET_PORT_general(mdev->dev, priv->port, - priv->rx_mb_size + ETH_FCS_LEN, + priv->rx_skb_size + ETH_FCS_LEN, priv->prof->tx_pause, priv->prof->tx_ppp, priv->prof->rx_pause, priv->prof->rx_ppp); if (err) en_err(priv, "Failed setting pause params\n"); return err; } static void mlx4_en_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *pause) { struct mlx4_en_priv *priv = netdev_priv(dev); pause->tx_pause = priv->prof->tx_pause; pause->rx_pause = priv->prof->rx_pause; + pause->autoneg = mlx4_en_autoneg_get(dev); } +/* rtnl lock must be taken before calling */ +int mlx4_en_pre_config(struct mlx4_en_priv *priv) +{ +#ifdef CONFIG_RFS_ACCEL + struct cpu_rmap *rmap; + + if (!priv->dev->rx_cpu_rmap) + return 0; + + /* Disable RFS events + * Must have all RFS jobs flushed before freeing resources + */ + rmap = priv->dev->rx_cpu_rmap; + priv->dev->rx_cpu_rmap = NULL; + + rtnl_unlock(); + free_irq_cpu_rmap(rmap); + rtnl_lock(); + + if (priv->dev->rx_cpu_rmap) + return -EBUSY; /* another configuration completed while lock + * was free + */ + + /* Make sure all currently running filter_work are being processed + * Other work will return immediatly because of disable_rfs + */ + flush_workqueue(priv->mdev->workqueue); + +#endif + + return 0; +} + static int mlx4_en_set_ringparam(struct net_device *dev, struct ethtool_ringparam *param) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_dev *mdev = priv->mdev; u32 rx_size, tx_size; int port_up = 0; int err = 0; + int i, n_stats; + u64 *data = NULL; + if (!priv->port_up) + return -ENOMEM; + if (param->rx_jumbo_pending || param->rx_mini_pending) return -EINVAL; rx_size = roundup_pow_of_two(param->rx_pending); rx_size = max_t(u32, rx_size, MLX4_EN_MIN_RX_SIZE); rx_size = min_t(u32, rx_size, MLX4_EN_MAX_RX_SIZE); tx_size = roundup_pow_of_two(param->tx_pending); tx_size = max_t(u32, tx_size, MLX4_EN_MIN_TX_SIZE); tx_size = min_t(u32, tx_size, MLX4_EN_MAX_TX_SIZE); - if (rx_size == (priv->port_up ? priv->rx_ring[0].actual_size : - priv->rx_ring[0].size) && - tx_size == priv->tx_ring[0].size) + if (rx_size == (priv->port_up ? priv->rx_ring[0]->actual_size : + priv->rx_ring[0]->size) && + tx_size == priv->tx_ring[0]->size) return 0; + err = mlx4_en_pre_config(priv); + if (err) + return err; mutex_lock(&mdev->state_lock); if (priv->port_up) { port_up = 1; mlx4_en_stop_port(dev); } + /* Cache port statistics */ + n_stats = mlx4_en_get_sset_count(dev, ETH_SS_STATS); + if (n_stats > 0) { + data = kmalloc(n_stats * sizeof(u64), GFP_KERNEL); + if (data) + mlx4_en_get_ethtool_stats(dev, NULL, data); + } + mlx4_en_free_resources(priv); priv->prof->tx_ring_size = tx_size; priv->prof->rx_ring_size = rx_size; err = mlx4_en_alloc_resources(priv); if (err) { en_err(priv, "Failed reallocating port resources\n"); goto out; } + + /* Restore port statistics */ + if (n_stats > 0 && data) + mlx4_en_restore_ethtool_stats(priv, data); + if (port_up) { err = mlx4_en_start_port(dev); - if (err) + if (err) { en_err(priv, "Failed starting port\n"); + goto out; + } + + for (i = 0; i < priv->rx_ring_num; i++) { + priv->rx_cq[i]->moder_cnt = priv->rx_frames; + priv->rx_cq[i]->moder_time = priv->rx_usecs; + priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; + err = mlx4_en_set_cq_moder(priv, priv->rx_cq[i]); + if (err) + goto out; + } } out: + kfree(data); mutex_unlock(&mdev->state_lock); return err; } static void mlx4_en_get_ringparam(struct net_device *dev, struct ethtool_ringparam *param) { struct mlx4_en_priv *priv = netdev_priv(dev); + if (!priv->port_up) + return; + memset(param, 0, sizeof(*param)); param->rx_max_pending = MLX4_EN_MAX_RX_SIZE; param->tx_max_pending = MLX4_EN_MAX_TX_SIZE; param->rx_pending = priv->port_up ? - priv->rx_ring[0].actual_size : priv->rx_ring[0].size; - param->tx_pending = priv->tx_ring[0].size; + priv->rx_ring[0]->actual_size : priv->rx_ring[0]->size; + param->tx_pending = priv->tx_ring[0]->size; } +static u32 mlx4_en_get_rxfh_indir_size(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + + return priv->rx_ring_num; +} + +static int mlx4_en_get_rxfh_indir(struct net_device *dev, u32 *ring_index) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_rss_map *rss_map = &priv->rss_map; + int rss_rings; + size_t n = priv->rx_ring_num; + int err = 0; + + rss_rings = priv->prof->rss_rings ?: priv->rx_ring_num; + rss_rings = 1 << ilog2(rss_rings); + + while (n--) { + ring_index[n] = rss_map->qps[n % rss_rings].qpn - + rss_map->base_qpn; + } + + return err; +} + +static int mlx4_en_set_rxfh_indir(struct net_device *dev, + const u32 *ring_index) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int port_up = 0; + int err = 0; + int i; + int rss_rings = 0; + + /* Calculate RSS table size and make sure flows are spread evenly + * between rings + */ + for (i = 0; i < priv->rx_ring_num; i++) { + if (i > 0 && !ring_index[i] && !rss_rings) + rss_rings = i; + + if (ring_index[i] != (i % (rss_rings ?: priv->rx_ring_num))) + return -EINVAL; + } + + if (!rss_rings) + rss_rings = priv->rx_ring_num; + + /* RSS table size must be an order of 2 */ + if (!is_power_of_2(rss_rings)) + return -EINVAL; + + mutex_lock(&mdev->state_lock); + if (priv->port_up) { + port_up = 1; + mlx4_en_stop_port(dev); + } + + priv->prof->rss_rings = rss_rings; + + if (port_up) { + err = mlx4_en_start_port(dev); + if (err) + en_err(priv, "Failed starting port\n"); + } + + mutex_unlock(&mdev->state_lock); + return err; +} + +#define all_zeros_or_all_ones(field) \ + ((field) == 0 || (field) == (__force typeof(field))-1) + +static int mlx4_en_validate_flow(struct net_device *dev, + struct mlx4_ethtool_rxnfc *cmd) +{ + struct ethtool_usrip4_spec *l3_mask; + struct ethtool_tcpip4_spec *l4_mask; + struct ethhdr *eth_mask; + + if (cmd->fs.location >= MAX_NUM_OF_FS_RULES) + return -EINVAL; + + if (cmd->fs.flow_type & FLOW_MAC_EXT) { + /* dest mac mask must be ff:ff:ff:ff:ff:ff */ + if (!is_broadcast_ether_addr(cmd->fs.m_ext.h_dest)) + return -EINVAL; + } + + switch (cmd->fs.flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) { + case TCP_V4_FLOW: + case UDP_V4_FLOW: + if (cmd->fs.m_u.tcp_ip4_spec.tos) + return -EINVAL; + l4_mask = &cmd->fs.m_u.tcp_ip4_spec; + /* don't allow mask which isn't all 0 or 1 */ + if (!all_zeros_or_all_ones(l4_mask->ip4src) || + !all_zeros_or_all_ones(l4_mask->ip4dst) || + !all_zeros_or_all_ones(l4_mask->psrc) || + !all_zeros_or_all_ones(l4_mask->pdst)) + return -EINVAL; + break; + case IP_USER_FLOW: + l3_mask = &cmd->fs.m_u.usr_ip4_spec; + if (l3_mask->l4_4_bytes || l3_mask->tos || l3_mask->proto || + cmd->fs.h_u.usr_ip4_spec.ip_ver != ETH_RX_NFC_IP4 || + (!l3_mask->ip4src && !l3_mask->ip4dst) || + !all_zeros_or_all_ones(l3_mask->ip4src) || + !all_zeros_or_all_ones(l3_mask->ip4dst)) + return -EINVAL; + break; + case ETHER_FLOW: + eth_mask = &cmd->fs.m_u.ether_spec; + /* source mac mask must not be set */ + if (!is_zero_ether_addr(eth_mask->h_source)) + return -EINVAL; + + /* dest mac mask must be ff:ff:ff:ff:ff:ff */ + if (!is_broadcast_ether_addr(eth_mask->h_dest)) + return -EINVAL; + + if (!all_zeros_or_all_ones(eth_mask->h_proto)) + return -EINVAL; + break; + default: + return -EINVAL; + } + + if ((cmd->fs.flow_type & FLOW_EXT)) { + if (cmd->fs.m_ext.vlan_etype || + !(cmd->fs.m_ext.vlan_tci == 0 || + cmd->fs.m_ext.vlan_tci == cpu_to_be16(0xfff))) + return -EINVAL; + if (cmd->fs.m_ext.vlan_tci) { + if (be16_to_cpu(cmd->fs.h_ext.vlan_tci) < + VLAN_MIN_VALUE || + be16_to_cpu(cmd->fs.h_ext.vlan_tci) > + VLAN_MAX_VALUE) + return -EINVAL; + } + } + + return 0; +} + +static int mlx4_en_ethtool_add_mac_rule(struct mlx4_ethtool_rxnfc *cmd, + struct list_head *rule_list_h, + struct mlx4_spec_list *spec_l2, + unsigned char *mac) +{ + int err = 0; + __be64 mac_msk = cpu_to_be64(MLX4_MAC_MASK << 16); + + spec_l2->id = MLX4_NET_TRANS_RULE_ID_ETH; + memcpy(spec_l2->eth.dst_mac_msk, &mac_msk, ETH_ALEN); + memcpy(spec_l2->eth.dst_mac, mac, ETH_ALEN); + + if ((cmd->fs.flow_type & FLOW_EXT) && cmd->fs.m_ext.vlan_tci) { + spec_l2->eth.vlan_id = cmd->fs.h_ext.vlan_tci; + spec_l2->eth.vlan_id_msk = cpu_to_be16(0xfff); + } + + list_add_tail(&spec_l2->list, rule_list_h); + + return err; +} + +static int mlx4_en_ethtool_add_mac_rule_by_ipv4(struct mlx4_en_priv *priv, + struct mlx4_ethtool_rxnfc *cmd, + struct list_head *rule_list_h, + struct mlx4_spec_list *spec_l2, + __be32 ipv4_dst) +{ + unsigned char mac[ETH_ALEN]; + + if (!ipv4_is_multicast(ipv4_dst)) { + if (cmd->fs.flow_type & FLOW_MAC_EXT) + memcpy(&mac, cmd->fs.h_ext.h_dest, ETH_ALEN); + else + memcpy(&mac, priv->dev->dev_addr, ETH_ALEN); + } else { + ip_eth_mc_map(ipv4_dst, mac); + } + + return mlx4_en_ethtool_add_mac_rule(cmd, rule_list_h, spec_l2, &mac[0]); +} + +static int add_ip_rule(struct mlx4_en_priv *priv, + struct mlx4_ethtool_rxnfc *cmd, + struct list_head *list_h) +{ + struct mlx4_spec_list *spec_l2 = NULL; + struct mlx4_spec_list *spec_l3 = NULL; + struct ethtool_usrip4_spec *l3_mask = &cmd->fs.m_u.usr_ip4_spec; + + spec_l3 = kzalloc(sizeof(*spec_l3), GFP_KERNEL); + spec_l2 = kzalloc(sizeof(*spec_l2), GFP_KERNEL); + if (!spec_l2 || !spec_l3) { + en_err(priv, "Fail to alloc ethtool rule.\n"); + kfree(spec_l2); + kfree(spec_l3); + return -ENOMEM; + } + + mlx4_en_ethtool_add_mac_rule_by_ipv4(priv, cmd, list_h, spec_l2, + cmd->fs.h_u. + usr_ip4_spec.ip4dst); + spec_l3->id = MLX4_NET_TRANS_RULE_ID_IPV4; + spec_l3->ipv4.src_ip = cmd->fs.h_u.usr_ip4_spec.ip4src; + if (l3_mask->ip4src) + spec_l3->ipv4.src_ip_msk = MLX4_BE_WORD_MASK; + spec_l3->ipv4.dst_ip = cmd->fs.h_u.usr_ip4_spec.ip4dst; + if (l3_mask->ip4dst) + spec_l3->ipv4.dst_ip_msk = MLX4_BE_WORD_MASK; + list_add_tail(&spec_l3->list, list_h); + + return 0; +} + +static int add_tcp_udp_rule(struct mlx4_en_priv *priv, + struct mlx4_ethtool_rxnfc *cmd, + struct list_head *list_h, int proto) +{ + struct mlx4_spec_list *spec_l2 = NULL; + struct mlx4_spec_list *spec_l3 = NULL; + struct mlx4_spec_list *spec_l4 = NULL; + struct ethtool_tcpip4_spec *l4_mask = &cmd->fs.m_u.tcp_ip4_spec; + + spec_l2 = kzalloc(sizeof(*spec_l2), GFP_KERNEL); + spec_l3 = kzalloc(sizeof(*spec_l3), GFP_KERNEL); + spec_l4 = kzalloc(sizeof(*spec_l4), GFP_KERNEL); + if (!spec_l2 || !spec_l3 || !spec_l4) { + en_err(priv, "Fail to alloc ethtool rule.\n"); + kfree(spec_l2); + kfree(spec_l3); + kfree(spec_l4); + return -ENOMEM; + } + + spec_l3->id = MLX4_NET_TRANS_RULE_ID_IPV4; + + if (proto == TCP_V4_FLOW) { + mlx4_en_ethtool_add_mac_rule_by_ipv4(priv, cmd, list_h, + spec_l2, + cmd->fs.h_u. + tcp_ip4_spec.ip4dst); + spec_l4->id = MLX4_NET_TRANS_RULE_ID_TCP; + spec_l3->ipv4.src_ip = cmd->fs.h_u.tcp_ip4_spec.ip4src; + spec_l3->ipv4.dst_ip = cmd->fs.h_u.tcp_ip4_spec.ip4dst; + spec_l4->tcp_udp.src_port = cmd->fs.h_u.tcp_ip4_spec.psrc; + spec_l4->tcp_udp.dst_port = cmd->fs.h_u.tcp_ip4_spec.pdst; + } else { + mlx4_en_ethtool_add_mac_rule_by_ipv4(priv, cmd, list_h, + spec_l2, + cmd->fs.h_u. + udp_ip4_spec.ip4dst); + spec_l4->id = MLX4_NET_TRANS_RULE_ID_UDP; + spec_l3->ipv4.src_ip = cmd->fs.h_u.udp_ip4_spec.ip4src; + spec_l3->ipv4.dst_ip = cmd->fs.h_u.udp_ip4_spec.ip4dst; + spec_l4->tcp_udp.src_port = cmd->fs.h_u.udp_ip4_spec.psrc; + spec_l4->tcp_udp.dst_port = cmd->fs.h_u.udp_ip4_spec.pdst; + } + + if (l4_mask->ip4src) + spec_l3->ipv4.src_ip_msk = MLX4_BE_WORD_MASK; + if (l4_mask->ip4dst) + spec_l3->ipv4.dst_ip_msk = MLX4_BE_WORD_MASK; + + if (l4_mask->psrc) + spec_l4->tcp_udp.src_port_msk = MLX4_BE_SHORT_MASK; + if (l4_mask->pdst) + spec_l4->tcp_udp.dst_port_msk = MLX4_BE_SHORT_MASK; + + list_add_tail(&spec_l3->list, list_h); + list_add_tail(&spec_l4->list, list_h); + + return 0; +} + +static int mlx4_en_ethtool_to_net_trans_rule(struct net_device *dev, + struct mlx4_ethtool_rxnfc *cmd, + struct list_head *rule_list_h) +{ + int err; + struct ethhdr *eth_spec; + struct mlx4_spec_list *spec_l2; + struct mlx4_en_priv *priv = netdev_priv(dev); + + err = mlx4_en_validate_flow(dev, cmd); + if (err) + return err; + + switch (cmd->fs.flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) { + case ETHER_FLOW: + spec_l2 = kzalloc(sizeof(*spec_l2), GFP_KERNEL); + if (!spec_l2) + return -ENOMEM; + + eth_spec = &cmd->fs.h_u.ether_spec; + mlx4_en_ethtool_add_mac_rule(cmd, rule_list_h, spec_l2, ð_spec->h_dest[0]); + spec_l2->eth.ether_type = eth_spec->h_proto; + if (eth_spec->h_proto) + spec_l2->eth.ether_type_enable = 1; + break; + case IP_USER_FLOW: + err = add_ip_rule(priv, cmd, rule_list_h); + break; + case TCP_V4_FLOW: + err = add_tcp_udp_rule(priv, cmd, rule_list_h, TCP_V4_FLOW); + break; + case UDP_V4_FLOW: + err = add_tcp_udp_rule(priv, cmd, rule_list_h, UDP_V4_FLOW); + break; + } + + return err; +} + +static int mlx4_en_flow_replace(struct net_device *dev, + struct mlx4_ethtool_rxnfc *cmd) +{ + int err; + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + struct ethtool_flow_id *loc_rule; + struct mlx4_spec_list *spec, *tmp_spec; + u32 qpn; + u64 reg_id; + + struct mlx4_net_trans_rule rule = { + .queue_mode = MLX4_NET_TRANS_Q_FIFO, + .exclusive = 0, + .allow_loopback = 1, + .promisc_mode = MLX4_FS_REGULAR, + }; + + rule.port = priv->port; + rule.priority = MLX4_DOMAIN_ETHTOOL | cmd->fs.location; + INIT_LIST_HEAD(&rule.list); + + /* Allow direct QP attaches if the EN_ETHTOOL_QP_ATTACH flag is set */ + if (cmd->fs.ring_cookie == RX_CLS_FLOW_DISC) + qpn = priv->drop_qp.qpn; + else if (cmd->fs.ring_cookie & EN_ETHTOOL_QP_ATTACH) { + qpn = cmd->fs.ring_cookie & (EN_ETHTOOL_QP_ATTACH - 1); + } else { + if (cmd->fs.ring_cookie >= priv->rx_ring_num) { + en_warn(priv, "rxnfc: RX ring (%llu) doesn't exist.\n", + cmd->fs.ring_cookie); + return -EINVAL; + } + qpn = priv->rss_map.qps[cmd->fs.ring_cookie].qpn; + if (!qpn) { + en_warn(priv, "rxnfc: RX ring (%llu) is inactive.\n", + cmd->fs.ring_cookie); + return -EINVAL; + } + } + rule.qpn = qpn; + err = mlx4_en_ethtool_to_net_trans_rule(dev, cmd, &rule.list); + if (err) + goto out_free_list; + + mutex_lock(&mdev->state_lock); + loc_rule = &priv->ethtool_rules[cmd->fs.location]; + if (loc_rule->id) { + err = mlx4_flow_detach(priv->mdev->dev, loc_rule->id); + if (err) { + en_err(priv, "Fail to detach network rule at location %d. registration id = %llx\n", + cmd->fs.location, loc_rule->id); + goto unlock; + } + loc_rule->id = 0; + memset(&loc_rule->flow_spec, 0, + sizeof(struct ethtool_rx_flow_spec)); + list_del(&loc_rule->list); + } + err = mlx4_flow_attach(priv->mdev->dev, &rule, ®_id); + if (err) { + en_err(priv, "Fail to attach network rule at location %d.\n", + cmd->fs.location); + goto unlock; + } + loc_rule->id = reg_id; + memcpy(&loc_rule->flow_spec, &cmd->fs, + sizeof(struct ethtool_rx_flow_spec)); + list_add_tail(&loc_rule->list, &priv->ethtool_list); + +unlock: + mutex_unlock(&mdev->state_lock); +out_free_list: + list_for_each_entry_safe(spec, tmp_spec, &rule.list, list) { + list_del(&spec->list); + kfree(spec); + } + return err; +} + +static int mlx4_en_flow_detach(struct net_device *dev, + struct mlx4_ethtool_rxnfc *cmd) +{ + int err = 0; + struct ethtool_flow_id *rule; + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + + if (cmd->fs.location >= MAX_NUM_OF_FS_RULES) + return -EINVAL; + + mutex_lock(&mdev->state_lock); + rule = &priv->ethtool_rules[cmd->fs.location]; + if (!rule->id) { + err = -ENOENT; + goto out; + } + + err = mlx4_flow_detach(priv->mdev->dev, rule->id); + if (err) { + en_err(priv, "Fail to detach network rule at location %d. registration id = 0x%llx\n", + cmd->fs.location, rule->id); + goto out; + } + rule->id = 0; + memset(&rule->flow_spec, 0, sizeof(struct ethtool_rx_flow_spec)); + + list_del(&rule->list); +out: + mutex_unlock(&mdev->state_lock); + return err; + +} + +static int mlx4_en_get_flow(struct net_device *dev, struct mlx4_ethtool_rxnfc *cmd, + int loc) +{ + int err = 0; + struct ethtool_flow_id *rule; + struct mlx4_en_priv *priv = netdev_priv(dev); + + if (loc < 0 || loc >= MAX_NUM_OF_FS_RULES) + return -EINVAL; + + rule = &priv->ethtool_rules[loc]; + if (rule->id) + memcpy(&cmd->fs, &rule->flow_spec, + sizeof(struct ethtool_rx_flow_spec)); + else + err = -ENOENT; + + return err; +} + +static int mlx4_en_get_num_flows(struct mlx4_en_priv *priv) +{ + + int i, res = 0; + for (i = 0; i < MAX_NUM_OF_FS_RULES; i++) { + if (priv->ethtool_rules[i].id) + res++; + } + return res; + +} + +static int mlx4_en_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *c, + u32 *rule_locs) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int err = 0; + int i = 0, priority = 0; + struct mlx4_ethtool_rxnfc *cmd = (struct mlx4_ethtool_rxnfc *)c; + + if ((cmd->cmd == ETHTOOL_GRXCLSRLCNT || + cmd->cmd == ETHTOOL_GRXCLSRULE || + cmd->cmd == ETHTOOL_GRXCLSRLALL) && + (mdev->dev->caps.steering_mode != + MLX4_STEERING_MODE_DEVICE_MANAGED || !priv->port_up)) + return -EINVAL; + + switch (cmd->cmd) { + case ETHTOOL_GRXRINGS: + cmd->data = priv->rx_ring_num; + break; + case ETHTOOL_GRXCLSRLCNT: + cmd->rule_cnt = mlx4_en_get_num_flows(priv); + break; + case ETHTOOL_GRXCLSRULE: + err = mlx4_en_get_flow(dev, cmd, cmd->fs.location); + break; + case ETHTOOL_GRXCLSRLALL: + while ((!err || err == -ENOENT) && priority < cmd->rule_cnt) { + err = mlx4_en_get_flow(dev, cmd, i); + if (!err) + rule_locs[priority++] = i; + i++; + } + err = 0; + break; + default: + err = -EOPNOTSUPP; + break; + } + + return err; +} + +static int mlx4_en_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *c) +{ + int err = 0; + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_ethtool_rxnfc *cmd = (struct mlx4_ethtool_rxnfc *)c; + + if (mdev->dev->caps.steering_mode != + MLX4_STEERING_MODE_DEVICE_MANAGED || !priv->port_up) + return -EINVAL; + + switch (cmd->cmd) { + case ETHTOOL_SRXCLSRLINS: + err = mlx4_en_flow_replace(dev, cmd); + break; + case ETHTOOL_SRXCLSRLDEL: + err = mlx4_en_flow_detach(dev, cmd); + break; + default: + en_warn(priv, "Unsupported ethtool command. (%d)\n", cmd->cmd); + return -EINVAL; + } + + return err; +} + +static void mlx4_en_get_channels(struct net_device *dev, + struct ethtool_channels *channel) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + + memset(channel, 0, sizeof(*channel)); + + channel->max_rx = MAX_RX_RINGS; + channel->max_tx = MLX4_EN_MAX_TX_RING_P_UP; + + channel->rx_count = priv->rx_ring_num; + channel->tx_count = priv->tx_ring_num / MLX4_EN_NUM_UP; +} + +static int mlx4_en_set_channels(struct net_device *dev, + struct ethtool_channels *channel) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int port_up = 0; + int i; + int err = 0; + + if (channel->other_count || channel->combined_count || + channel->tx_count > MLX4_EN_MAX_TX_RING_P_UP || + channel->rx_count > MAX_RX_RINGS || + !channel->tx_count || !channel->rx_count) + return -EINVAL; + + err = mlx4_en_pre_config(priv); + if (err) + return err; + + mutex_lock(&mdev->state_lock); + if (priv->port_up) { + port_up = 1; + mlx4_en_stop_port(dev); + } + + mlx4_en_free_resources(priv); + + priv->num_tx_rings_p_up = channel->tx_count; + priv->tx_ring_num = channel->tx_count * MLX4_EN_NUM_UP; + priv->rx_ring_num = channel->rx_count; + + err = mlx4_en_alloc_resources(priv); + if (err) { + en_err(priv, "Failed reallocating port resources\n"); + goto out; + } + + netif_set_real_num_tx_queues(dev, priv->tx_ring_num); + netif_set_real_num_rx_queues(dev, priv->rx_ring_num); + + mlx4_en_setup_tc(dev, MLX4_EN_NUM_UP); + + en_warn(priv, "Using %d TX rings\n", priv->tx_ring_num); + en_warn(priv, "Using %d RX rings\n", priv->rx_ring_num); + + if (port_up) { + err = mlx4_en_start_port(dev); + if (err) + en_err(priv, "Failed starting port\n"); + + for (i = 0; i < priv->rx_ring_num; i++) { + priv->rx_cq[i]->moder_cnt = priv->rx_frames; + priv->rx_cq[i]->moder_time = priv->rx_usecs; + priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; + err = mlx4_en_set_cq_moder(priv, priv->rx_cq[i]); + if (err) + goto out; + } + } + +out: + mutex_unlock(&mdev->state_lock); + return err; +} + +static int mlx4_en_get_ts_info(struct net_device *dev, + struct ethtool_ts_info *info) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int ret; + + ret = ethtool_op_get_ts_info(dev, info); + if (ret) + return ret; + + if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) { + info->so_timestamping |= + SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + + info->tx_types = + (1 << HWTSTAMP_TX_OFF) | + (1 << HWTSTAMP_TX_ON); + + info->rx_filters = + (1 << HWTSTAMP_FILTER_NONE) | + (1 << HWTSTAMP_FILTER_ALL); + } + + return ret; +} + const struct ethtool_ops mlx4_en_ethtool_ops = { .get_drvinfo = mlx4_en_get_drvinfo, .get_settings = mlx4_en_get_settings, .set_settings = mlx4_en_set_settings, -#ifdef NETIF_F_TSO - .get_tso = mlx4_en_get_tso, - .set_tso = mlx4_en_set_tso, -#endif - .get_sg = ethtool_op_get_sg, - .set_sg = ethtool_op_set_sg, .get_link = ethtool_op_get_link, - .get_rx_csum = mlx4_en_get_rx_csum, - .set_rx_csum = mlx4_en_set_rx_csum, - .get_tx_csum = ethtool_op_get_tx_csum, - .set_tx_csum = ethtool_op_set_tx_ipv6_csum, .get_strings = mlx4_en_get_strings, .get_sset_count = mlx4_en_get_sset_count, .get_ethtool_stats = mlx4_en_get_ethtool_stats, .self_test = mlx4_en_self_test, .get_wol = mlx4_en_get_wol, .set_wol = mlx4_en_set_wol, .get_msglevel = mlx4_en_get_msglevel, .set_msglevel = mlx4_en_set_msglevel, .get_coalesce = mlx4_en_get_coalesce, .set_coalesce = mlx4_en_set_coalesce, .get_pauseparam = mlx4_en_get_pauseparam, .set_pauseparam = mlx4_en_set_pauseparam, .get_ringparam = mlx4_en_get_ringparam, .set_ringparam = mlx4_en_set_ringparam, - .get_flags = ethtool_op_get_flags, - .set_flags = ethtool_op_set_flags, + .get_rxnfc = mlx4_en_get_rxnfc, + .set_rxnfc = mlx4_en_set_rxnfc, + .get_rxfh_indir_size = mlx4_en_get_rxfh_indir_size, + .get_rxfh_indir = mlx4_en_get_rxfh_indir, + .set_rxfh_indir = mlx4_en_set_rxfh_indir, + .get_channels = mlx4_en_get_channels, + .set_channels = mlx4_en_set_channels, + .get_ts_info = mlx4_en_get_ts_info, }; Index: stable/9/sys/ofed/drivers/net/mlx4/en_main.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/en_main.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/en_main.c (revision 273864) @@ -1,383 +1,351 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ #include #include #include +#include #include #include #include #include "mlx4_en.h" MODULE_AUTHOR("Liran Liss, Yevgeny Petrilin"); MODULE_DESCRIPTION("Mellanox ConnectX HCA Ethernet driver"); MODULE_LICENSE("Dual BSD/GPL"); MODULE_VERSION(DRV_VERSION " ("DRV_RELDATE")"); static const char mlx4_en_version[] = DRV_NAME ": Mellanox ConnectX HCA Ethernet driver v" DRV_VERSION " (" DRV_RELDATE ")\n"; #define MLX4_EN_PARM_INT(X, def_val, desc) \ static unsigned int X = def_val;\ module_param(X , uint, 0444); \ MODULE_PARM_DESC(X, desc); /* * Device scope module parameters */ - -/* Enable RSS TCP traffic */ -MLX4_EN_PARM_INT(tcp_rss, 1, - "Enable RSS for incomming TCP traffic or disabled (0)"); /* Enable RSS UDP traffic */ MLX4_EN_PARM_INT(udp_rss, 1, - "Enable RSS for incomming UDP traffic or disabled (0)"); + "Enable RSS for incoming UDP traffic"); -/* Number of LRO sessions per Rx ring (rounded up to a power of two) */ -MLX4_EN_PARM_INT(num_lro, MLX4_EN_MAX_LRO_DESCRIPTORS, - "Number of LRO sessions per ring or disabled (0)"); - -/* Allow reassembly of fragmented IP packets */ -MLX4_EN_PARM_INT(ip_reasm, 1, "Allow reassembly of fragmented IP packets (!0)"); - /* Priority pausing */ MLX4_EN_PARM_INT(pfctx, 0, "Priority based Flow Control policy on TX[7:0]." " Per priority bit mask"); MLX4_EN_PARM_INT(pfcrx, 0, "Priority based Flow Control policy on RX[7:0]." " Per priority bit mask"); +#define MAX_PFC_TX 0xff +#define MAX_PFC_RX 0xff + + static int mlx4_en_get_profile(struct mlx4_en_dev *mdev) { struct mlx4_en_profile *params = &mdev->profile; int i; - params->tcp_rss = tcp_rss; params->udp_rss = udp_rss; - if (params->udp_rss && !mdev->dev->caps.udp_rss) { + params->num_tx_rings_p_up = min_t(int, mp_ncpus, + MLX4_EN_MAX_TX_RING_P_UP); + if (params->udp_rss && !(mdev->dev->caps.flags + & MLX4_DEV_CAP_FLAG_UDP_RSS)) { mlx4_warn(mdev, "UDP RSS is not supported on this device.\n"); params->udp_rss = 0; } - params->num_lro = min_t(int, num_lro , MLX4_EN_MAX_LRO_DESCRIPTORS); - params->ip_reasm = ip_reasm; for (i = 1; i <= MLX4_MAX_PORTS; i++) { params->prof[i].rx_pause = 1; params->prof[i].rx_ppp = pfcrx; params->prof[i].tx_pause = 1; params->prof[i].tx_ppp = pfctx; params->prof[i].tx_ring_size = MLX4_EN_DEF_TX_RING_SIZE; params->prof[i].rx_ring_size = MLX4_EN_DEF_RX_RING_SIZE; - params->prof[i].tx_ring_num = MLX4_EN_NUM_HASH_RINGS + 1 + - (!!pfcrx) * MLX4_EN_NUM_PPP_RINGS; + params->prof[i].tx_ring_num = params->num_tx_rings_p_up * + MLX4_EN_NUM_UP; + params->prof[i].rss_rings = 0; } return 0; } -static void *get_netdev(struct mlx4_dev *dev, void *ctx, u8 port) +static void *mlx4_en_get_netdev(struct mlx4_dev *dev, void *ctx, u8 port) { struct mlx4_en_dev *endev = ctx; return endev->pndev[port]; } static void mlx4_en_event(struct mlx4_dev *dev, void *endev_ptr, - enum mlx4_dev_event event, int port) + enum mlx4_dev_event event, unsigned long port) { struct mlx4_en_dev *mdev = (struct mlx4_en_dev *) endev_ptr; struct mlx4_en_priv *priv; - if (!mdev->pndev[port]) - return; - - priv = netdev_priv(mdev->pndev[port]); switch (event) { case MLX4_DEV_EVENT_PORT_UP: case MLX4_DEV_EVENT_PORT_DOWN: + if (!mdev->pndev[port]) + return; + priv = netdev_priv(mdev->pndev[port]); /* To prevent races, we poll the link state in a separate task rather than changing it here */ priv->link_state = event; queue_work(mdev->workqueue, &priv->linkstate_task); break; case MLX4_DEV_EVENT_CATASTROPHIC_ERROR: mlx4_err(mdev, "Internal error detected, restarting device\n"); break; + case MLX4_DEV_EVENT_SLAVE_INIT: + case MLX4_DEV_EVENT_SLAVE_SHUTDOWN: + break; default: - mlx4_warn(mdev, "Unhandled event: %d\n", event); + if (port < 1 || port > dev->caps.num_ports || + !mdev->pndev[port]) + return; + mlx4_warn(mdev, "Unhandled event %d for port %d\n", event, + (int) port); } } static void mlx4_en_remove(struct mlx4_dev *dev, void *endev_ptr) { struct mlx4_en_dev *mdev = endev_ptr; - int i; + int i, ret; mutex_lock(&mdev->state_lock); mdev->device_up = false; mutex_unlock(&mdev->state_lock); mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) if (mdev->pndev[i]) mlx4_en_destroy_netdev(mdev->pndev[i]); flush_workqueue(mdev->workqueue); destroy_workqueue(mdev->workqueue); - mlx4_mr_free(dev, &mdev->mr); + ret = mlx4_mr_free(dev, &mdev->mr); + if (ret) + mlx4_err(mdev, "Error deregistering MR. The system may have become unstable."); + iounmap(mdev->uar_map); mlx4_uar_free(dev, &mdev->priv_uar); mlx4_pd_free(dev, mdev->priv_pdn); - sx_destroy(&mdev->state_lock.sx); - mtx_destroy(&mdev->uar_lock.m); kfree(mdev); } static void *mlx4_en_add(struct mlx4_dev *dev) { - static int mlx4_en_version_printed; struct mlx4_en_dev *mdev; int i; int err; - if (!mlx4_en_version_printed) { - printk(KERN_INFO "%s", mlx4_en_version); - mlx4_en_version_printed++; - } + printk_once(KERN_INFO "%s", mlx4_en_version); mdev = kzalloc(sizeof *mdev, GFP_KERNEL); if (!mdev) { dev_err(&dev->pdev->dev, "Device struct alloc failed, " "aborting.\n"); err = -ENOMEM; goto err_free_res; } if (mlx4_pd_alloc(dev, &mdev->priv_pdn)) goto err_free_dev; if (mlx4_uar_alloc(dev, &mdev->priv_uar)) goto err_pd; - mtx_init(&mdev->uar_lock.m, "mlx4 uar", NULL, MTX_DEF); - mdev->uar_map = ioremap(mdev->priv_uar.pfn << PAGE_SHIFT, PAGE_SIZE); + mdev->uar_map = ioremap((phys_addr_t) mdev->priv_uar.pfn << PAGE_SHIFT, + PAGE_SIZE); if (!mdev->uar_map) goto err_uar; + spin_lock_init(&mdev->uar_lock); mdev->dev = dev; mdev->dma_device = &(dev->pdev->dev); mdev->pdev = dev->pdev; mdev->device_up = false; mdev->LSO_support = !!(dev->caps.flags & (1 << 15)); if (!mdev->LSO_support) mlx4_warn(mdev, "LSO not supported, please upgrade to later " "FW version to enable LSO\n"); if (mlx4_mr_alloc(mdev->dev, mdev->priv_pdn, 0, ~0ull, MLX4_PERM_LOCAL_WRITE | MLX4_PERM_LOCAL_READ, 0, 0, &mdev->mr)) { mlx4_err(mdev, "Failed allocating memory region\n"); - goto err_uar; + goto err_map; } if (mlx4_mr_enable(mdev->dev, &mdev->mr)) { mlx4_err(mdev, "Failed enabling memory region\n"); goto err_mr; } /* Build device profile according to supplied module parameters */ err = mlx4_en_get_profile(mdev); if (err) { mlx4_err(mdev, "Bad module parameters, aborting.\n"); goto err_mr; } - /* Configure wich ports to start according to module parameters */ + /* Configure which ports to start according to module parameters */ mdev->port_cnt = 0; mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) mdev->port_cnt++; - /* If we did not receive an explicit number of Rx rings, default to - * the number of completion vectors populated by the mlx4_core */ - mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { - mlx4_info(mdev, "Using %d tx rings for port:%d\n", - mdev->profile.prof[i].tx_ring_num, i); - mdev->profile.prof[i].rx_ring_num = rounddown_pow_of_two( - min_t(int, dev->caps.num_comp_vectors, MAX_RX_RINGS)); - mlx4_info(mdev, "Defaulting to %d rx rings for port:%d\n", - mdev->profile.prof[i].rx_ring_num, i); + mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { + if (!dev->caps.comp_pool) { + mdev->profile.prof[i].rx_ring_num = + rounddown_pow_of_two(max_t(int, MIN_RX_RINGS, + min_t(int, + dev->caps.num_comp_vectors, + DEF_RX_RINGS))); + } else { + mdev->profile.prof[i].rx_ring_num = rounddown_pow_of_two( + min_t(int, dev->caps.comp_pool/ + dev->caps.num_ports - 1 , MAX_MSIX_P_PORT - 1)); + } } /* Create our own workqueue for reset/multicast tasks * Note: we cannot use the shared workqueue because of deadlocks caused * by the rtnl lock */ mdev->workqueue = create_singlethread_workqueue("mlx4_en"); if (!mdev->workqueue) { err = -ENOMEM; goto err_mr; } /* At this stage all non-port specific tasks are complete: * mark the card state as up */ - sx_init(&mdev->state_lock.sx, "mlxen state"); + mutex_init(&mdev->state_lock); mdev->device_up = true; /* Setup ports */ /* Create a netdev for each port */ mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { mlx4_info(mdev, "Activating port:%d\n", i); - if (mlx4_en_init_netdev(mdev, i, &mdev->profile.prof[i])) { + if (mlx4_en_init_netdev(mdev, i, &mdev->profile.prof[i])) mdev->pndev[i] = NULL; - goto err_free_netdev; - } } + return mdev; - -err_free_netdev: - mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { - if (mdev->pndev[i]) - mlx4_en_destroy_netdev(mdev->pndev[i]); - } - - mutex_lock(&mdev->state_lock); - mdev->device_up = false; - mutex_unlock(&mdev->state_lock); - flush_workqueue(mdev->workqueue); - - /* Stop event queue before we drop down to release shared SW state */ - destroy_workqueue(mdev->workqueue); - err_mr: - mlx4_mr_free(dev, &mdev->mr); + err = mlx4_mr_free(dev, &mdev->mr); + if (err) + mlx4_err(mdev, "Error deregistering MR. The system may have become unstable."); +err_map: + if (mdev->uar_map) + iounmap(mdev->uar_map); err_uar: - mtx_destroy(&mdev->uar_lock.m); mlx4_uar_free(dev, &mdev->priv_uar); err_pd: mlx4_pd_free(dev, mdev->priv_pdn); err_free_dev: kfree(mdev); err_free_res: return NULL; } -enum mlx4_query_reply mlx4_en_query(void *endev_ptr, void *int_dev) +static struct mlx4_interface mlx4_en_interface = { + .add = mlx4_en_add, + .remove = mlx4_en_remove, + .event = mlx4_en_event, + .get_dev = mlx4_en_get_netdev, + .protocol = MLX4_PROT_ETH, +}; + +static void mlx4_en_verify_params(void) { - struct mlx4_en_dev *mdev = endev_ptr; - struct net_device *netdev = int_dev; - int p; - - for (p = 1; p <= MLX4_MAX_PORTS; ++p) - if (mdev->pndev[p] == netdev) - return p; + if (pfctx > MAX_PFC_TX) { + pr_warn("mlx4_en: WARNING: illegal module parameter pfctx 0x%x - " + "should be in range 0-0x%x, will be changed to default (0)\n", + pfctx, MAX_PFC_TX); + pfctx = 0; + } - return MLX4_QUERY_NOT_MINE; + if (pfcrx > MAX_PFC_RX) { + pr_warn("mlx4_en: WARNING: illegal module parameter pfcrx 0x%x - " + "should be in range 0-0x%x, will be changed to default (0)\n", + pfcrx, MAX_PFC_RX); + pfcrx = 0; + } } -#if 0 -static struct pci_device_id mlx4_en_pci_table[] = { - { PCI_VDEVICE(MELLANOX, 0x6340) }, /* MT25408 "Hermon" SDR */ - { PCI_VDEVICE(MELLANOX, 0x634a) }, /* MT25408 "Hermon" DDR */ - { PCI_VDEVICE(MELLANOX, 0x6354) }, /* MT25408 "Hermon" QDR */ - { PCI_VDEVICE(MELLANOX, 0x6732) }, /* MT25408 "Hermon" DDR PCIe gen2 */ - { PCI_VDEVICE(MELLANOX, 0x673c) }, /* MT25408 "Hermon" QDR PCIe gen2 */ - { PCI_VDEVICE(MELLANOX, 0x6368) }, /* MT25408 "Hermon" EN 10GigE */ - { PCI_VDEVICE(MELLANOX, 0x6750) }, /* MT25408 "Hermon" EN 10GigE PCIe gen2 */ - { PCI_VDEVICE(MELLANOX, 0x6372) }, /* MT25458 ConnectX EN 10GBASE-T 10GigE */ - { PCI_VDEVICE(MELLANOX, 0x675a) }, /* MT25458 ConnectX EN 10GBASE-T+Gen2 10GigE */ - { PCI_VDEVICE(MELLANOX, 0x6764) }, /* MT26468 ConnectX EN 10GigE PCIe gen2 */ - { PCI_VDEVICE(MELLANOX, 0x6746) }, /* MT26438 ConnectX VPI PCIe 2.0 5GT/s - IB QDR / 10GigE Virt+ */ - { PCI_VDEVICE(MELLANOX, 0x676e) }, /* MT26478 ConnectX EN 40GigE PCIe 2.0 5GT/s */ - { PCI_VDEVICE(MELLANOX, 0x6778) }, /* MT26488 ConnectX VPI PCIe 2.0 5GT/s - IB DDR / 10GigE Virt+ */ - { PCI_VDEVICE(MELLANOX, 0x1000) }, - { PCI_VDEVICE(MELLANOX, 0x1001) }, - { PCI_VDEVICE(MELLANOX, 0x1002) }, - { PCI_VDEVICE(MELLANOX, 0x1003) }, - { PCI_VDEVICE(MELLANOX, 0x1004) }, - { PCI_VDEVICE(MELLANOX, 0x1005) }, - { PCI_VDEVICE(MELLANOX, 0x1006) }, - { PCI_VDEVICE(MELLANOX, 0x1007) }, - { PCI_VDEVICE(MELLANOX, 0x1008) }, - { PCI_VDEVICE(MELLANOX, 0x1009) }, - { PCI_VDEVICE(MELLANOX, 0x100a) }, - { PCI_VDEVICE(MELLANOX, 0x100b) }, - { PCI_VDEVICE(MELLANOX, 0x100c) }, - { PCI_VDEVICE(MELLANOX, 0x100d) }, - { PCI_VDEVICE(MELLANOX, 0x100e) }, - { PCI_VDEVICE(MELLANOX, 0x100f) }, - { 0, } -}; -MODULE_DEVICE_TABLE(pci, mlx4_en_pci_table); -#endif - -static struct mlx4_interface mlx4_en_interface = { - .add = mlx4_en_add, - .remove = mlx4_en_remove, - .event = mlx4_en_event, - .query = mlx4_en_query, - .get_prot_dev = get_netdev, - .protocol = MLX4_PROT_EN, -}; - static int __init mlx4_en_init(void) { + mlx4_en_verify_params(); + +#ifdef CONFIG_DEBUG_FS + int err = 0; + err = mlx4_en_register_debugfs(); + if (err) + pr_err(KERN_ERR "Failed to register debugfs\n"); +#endif return mlx4_register_interface(&mlx4_en_interface); } static void __exit mlx4_en_cleanup(void) { mlx4_unregister_interface(&mlx4_en_interface); +#ifdef CONFIG_DEBUG_FS + mlx4_en_unregister_debugfs(); +#endif } module_init(mlx4_en_init); module_exit(mlx4_en_cleanup); #undef MODULE_VERSION #include static int mlxen_evhand(module_t mod, int event, void *arg) { return (0); } static moduledata_t mlxen_mod = { .name = "mlxen", .evhand = mlxen_evhand, }; DECLARE_MODULE(mlxen, mlxen_mod, SI_SUB_OFED_PREINIT, SI_ORDER_ANY); MODULE_DEPEND(mlxen, mlx4, 1, 1, 1); Index: stable/9/sys/ofed/drivers/net/mlx4/en_netdev.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/en_netdev.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/en_netdev.c (revision 273864) @@ -1,1636 +1,2577 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ -#include "mlx4_en.h" +#include +#include +#include +#ifdef CONFIG_NET_RX_BUSY_POLL +#include +#endif +#include +#include + #include #include #include #include -#include -#include -#include #include +#include -static void mlx4_en_init_locked(struct mlx4_en_priv *priv); +#include "mlx4_en.h" +#include "en_port.h" + static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv); +static void mlx4_en_sysctl_conf(struct mlx4_en_priv *priv); +static int mlx4_en_unit; +#ifdef CONFIG_NET_RX_BUSY_POLL +/* must be called with local_bh_disable()d */ +static int mlx4_en_low_latency_recv(struct napi_struct *napi) +{ + struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi); + struct net_device *dev = cq->dev; + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_rx_ring *rx_ring = priv->rx_ring[cq->ring]; + int done; + + if (!priv->port_up) + return LL_FLUSH_FAILED; + + if (!mlx4_en_cq_lock_poll(cq)) + return LL_FLUSH_BUSY; + + done = mlx4_en_process_rx_cq(dev, cq, 4); +#ifdef LL_EXTENDED_STATS + if (done) + rx_ring->cleaned += done; + else + rx_ring->misses++; +#endif + + mlx4_en_cq_unlock_poll(cq); + + return done; +} +#endif /* CONFIG_NET_RX_BUSY_POLL */ + +#ifdef CONFIG_RFS_ACCEL + +struct mlx4_en_filter { + struct list_head next; + struct work_struct work; + + u8 ip_proto; + __be32 src_ip; + __be32 dst_ip; + __be16 src_port; + __be16 dst_port; + + int rxq_index; + struct mlx4_en_priv *priv; + u32 flow_id; /* RFS infrastructure id */ + int id; /* mlx4_en driver id */ + u64 reg_id; /* Flow steering API id */ + u8 activated; /* Used to prevent expiry before filter + * is attached + */ + struct hlist_node filter_chain; +}; + +static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv); + +static enum mlx4_net_trans_rule_id mlx4_ip_proto_to_trans_rule_id(u8 ip_proto) +{ + switch (ip_proto) { + case IPPROTO_UDP: + return MLX4_NET_TRANS_RULE_ID_UDP; + case IPPROTO_TCP: + return MLX4_NET_TRANS_RULE_ID_TCP; + default: + return -EPROTONOSUPPORT; + } +}; + +static void mlx4_en_filter_work(struct work_struct *work) +{ + struct mlx4_en_filter *filter = container_of(work, + struct mlx4_en_filter, + work); + struct mlx4_en_priv *priv = filter->priv; + struct mlx4_spec_list spec_tcp_udp = { + .id = mlx4_ip_proto_to_trans_rule_id(filter->ip_proto), + { + .tcp_udp = { + .dst_port = filter->dst_port, + .dst_port_msk = (__force __be16)-1, + .src_port = filter->src_port, + .src_port_msk = (__force __be16)-1, + }, + }, + }; + struct mlx4_spec_list spec_ip = { + .id = MLX4_NET_TRANS_RULE_ID_IPV4, + { + .ipv4 = { + .dst_ip = filter->dst_ip, + .dst_ip_msk = (__force __be32)-1, + .src_ip = filter->src_ip, + .src_ip_msk = (__force __be32)-1, + }, + }, + }; + struct mlx4_spec_list spec_eth = { + .id = MLX4_NET_TRANS_RULE_ID_ETH, + }; + struct mlx4_net_trans_rule rule = { + .list = LIST_HEAD_INIT(rule.list), + .queue_mode = MLX4_NET_TRANS_Q_LIFO, + .exclusive = 1, + .allow_loopback = 1, + .promisc_mode = MLX4_FS_REGULAR, + .port = priv->port, + .priority = MLX4_DOMAIN_RFS, + }; + int rc; + __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); + + if (spec_tcp_udp.id < 0) { + en_warn(priv, "RFS: ignoring unsupported ip protocol (%d)\n", + filter->ip_proto); + goto ignore; + } + list_add_tail(&spec_eth.list, &rule.list); + list_add_tail(&spec_ip.list, &rule.list); + list_add_tail(&spec_tcp_udp.list, &rule.list); + + rule.qpn = priv->rss_map.qps[filter->rxq_index].qpn; + memcpy(spec_eth.eth.dst_mac, priv->dev->dev_addr, ETH_ALEN); + memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); + + filter->activated = 0; + + if (filter->reg_id) { + rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id); + if (rc && rc != -ENOENT) + en_err(priv, "Error detaching flow. rc = %d\n", rc); + } + + rc = mlx4_flow_attach(priv->mdev->dev, &rule, &filter->reg_id); + if (rc) + en_err(priv, "Error attaching flow. err = %d\n", rc); + +ignore: + mlx4_en_filter_rfs_expire(priv); + + filter->activated = 1; +} + +static inline struct hlist_head * +filter_hash_bucket(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, + __be16 src_port, __be16 dst_port) +{ + unsigned long l; + int bucket_idx; + + l = (__force unsigned long)src_port | + ((__force unsigned long)dst_port << 2); + l ^= (__force unsigned long)(src_ip ^ dst_ip); + + bucket_idx = hash_long(l, MLX4_EN_FILTER_HASH_SHIFT); + + return &priv->filter_hash[bucket_idx]; +} + +static struct mlx4_en_filter * +mlx4_en_filter_alloc(struct mlx4_en_priv *priv, int rxq_index, __be32 src_ip, + __be32 dst_ip, u8 ip_proto, __be16 src_port, + __be16 dst_port, u32 flow_id) +{ + struct mlx4_en_filter *filter = NULL; + + filter = kzalloc(sizeof(struct mlx4_en_filter), GFP_ATOMIC); + if (!filter) + return NULL; + + filter->priv = priv; + filter->rxq_index = rxq_index; + INIT_WORK(&filter->work, mlx4_en_filter_work); + + filter->src_ip = src_ip; + filter->dst_ip = dst_ip; + filter->ip_proto = ip_proto; + filter->src_port = src_port; + filter->dst_port = dst_port; + + filter->flow_id = flow_id; + + filter->id = priv->last_filter_id++ % RPS_NO_FILTER; + + list_add_tail(&filter->next, &priv->filters); + hlist_add_head(&filter->filter_chain, + filter_hash_bucket(priv, src_ip, dst_ip, src_port, + dst_port)); + + return filter; +} + +static void mlx4_en_filter_free(struct mlx4_en_filter *filter) +{ + struct mlx4_en_priv *priv = filter->priv; + int rc; + + list_del(&filter->next); + + rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id); + if (rc && rc != -ENOENT) + en_err(priv, "Error detaching flow. rc = %d\n", rc); + + kfree(filter); +} + +static inline struct mlx4_en_filter * +mlx4_en_filter_find(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, + u8 ip_proto, __be16 src_port, __be16 dst_port) +{ + struct hlist_node *elem; + struct mlx4_en_filter *filter; + struct mlx4_en_filter *ret = NULL; + + hlist_for_each_entry(filter, elem, + filter_hash_bucket(priv, src_ip, dst_ip, + src_port, dst_port), + filter_chain) { + if (filter->src_ip == src_ip && + filter->dst_ip == dst_ip && + filter->ip_proto == ip_proto && + filter->src_port == src_port && + filter->dst_port == dst_port) { + ret = filter; + break; + } + } + + return ret; +} + +static int +mlx4_en_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, + u16 rxq_index, u32 flow_id) +{ + struct mlx4_en_priv *priv = netdev_priv(net_dev); + struct mlx4_en_filter *filter; + const struct iphdr *ip; + const __be16 *ports; + u8 ip_proto; + __be32 src_ip; + __be32 dst_ip; + __be16 src_port; + __be16 dst_port; + int nhoff = skb_network_offset(skb); + int ret = 0; + + if (skb->protocol != htons(ETH_P_IP)) + return -EPROTONOSUPPORT; + + ip = (const struct iphdr *)(skb->data + nhoff); + if (ip_is_fragment(ip)) + return -EPROTONOSUPPORT; + + if ((ip->protocol != IPPROTO_TCP) && (ip->protocol != IPPROTO_UDP)) + return -EPROTONOSUPPORT; + ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); + + ip_proto = ip->protocol; + src_ip = ip->saddr; + dst_ip = ip->daddr; + src_port = ports[0]; + dst_port = ports[1]; + + spin_lock_bh(&priv->filters_lock); + filter = mlx4_en_filter_find(priv, src_ip, dst_ip, ip_proto, + src_port, dst_port); + if (filter) { + if (filter->rxq_index == rxq_index) + goto out; + + filter->rxq_index = rxq_index; + } else { + filter = mlx4_en_filter_alloc(priv, rxq_index, + src_ip, dst_ip, ip_proto, + src_port, dst_port, flow_id); + if (!filter) { + ret = -ENOMEM; + goto err; + } + } + + queue_work(priv->mdev->workqueue, &filter->work); + +out: + ret = filter->id; +err: + spin_unlock_bh(&priv->filters_lock); + + return ret; +} + +void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring *rx_ring) +{ + struct mlx4_en_filter *filter, *tmp; + LIST_HEAD(del_list); + + spin_lock_bh(&priv->filters_lock); + list_for_each_entry_safe(filter, tmp, &priv->filters, next) { + list_move(&filter->next, &del_list); + hlist_del(&filter->filter_chain); + } + spin_unlock_bh(&priv->filters_lock); + + list_for_each_entry_safe(filter, tmp, &del_list, next) { + cancel_work_sync(&filter->work); + mlx4_en_filter_free(filter); + } +} + +static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv) +{ + struct mlx4_en_filter *filter = NULL, *tmp, *last_filter = NULL; + LIST_HEAD(del_list); + int i = 0; + + spin_lock_bh(&priv->filters_lock); + list_for_each_entry_safe(filter, tmp, &priv->filters, next) { + if (i > MLX4_EN_FILTER_EXPIRY_QUOTA) + break; + + if (filter->activated && + !work_pending(&filter->work) && + rps_may_expire_flow(priv->dev, + filter->rxq_index, filter->flow_id, + filter->id)) { + list_move(&filter->next, &del_list); + hlist_del(&filter->filter_chain); + } else + last_filter = filter; + + i++; + } + + if (last_filter && (&last_filter->next != priv->filters.next)) + list_move(&priv->filters, &last_filter->next); + + spin_unlock_bh(&priv->filters_lock); + + list_for_each_entry_safe(filter, tmp, &del_list, next) + mlx4_en_filter_free(filter); +} +#endif + static void mlx4_en_vlan_rx_add_vid(void *arg, struct net_device *dev, u16 vid) { struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int err; int idx; - u8 field; - if ((vid == 0) || (vid > 4095)) /* Invalid */ + if (arg != priv) return; + en_dbg(HW, priv, "adding VLAN:%d\n", vid); - idx = vid >> 5; - field = 1 << (vid & 0x1f); - spin_lock(&priv->vlan_lock); - priv->vlgrp_modified = true; - if (priv->vlan_unregister[idx] & field) - priv->vlan_unregister[idx] &= ~field; - else - priv->vlan_register[idx] |= field; - priv->vlans[idx] |= field; - spin_unlock(&priv->vlan_lock); + + set_bit(vid, priv->active_vlans); + + /* Add VID to port VLAN filter */ + mutex_lock(&mdev->state_lock); + if (mdev->device_up && priv->port_up) { + err = mlx4_SET_VLAN_FLTR(mdev->dev, priv); + if (err) + en_err(priv, "Failed configuring VLAN filter\n"); + } + if (mlx4_register_vlan(mdev->dev, priv->port, vid, &idx)) + en_dbg(HW, priv, "failed adding vlan %d\n", vid); + mutex_unlock(&mdev->state_lock); + } static void mlx4_en_vlan_rx_kill_vid(void *arg, struct net_device *dev, u16 vid) { struct mlx4_en_priv *priv = netdev_priv(dev); - int idx; - u8 field; + struct mlx4_en_dev *mdev = priv->mdev; + int err; - if ((vid == 0) || (vid > 4095)) /* Invalid */ + if (arg != priv) return; + en_dbg(HW, priv, "Killing VID:%d\n", vid); - idx = vid >> 5; - field = 1 << (vid & 0x1f); - spin_lock(&priv->vlan_lock); - priv->vlgrp_modified = true; - if (priv->vlan_register[idx] & field) - priv->vlan_register[idx] &= ~field; - else - priv->vlan_unregister[idx] |= field; - priv->vlans[idx] &= ~field; - spin_unlock(&priv->vlan_lock); + + clear_bit(vid, priv->active_vlans); + + /* Remove VID from port VLAN filter */ + mutex_lock(&mdev->state_lock); + mlx4_unregister_vlan(mdev->dev, priv->port, vid); + + if (mdev->device_up && priv->port_up) { + err = mlx4_SET_VLAN_FLTR(mdev->dev, priv); + if (err) + en_err(priv, "Failed configuring VLAN filter\n"); + } + mutex_unlock(&mdev->state_lock); + } -u64 mlx4_en_mac_to_u64(u8 *addr) +static int mlx4_en_uc_steer_add(struct mlx4_en_priv *priv, + unsigned char *mac, int *qpn, u64 *reg_id) { - u64 mac = 0; - int i; + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_dev *dev = mdev->dev; + int err; - for (i = 0; i < ETHER_ADDR_LEN; i++) { - mac <<= 8; - mac |= addr[i]; + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_B0: { + struct mlx4_qp qp; + u8 gid[16] = {0}; + + qp.qpn = *qpn; + memcpy(&gid[10], mac, ETH_ALEN); + gid[5] = priv->port; + + err = mlx4_unicast_attach(dev, &qp, gid, 0, MLX4_PROT_ETH); + break; } - return mac; + case MLX4_STEERING_MODE_DEVICE_MANAGED: { + struct mlx4_spec_list spec_eth = { {NULL} }; + __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); + + struct mlx4_net_trans_rule rule = { + .queue_mode = MLX4_NET_TRANS_Q_FIFO, + .exclusive = 0, + .allow_loopback = 1, + .promisc_mode = MLX4_FS_REGULAR, + .priority = MLX4_DOMAIN_NIC, + }; + + rule.port = priv->port; + rule.qpn = *qpn; + INIT_LIST_HEAD(&rule.list); + + spec_eth.id = MLX4_NET_TRANS_RULE_ID_ETH; + memcpy(spec_eth.eth.dst_mac, mac, ETH_ALEN); + memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); + list_add_tail(&spec_eth.list, &rule.list); + + err = mlx4_flow_attach(dev, &rule, reg_id); + break; + } + default: + return -EINVAL; + } + if (err) + en_warn(priv, "Failed Attaching Unicast\n"); + + return err; } -static int mlx4_en_cache_mclist(struct net_device *dev, u64 **mcaddrp) +static void mlx4_en_uc_steer_release(struct mlx4_en_priv *priv, + unsigned char *mac, int qpn, u64 reg_id) { - struct ifmultiaddr *ifma; - u64 *mcaddr; - int cnt; - int i; + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_dev *dev = mdev->dev; - *mcaddrp = NULL; -restart: - cnt = 0; - if_maddr_rlock(dev); - TAILQ_FOREACH(ifma, &dev->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - if (((struct sockaddr_dl *)ifma->ifma_addr)->sdl_alen != - ETHER_ADDR_LEN) - continue; - cnt++; + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_B0: { + struct mlx4_qp qp; + u8 gid[16] = {0}; + + qp.qpn = qpn; + memcpy(&gid[10], mac, ETH_ALEN); + gid[5] = priv->port; + + mlx4_unicast_detach(dev, &qp, gid, MLX4_PROT_ETH); + break; } - if_maddr_runlock(dev); - if (cnt == 0) - return (0); - mcaddr = kmalloc(sizeof(u64) * cnt, GFP_KERNEL); - if (mcaddr == NULL) - return (0); - i = 0; - if_maddr_rlock(dev); - TAILQ_FOREACH(ifma, &dev->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - if (((struct sockaddr_dl *)ifma->ifma_addr)->sdl_alen != - ETHER_ADDR_LEN) - continue; - /* Make sure the list didn't grow. */ - if (i == cnt) { - if_maddr_runlock(dev); - kfree(mcaddr); - goto restart; - } - mcaddr[i++] = mlx4_en_mac_to_u64( - LLADDR((struct sockaddr_dl *)ifma->ifma_addr)); + case MLX4_STEERING_MODE_DEVICE_MANAGED: { + mlx4_flow_detach(dev, reg_id); + break; } - if_maddr_runlock(dev); - *mcaddrp = mcaddr; - return (i); + default: + en_err(priv, "Invalid steering mode.\n"); + } } - -static void mlx4_en_set_multicast(struct net_device *dev) +static int mlx4_en_get_qp(struct mlx4_en_priv *priv) { - struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_dev *dev = mdev->dev; + struct mlx4_mac_entry *entry; + int index = 0; + int err = 0; + u64 reg_id; + int *qpn = &priv->base_qpn; + u64 mac = mlx4_mac_to_u64(IF_LLADDR(priv->dev)); - if (!priv->port_up) - return; + en_dbg(DRV, priv, "Registering MAC: %pM for adding\n", + IF_LLADDR(priv->dev)); + index = mlx4_register_mac(dev, priv->port, mac); + if (index < 0) { + err = index; + en_err(priv, "Failed adding MAC: %pM\n", + IF_LLADDR(priv->dev)); + return err; + } - queue_work(priv->mdev->workqueue, &priv->mcast_task); + if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { + int base_qpn = mlx4_get_base_qpn(dev, priv->port); + *qpn = base_qpn + index; + return 0; + } + + err = mlx4_qp_reserve_range(dev, 1, 1, qpn, 0); + en_dbg(DRV, priv, "Reserved qp %d\n", *qpn); + if (err) { + en_err(priv, "Failed to reserve qp for mac registration\n"); + goto qp_err; + } + + err = mlx4_en_uc_steer_add(priv, IF_LLADDR(priv->dev), qpn, ®_id); + if (err) + goto steer_err; + + entry = kmalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) { + err = -ENOMEM; + goto alloc_err; + } + memcpy(entry->mac, IF_LLADDR(priv->dev), sizeof(entry->mac)); + entry->reg_id = reg_id; + + hlist_add_head(&entry->hlist, + &priv->mac_hash[entry->mac[MLX4_EN_MAC_HASH_IDX]]); + + return 0; + +alloc_err: + mlx4_en_uc_steer_release(priv, IF_LLADDR(priv->dev), *qpn, reg_id); + +steer_err: + mlx4_qp_release_range(dev, *qpn, 1); + +qp_err: + mlx4_unregister_mac(dev, priv->port, mac); + return err; } -static void mlx4_en_do_set_multicast(struct work_struct *work) +static void mlx4_en_put_qp(struct mlx4_en_priv *priv) { - struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, - mcast_task); - struct net_device *dev = priv->dev; struct mlx4_en_dev *mdev = priv->mdev; - int err; + struct mlx4_dev *dev = mdev->dev; + int qpn = priv->base_qpn; + u64 mac; - mutex_lock(&mdev->state_lock); - if (!mdev->device_up) { - en_dbg(HW, priv, "Card is not up, " - "ignoring multicast change.\n"); - goto out; + if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { + mac = mlx4_mac_to_u64(IF_LLADDR(priv->dev)); + en_dbg(DRV, priv, "Registering MAC: %pM for deleting\n", + IF_LLADDR(priv->dev)); + mlx4_unregister_mac(dev, priv->port, mac); + } else { + struct mlx4_mac_entry *entry; + struct hlist_node *n, *tmp; + struct hlist_head *bucket; + unsigned int i; + + for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) { + bucket = &priv->mac_hash[i]; + hlist_for_each_entry_safe(entry, n, tmp, bucket, hlist) { + mac = mlx4_mac_to_u64(entry->mac); + en_dbg(DRV, priv, "Registering MAC: %pM for deleting\n", + entry->mac); + mlx4_en_uc_steer_release(priv, entry->mac, + qpn, entry->reg_id); + + mlx4_unregister_mac(dev, priv->port, mac); + hlist_del(&entry->hlist); + kfree(entry); + } + } + + en_dbg(DRV, priv, "Releasing qp: port %d, qpn %d\n", + priv->port, qpn); + mlx4_qp_release_range(dev, qpn, 1); + priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC; } - if (!priv->port_up) { - en_dbg(HW, priv, "Port is down, " - "ignoring multicast change.\n"); - goto out; +} + +static void mlx4_en_clear_list(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_mc_list *tmp, *mc_to_del; + + list_for_each_entry_safe(mc_to_del, tmp, &priv->mc_list, list) { + list_del(&mc_to_del->list); + kfree(mc_to_del); } +} - /* - * Promsicuous mode: disable all filters +static void mlx4_en_cache_mclist(struct net_device *dev) +{ + struct ifmultiaddr *ifma; + struct mlx4_en_mc_list *tmp; + struct mlx4_en_priv *priv = netdev_priv(dev); + + TAILQ_FOREACH(ifma, &dev->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + if (((struct sockaddr_dl *)ifma->ifma_addr)->sdl_alen != + ETHER_ADDR_LEN) + continue; + /* Make sure the list didn't grow. */ + tmp = kzalloc(sizeof(struct mlx4_en_mc_list), GFP_ATOMIC); + memcpy(tmp->addr, + LLADDR((struct sockaddr_dl *)ifma->ifma_addr), ETH_ALEN); + list_add_tail(&tmp->list, &priv->mc_list); + } +} + +static void update_mclist_flags(struct mlx4_en_priv *priv, + struct list_head *dst, + struct list_head *src) +{ + struct mlx4_en_mc_list *dst_tmp, *src_tmp, *new_mc; + bool found; + + /* Find all the entries that should be removed from dst, + * These are the entries that are not found in src */ + list_for_each_entry(dst_tmp, dst, list) { + found = false; + list_for_each_entry(src_tmp, src, list) { + if (!memcmp(dst_tmp->addr, src_tmp->addr, ETH_ALEN)) { + found = true; + break; + } + } + if (!found) + dst_tmp->action = MCLIST_REM; + } - if (dev->if_flags & IFF_PROMISC) { - if (!(priv->flags & MLX4_EN_FLAG_PROMISC)) { - priv->flags |= MLX4_EN_FLAG_PROMISC; + /* Add entries that exist in src but not in dst + * mark them as need to add + */ + list_for_each_entry(src_tmp, src, list) { + found = false; + list_for_each_entry(dst_tmp, dst, list) { + if (!memcmp(dst_tmp->addr, src_tmp->addr, ETH_ALEN)) { + dst_tmp->action = MCLIST_NONE; + found = true; + break; + } + } + if (!found) { + new_mc = kmalloc(sizeof(struct mlx4_en_mc_list), + GFP_KERNEL); + if (!new_mc) { + en_err(priv, "Failed to allocate current multicast list\n"); + return; + } + memcpy(new_mc, src_tmp, + sizeof(struct mlx4_en_mc_list)); + new_mc->action = MCLIST_ADD; + list_add_tail(&new_mc->list, dst); + } + } +} - /* Enable promiscouos mode */ - err = mlx4_SET_PORT_qpn_calc(mdev->dev, priv->port, - priv->base_qpn, 1); +static void mlx4_en_set_rx_mode(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + + if (!priv->port_up) + return; + + queue_work(priv->mdev->workqueue, &priv->rx_mode_task); +} + +static void mlx4_en_set_promisc_mode(struct mlx4_en_priv *priv, + struct mlx4_en_dev *mdev) +{ + int err = 0; + if (!(priv->flags & MLX4_EN_FLAG_PROMISC)) { + priv->flags |= MLX4_EN_FLAG_PROMISC; + + /* Enable promiscouos mode */ + switch (mdev->dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: + err = mlx4_flow_steer_promisc_add(mdev->dev, + priv->port, + priv->base_qpn, + MLX4_FS_ALL_DEFAULT); if (err) - en_err(priv, "Failed enabling " - "promiscous mode\n"); + en_err(priv, "Failed enabling promiscuous mode\n"); + priv->flags |= MLX4_EN_FLAG_MC_PROMISC; + break; - /* Disable port multicast filter (unconditionally) */ - err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, - 0, MLX4_MCAST_DISABLE); + case MLX4_STEERING_MODE_B0: + err = mlx4_unicast_promisc_add(mdev->dev, + priv->base_qpn, + priv->port); if (err) - en_err(priv, "Failed disabling " - "multicast filter\n"); + en_err(priv, "Failed enabling unicast promiscuous mode\n"); - /* Disable port VLAN filter */ - err = mlx4_SET_VLAN_FLTR(mdev->dev, priv->port, NULL); + /* Add the default qp number as multicast + * promisc + */ + if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { + err = mlx4_multicast_promisc_add(mdev->dev, + priv->base_qpn, + priv->port); + if (err) + en_err(priv, "Failed enabling multicast promiscuous mode\n"); + priv->flags |= MLX4_EN_FLAG_MC_PROMISC; + } + break; + + case MLX4_STEERING_MODE_A0: + err = mlx4_SET_PORT_qpn_calc(mdev->dev, + priv->port, + priv->base_qpn, + 1); if (err) - en_err(priv, "Failed disabling VLAN filter\n"); + en_err(priv, "Failed enabling promiscuous mode\n"); + break; } - goto out; + + /* Disable port multicast filter (unconditionally) */ + err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, + 0, MLX4_MCAST_DISABLE); + if (err) + en_err(priv, "Failed disabling multicast filter\n"); } +} - /* - * Not in promiscous mode - */ +static void mlx4_en_clear_promisc_mode(struct mlx4_en_priv *priv, + struct mlx4_en_dev *mdev) +{ + int err = 0; - if (priv->flags & MLX4_EN_FLAG_PROMISC) { - priv->flags &= ~MLX4_EN_FLAG_PROMISC; + priv->flags &= ~MLX4_EN_FLAG_PROMISC; - /* Disable promiscouos mode */ - err = mlx4_SET_PORT_qpn_calc(mdev->dev, priv->port, - priv->base_qpn, 0); + /* Disable promiscouos mode */ + switch (mdev->dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: + err = mlx4_flow_steer_promisc_remove(mdev->dev, + priv->port, + MLX4_FS_ALL_DEFAULT); if (err) - en_err(priv, "Failed disabling promiscous mode\n"); + en_err(priv, "Failed disabling promiscuous mode\n"); + priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; + break; - /* Enable port VLAN filter */ - err = mlx4_SET_VLAN_FLTR(mdev->dev, priv->port, priv->vlans); + case MLX4_STEERING_MODE_B0: + err = mlx4_unicast_promisc_remove(mdev->dev, + priv->base_qpn, + priv->port); if (err) - en_err(priv, "Failed enabling VLAN filter\n"); + en_err(priv, "Failed disabling unicast promiscuous mode\n"); + /* Disable Multicast promisc */ + if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { + err = mlx4_multicast_promisc_remove(mdev->dev, + priv->base_qpn, + priv->port); + if (err) + en_err(priv, "Failed disabling multicast promiscuous mode\n"); + priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; + } + break; + + case MLX4_STEERING_MODE_A0: + err = mlx4_SET_PORT_qpn_calc(mdev->dev, + priv->port, + priv->base_qpn, 0); + if (err) + en_err(priv, "Failed disabling promiscuous mode\n"); + break; } +} +static void mlx4_en_do_multicast(struct mlx4_en_priv *priv, + struct net_device *dev, + struct mlx4_en_dev *mdev) +{ + struct mlx4_en_mc_list *mclist, *tmp; + u8 mc_list[16] = {0}; + int err = 0; + u64 mcast_addr = 0; + + /* Enable/disable the multicast filter according to IFF_ALLMULTI */ if (dev->if_flags & IFF_ALLMULTI) { err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 0, MLX4_MCAST_DISABLE); if (err) en_err(priv, "Failed disabling multicast filter\n"); + + /* Add the default qp number as multicast promisc */ + if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { + switch (mdev->dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: + err = mlx4_flow_steer_promisc_add(mdev->dev, + priv->port, + priv->base_qpn, + MLX4_FS_MC_DEFAULT); + break; + + case MLX4_STEERING_MODE_B0: + err = mlx4_multicast_promisc_add(mdev->dev, + priv->base_qpn, + priv->port); + break; + + case MLX4_STEERING_MODE_A0: + break; + } + if (err) + en_err(priv, "Failed entering multicast promisc mode\n"); + priv->flags |= MLX4_EN_FLAG_MC_PROMISC; + } } else { - u64 *mcaddr; - int mccount; - int i; + /* Disable Multicast promisc */ + if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { + switch (mdev->dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: + err = mlx4_flow_steer_promisc_remove(mdev->dev, + priv->port, + MLX4_FS_MC_DEFAULT); + break; + case MLX4_STEERING_MODE_B0: + err = mlx4_multicast_promisc_remove(mdev->dev, + priv->base_qpn, + priv->port); + break; + + case MLX4_STEERING_MODE_A0: + break; + } + if (err) + en_err(priv, "Failed disabling multicast promiscuous mode\n"); + priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; + } + err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 0, MLX4_MCAST_DISABLE); if (err) en_err(priv, "Failed disabling multicast filter\n"); /* Flush mcast filter and init it with broadcast address */ mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, ETH_BCAST, 1, MLX4_MCAST_CONFIG); /* Update multicast list - we cache all addresses so they won't * change while HW is updated holding the command semaphor */ - mccount = mlx4_en_cache_mclist(dev, &mcaddr); - for (i = 0; i < mccount; i++) + mlx4_en_cache_mclist(dev); + list_for_each_entry(mclist, &priv->mc_list, list) { + mcast_addr = mlx4_mac_to_u64(mclist->addr); mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, - mcaddr[i], 0, MLX4_MCAST_CONFIG); + mcast_addr, 0, MLX4_MCAST_CONFIG); + } err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 0, MLX4_MCAST_ENABLE); if (err) en_err(priv, "Failed enabling multicast filter\n"); - kfree(mcaddr); + update_mclist_flags(priv, &priv->curr_list, &priv->mc_list); + list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) { + if (mclist->action == MCLIST_REM) { + /* detach this address and delete from list */ + memcpy(&mc_list[10], mclist->addr, ETH_ALEN); + mc_list[5] = priv->port; + err = mlx4_multicast_detach(mdev->dev, + &priv->rss_map.indir_qp, + mc_list, + MLX4_PROT_ETH, + mclist->reg_id); + if (err) + en_err(priv, "Fail to detach multicast address\n"); + + /* remove from list */ + list_del(&mclist->list); + kfree(mclist); + } else if (mclist->action == MCLIST_ADD) { + /* attach the address */ + memcpy(&mc_list[10], mclist->addr, ETH_ALEN); + /* needed for B0 steering support */ + mc_list[5] = priv->port; + err = mlx4_multicast_attach(mdev->dev, + &priv->rss_map.indir_qp, + mc_list, + priv->port, 0, + MLX4_PROT_ETH, + &mclist->reg_id); + if (err) + en_err(priv, "Fail to attach multicast address\n"); + + } + } } +} + +static void mlx4_en_do_set_rx_mode(struct work_struct *work) +{ + struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, + rx_mode_task); + struct mlx4_en_dev *mdev = priv->mdev; + struct net_device *dev = priv->dev; + + + mutex_lock(&mdev->state_lock); + if (!mdev->device_up) { + en_dbg(HW, priv, "Card is not up, ignoring rx mode change.\n"); + goto out; + } + if (!priv->port_up) { + en_dbg(HW, priv, "Port is down, ignoring rx mode change.\n"); + goto out; + } + if (!mlx4_en_QUERY_PORT(mdev, priv->port)) { + if (priv->port_state.link_state) { + priv->last_link_state = MLX4_DEV_EVENT_PORT_UP; + /* Important note: the following call for if_link_state_change + * is needed for interface up scenario (start port, link state + * change) */ + if_link_state_change(priv->dev, LINK_STATE_UP); + en_dbg(HW, priv, "Link Up\n"); + } + } + + /* Promsicuous mode: disable all filters */ + if ((dev->if_flags & IFF_PROMISC) || + (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC)) { + mlx4_en_set_promisc_mode(priv, mdev); + goto out; + } + + /* Not in promiscuous mode */ + if (priv->flags & MLX4_EN_FLAG_PROMISC) + mlx4_en_clear_promisc_mode(priv, mdev); + + mlx4_en_do_multicast(priv, dev, mdev); out: mutex_unlock(&mdev->state_lock); } #ifdef CONFIG_NET_POLL_CONTROLLER static void mlx4_en_netpoll(struct net_device *dev) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_cq *cq; unsigned long flags; int i; for (i = 0; i < priv->rx_ring_num; i++) { - cq = &priv->rx_cq[i]; + cq = priv->rx_cq[i]; spin_lock_irqsave(&cq->lock, flags); napi_synchronize(&cq->napi); mlx4_en_process_rx_cq(dev, cq, 0); spin_unlock_irqrestore(&cq->lock, flags); } } #endif static void mlx4_en_watchdog_timeout(void *arg) { - struct mlx4_en_priv *priv = arg; - struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_priv *priv = arg; + struct mlx4_en_dev *mdev = priv->mdev; - en_dbg(DRV, priv, "Scheduling watchdog\n"); - queue_work(mdev->workqueue, &priv->watchdog_task); - if (priv->port_up) - callout_reset(&priv->watchdog_timer, MLX4_EN_WATCHDOG_TIMEOUT, - mlx4_en_watchdog_timeout, priv); + en_dbg(DRV, priv, "Scheduling watchdog\n"); + queue_work(mdev->workqueue, &priv->watchdog_task); + if (priv->port_up) + callout_reset(&priv->watchdog_timer, MLX4_EN_WATCHDOG_TIMEOUT, + mlx4_en_watchdog_timeout, priv); } -/* XXX This clears user settings in too many cases. */ + static void mlx4_en_set_default_moderation(struct mlx4_en_priv *priv) { struct mlx4_en_cq *cq; int i; /* If we haven't received a specific coalescing setting - * (module param), we set the moderation paramters as follows: + * (module param), we set the moderation parameters as follows: * - moder_cnt is set to the number of mtu sized packets to * satisfy our coelsing target. * - moder_time is set to a fixed value. */ priv->rx_frames = MLX4_EN_RX_COAL_TARGET / priv->dev->if_mtu + 1; priv->rx_usecs = MLX4_EN_RX_COAL_TIME; - en_dbg(INTR, priv, "Default coalesing params for mtu:%ld - " - "rx_frames:%d rx_usecs:%d\n", - priv->dev->if_mtu, priv->rx_frames, priv->rx_usecs); + priv->tx_frames = MLX4_EN_TX_COAL_PKTS; + priv->tx_usecs = MLX4_EN_TX_COAL_TIME; + en_dbg(INTR, priv, "Default coalesing params for mtu: %u - " + "rx_frames:%d rx_usecs:%d\n", + (unsigned)priv->dev->if_mtu, priv->rx_frames, priv->rx_usecs); /* Setup cq moderation params */ for (i = 0; i < priv->rx_ring_num; i++) { - cq = &priv->rx_cq[i]; + cq = priv->rx_cq[i]; cq->moder_cnt = priv->rx_frames; cq->moder_time = priv->rx_usecs; + priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; + priv->last_moder_packets[i] = 0; + priv->last_moder_bytes[i] = 0; } for (i = 0; i < priv->tx_ring_num; i++) { - cq = &priv->tx_cq[i]; - cq->moder_cnt = MLX4_EN_TX_COAL_PKTS; - cq->moder_time = MLX4_EN_TX_COAL_TIME; + cq = priv->tx_cq[i]; + cq->moder_cnt = priv->tx_frames; + cq->moder_time = priv->tx_usecs; } /* Reset auto-moderation params */ priv->pkt_rate_low = MLX4_EN_RX_RATE_LOW; priv->rx_usecs_low = MLX4_EN_RX_COAL_TIME_LOW; priv->pkt_rate_high = MLX4_EN_RX_RATE_HIGH; priv->rx_usecs_high = MLX4_EN_RX_COAL_TIME_HIGH; priv->sample_interval = MLX4_EN_SAMPLE_INTERVAL; priv->adaptive_rx_coal = 1; - priv->last_moder_time = MLX4_EN_AUTO_CONF; priv->last_moder_jiffies = 0; - priv->last_moder_packets = 0; priv->last_moder_tx_packets = 0; - priv->last_moder_bytes = 0; } static void mlx4_en_auto_moderation(struct mlx4_en_priv *priv) { unsigned long period = (unsigned long) (jiffies - priv->last_moder_jiffies); struct mlx4_en_cq *cq; unsigned long packets; unsigned long rate; unsigned long avg_pkt_size; unsigned long rx_packets; unsigned long rx_bytes; - unsigned long tx_packets; - unsigned long tx_pkt_diff; unsigned long rx_pkt_diff; int moder_time; - int i, err; + int ring, err; if (!priv->adaptive_rx_coal || period < priv->sample_interval * HZ) return; - spin_lock(&priv->stats_lock); - rx_packets = priv->dev->if_ipackets; - rx_bytes = priv->dev->if_ibytes; - tx_packets = priv->dev->if_opackets; - spin_unlock(&priv->stats_lock); + for (ring = 0; ring < priv->rx_ring_num; ring++) { + spin_lock(&priv->stats_lock); + rx_packets = priv->rx_ring[ring]->packets; + rx_bytes = priv->rx_ring[ring]->bytes; + spin_unlock(&priv->stats_lock); - if (!priv->last_moder_jiffies || !period) - goto out; + rx_pkt_diff = ((unsigned long) (rx_packets - + priv->last_moder_packets[ring])); + packets = rx_pkt_diff; + rate = packets * HZ / period; + avg_pkt_size = packets ? ((unsigned long) (rx_bytes - + priv->last_moder_bytes[ring])) / packets : 0; - tx_pkt_diff = ((unsigned long) (tx_packets - - priv->last_moder_tx_packets)); - rx_pkt_diff = ((unsigned long) (rx_packets - - priv->last_moder_packets)); - packets = max(tx_pkt_diff, rx_pkt_diff); - rate = packets * HZ / period; - avg_pkt_size = packets ? ((unsigned long) (rx_bytes - - priv->last_moder_bytes)) / packets : 0; - - /* Apply auto-moderation only when packet rate exceeds a rate that - * it matters */ - if (rate > MLX4_EN_RX_RATE_THRESH) { - /* If tx and rx packet rates are not balanced, assume that - * traffic is mainly BW bound and apply maximum moderation. - * Otherwise, moderate according to packet rate */ - if (2 * tx_pkt_diff > 3 * rx_pkt_diff || - 2 * rx_pkt_diff > 3 * tx_pkt_diff) { - moder_time = priv->rx_usecs_high; - } else { - if (rate < priv->pkt_rate_low || - avg_pkt_size < MLX4_EN_AVG_PKT_SMALL) + /* Apply auto-moderation only when packet rate + * exceeds a rate that it matters */ + if (rate > (MLX4_EN_RX_RATE_THRESH / priv->rx_ring_num) && + avg_pkt_size > MLX4_EN_AVG_PKT_SMALL) { + if (rate < priv->pkt_rate_low) moder_time = priv->rx_usecs_low; else if (rate > priv->pkt_rate_high) moder_time = priv->rx_usecs_high; else moder_time = (rate - priv->pkt_rate_low) * (priv->rx_usecs_high - priv->rx_usecs_low) / (priv->pkt_rate_high - priv->pkt_rate_low) + priv->rx_usecs_low; + } else { + moder_time = priv->rx_usecs_low; } - } else { - /* When packet rate is low, use default moderation rather than - * 0 to prevent interrupt storms if traffic suddenly increases */ - moder_time = priv->rx_usecs; - } - en_dbg(INTR, priv, "tx rate:%lu rx_rate:%lu\n", - tx_pkt_diff * HZ / period, rx_pkt_diff * HZ / period); - - en_dbg(INTR, priv, "Rx moder_time changed from:%d to %d period:%lu " - "[jiff] packets:%lu avg_pkt_size:%lu rate:%lu [p/s])\n", - priv->last_moder_time, moder_time, period, packets, - avg_pkt_size, rate); - - if (moder_time != priv->last_moder_time) { - priv->last_moder_time = moder_time; - for (i = 0; i < priv->rx_ring_num; i++) { - cq = &priv->rx_cq[i]; + if (moder_time != priv->last_moder_time[ring]) { + priv->last_moder_time[ring] = moder_time; + cq = priv->rx_cq[ring]; cq->moder_time = moder_time; err = mlx4_en_set_cq_moder(priv, cq); - if (err) { - en_err(priv, "Failed modifying moderation for cq:%d\n", i); - break; - } + if (err) + en_err(priv, "Failed modifying moderation for cq:%d\n", + ring); } + priv->last_moder_packets[ring] = rx_packets; + priv->last_moder_bytes[ring] = rx_bytes; } -out: - priv->last_moder_packets = rx_packets; - priv->last_moder_tx_packets = tx_packets; - priv->last_moder_bytes = rx_bytes; priv->last_moder_jiffies = jiffies; } -static void mlx4_en_handle_vlans(struct mlx4_en_priv *priv) -{ - u8 vlan_register[VLAN_FLTR_SIZE]; - u8 vlan_unregister[VLAN_FLTR_SIZE]; - int i, j, idx; - u16 vid; - - /* cache the vlan data for processing - * done under lock to avoid changes during work */ - spin_lock(&priv->vlan_lock); - for (i = 0; i < VLAN_FLTR_SIZE; i++) { - vlan_register[i] = priv->vlan_register[i]; - priv->vlan_register[i] = 0; - vlan_unregister[i] = priv->vlan_unregister[i]; - priv->vlan_unregister[i] = 0; - } - priv->vlgrp_modified = false; - spin_unlock(&priv->vlan_lock); - - /* Configure the vlan filter - * The vlgrp is updated with all the vids that need to be allowed */ - if (mlx4_SET_VLAN_FLTR(priv->mdev->dev, priv->port, priv->vlans)) - en_err(priv, "Failed configuring VLAN filter\n"); - - /* Configure the VLAN table */ - for (i = 0; i < VLAN_FLTR_SIZE; i++) { - for (j = 0; j < 32; j++) { - vid = (i << 5) + j; - if (vlan_register[i] & (1 << j)) - if (mlx4_register_vlan(priv->mdev->dev, priv->port, vid, &idx)) - en_dbg(HW, priv, "failed registering vlan %d\n", vid); - if (vlan_unregister[i] & (1 << j)) { - if (!mlx4_find_cached_vlan(priv->mdev->dev, priv->port, vid, &idx)) - mlx4_unregister_vlan(priv->mdev->dev, priv->port, idx); - else - en_dbg(HW, priv, "could not find vid %d in cache\n", vid); - } - } - } -} - static void mlx4_en_do_get_stats(struct work_struct *work) { struct delayed_work *delay = to_delayed_work(work); struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, stats_task); struct mlx4_en_dev *mdev = priv->mdev; int err; - err = mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 0); - if (err) - en_dbg(HW, priv, "Could not update stats \n"); - - mutex_lock(&mdev->state_lock); if (mdev->device_up) { if (priv->port_up) { - if (priv->vlgrp_modified) - mlx4_en_handle_vlans(priv); + err = mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 0); + if (err) + en_dbg(HW, priv, "Could not update stats\n"); mlx4_en_auto_moderation(priv); } queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY); } mutex_unlock(&mdev->state_lock); } +/* mlx4_en_service_task - Run service task for tasks that needed to be done + * periodically + */ +static void mlx4_en_service_task(struct work_struct *work) +{ + struct delayed_work *delay = to_delayed_work(work); + struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, + service_task); + struct mlx4_en_dev *mdev = priv->mdev; + + mutex_lock(&mdev->state_lock); + if (mdev->device_up) { + queue_delayed_work(mdev->workqueue, &priv->service_task, + SERVICE_TASK_DELAY); + } + mutex_unlock(&mdev->state_lock); +} + static void mlx4_en_linkstate(struct work_struct *work) { struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, linkstate_task); struct mlx4_en_dev *mdev = priv->mdev; int linkstate = priv->link_state; mutex_lock(&mdev->state_lock); /* If observable port state changed set carrier state and * report to system log */ if (priv->last_link_state != linkstate) { if (linkstate == MLX4_DEV_EVENT_PORT_DOWN) { + en_info(priv, "Link Down\n"); if_link_state_change(priv->dev, LINK_STATE_DOWN); - } else { + /* make sure the port is up before notifying the OS. + * This is tricky since we get here on INIT_PORT and + * in such case we can't tell the OS the port is up. + * To solve this there is a call to if_link_state_change + * in set_rx_mode. + * */ + } else if (priv->port_up && (linkstate == MLX4_DEV_EVENT_PORT_UP)){ en_info(priv, "Link Up\n"); if_link_state_change(priv->dev, LINK_STATE_UP); } } priv->last_link_state = linkstate; mutex_unlock(&mdev->state_lock); } int mlx4_en_start_port(struct net_device *dev) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_cq *cq; struct mlx4_en_tx_ring *tx_ring; - u64 config; int rx_index = 0; int tx_index = 0; int err = 0; int i; int j; + u8 mc_list[16] = {0}; + if (priv->port_up) { en_dbg(DRV, priv, "start port called while port already up\n"); return 0; } + INIT_LIST_HEAD(&priv->mc_list); + INIT_LIST_HEAD(&priv->curr_list); + INIT_LIST_HEAD(&priv->ethtool_list); + /* Calculate Rx buf size */ dev->if_mtu = min(dev->if_mtu, priv->max_mtu); - mlx4_en_calc_rx_buf(dev); + mlx4_en_calc_rx_buf(dev); + priv->rx_alloc_size = max_t(int, 2 * roundup_pow_of_two(priv->rx_mb_size), + PAGE_SIZE); + priv->rx_alloc_order = get_order(priv->rx_alloc_size); + priv->rx_buf_size = roundup_pow_of_two(priv->rx_mb_size); + priv->log_rx_info = ROUNDUP_LOG2(sizeof(struct mlx4_en_rx_buf)); en_dbg(DRV, priv, "Rx buf size:%d\n", priv->rx_mb_size); /* Configure rx cq's and rings */ err = mlx4_en_activate_rx_rings(priv); if (err) { en_err(priv, "Failed to activate RX rings\n"); return err; } - for (i = 0; i < priv->rx_ring_num; i++) { - cq = &priv->rx_cq[i]; + cq = priv->rx_cq[i]; - err = mlx4_en_activate_cq(priv, cq); + mlx4_en_cq_init_lock(cq); + err = mlx4_en_activate_cq(priv, cq, i); if (err) { en_err(priv, "Failed activating Rx CQ\n"); goto cq_err; } for (j = 0; j < cq->size; j++) cq->buf[j].owner_sr_opcode = MLX4_CQE_OWNER_MASK; err = mlx4_en_set_cq_moder(priv, cq); if (err) { en_err(priv, "Failed setting cq moderation parameters"); mlx4_en_deactivate_cq(priv, cq); goto cq_err; } mlx4_en_arm_cq(priv, cq); - priv->rx_ring[i].cqn = cq->mcq.cqn; + priv->rx_ring[i]->cqn = cq->mcq.cqn; ++rx_index; } + /* Set qp number */ + en_dbg(DRV, priv, "Getting qp number for port %d\n", priv->port); + err = mlx4_en_get_qp(priv); + if (err) { + en_err(priv, "Failed getting eth qp\n"); + goto cq_err; + } + mdev->mac_removed[priv->port] = 0; + + /* gets default allocated counter index from func cap */ + /* or sink counter index if no resources */ + priv->counter_index = mdev->dev->caps.def_counter_index[priv->port - 1]; + + en_dbg(DRV, priv, "%s: default counter index %d for port %d\n", + __func__, priv->counter_index, priv->port); + err = mlx4_en_config_rss_steer(priv); if (err) { en_err(priv, "Failed configuring rss steering\n"); - goto cq_err; + goto mac_err; } + err = mlx4_en_create_drop_qp(priv); + if (err) + goto rss_err; + /* Configure tx cq's and rings */ for (i = 0; i < priv->tx_ring_num; i++) { /* Configure cq */ - cq = &priv->tx_cq[i]; - err = mlx4_en_activate_cq(priv, cq); + cq = priv->tx_cq[i]; + err = mlx4_en_activate_cq(priv, cq, i); if (err) { en_err(priv, "Failed allocating Tx CQ\n"); goto tx_err; } err = mlx4_en_set_cq_moder(priv, cq); if (err) { en_err(priv, "Failed setting cq moderation parameters"); mlx4_en_deactivate_cq(priv, cq); goto tx_err; } en_dbg(DRV, priv, "Resetting index of collapsed CQ:%d to -1\n", i); cq->buf->wqe_index = cpu_to_be16(0xffff); /* Configure ring */ - tx_ring = &priv->tx_ring[i]; - err = mlx4_en_activate_tx_ring(priv, tx_ring, cq->mcq.cqn); + tx_ring = priv->tx_ring[i]; + + err = mlx4_en_activate_tx_ring(priv, tx_ring, cq->mcq.cqn, + i / priv->num_tx_rings_p_up); if (err) { en_err(priv, "Failed allocating Tx ring\n"); mlx4_en_deactivate_cq(priv, cq); goto tx_err; } + + /* Arm CQ for TX completions */ + mlx4_en_arm_cq(priv, cq); + /* Set initial ownership of all Tx TXBBs to SW (1) */ for (j = 0; j < tx_ring->buf_size; j += STAMP_STRIDE) *((u32 *) (tx_ring->buf + j)) = 0xffffffff; ++tx_index; } /* Configure port */ err = mlx4_SET_PORT_general(mdev->dev, priv->port, - priv->rx_mb_size + ETHER_CRC_LEN, + priv->rx_mb_size, priv->prof->tx_pause, priv->prof->tx_ppp, priv->prof->rx_pause, priv->prof->rx_ppp); if (err) { - en_err(priv, "Failed setting port general configurations " - "for port %d, with error %d\n", priv->port, err); + en_err(priv, "Failed setting port general configurations for port %d, with error %d\n", + priv->port, err); goto tx_err; } /* Set default qp number */ err = mlx4_SET_PORT_qpn_calc(mdev->dev, priv->port, priv->base_qpn, 0); if (err) { en_err(priv, "Failed setting default qp numbers\n"); goto tx_err; } - /* Set port mac number */ - en_dbg(DRV, priv, "Setting mac for port %d\n", priv->port); - err = mlx4_register_mac(mdev->dev, priv->port, - mlx4_en_mac_to_u64(IF_LLADDR(dev)), - &priv->mac_index); - if (err) { - en_err(priv, "Failed setting port mac\n"); - goto tx_err; - } - mdev->mac_removed[priv->port] = 0; /* Init port */ en_dbg(HW, priv, "Initializing port\n"); err = mlx4_INIT_PORT(mdev->dev, priv->port); if (err) { en_err(priv, "Failed Initializing port\n"); - goto mac_err; + goto tx_err; } - /* Set the various hardware offload abilities */ - dev->if_hwassist = 0; - if (dev->if_capenable & IFCAP_TSO4) - dev->if_hwassist |= CSUM_TSO; - if (dev->if_capenable & IFCAP_TXCSUM) - dev->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP); - if (dev->if_capenable & IFCAP_RXCSUM) - priv->rx_csum = 1; - else - priv->rx_csum = 0; + /* Attach rx QP to bradcast address */ + memset(&mc_list[10], 0xff, ETH_ALEN); + mc_list[5] = priv->port; /* needed for B0 steering support */ + if (mlx4_multicast_attach(mdev->dev, &priv->rss_map.indir_qp, mc_list, + priv->port, 0, MLX4_PROT_ETH, + &priv->broadcast_id)) + mlx4_warn(mdev, "Failed Attaching Broadcast\n"); - err = mlx4_wol_read(priv->mdev->dev, &config, priv->port); - if (err) { - en_err(priv, "Failed to get WoL info, unable to modify\n"); - goto wol_err; - } - if (dev->if_capenable & IFCAP_WOL_MAGIC) { - config |= MLX4_EN_WOL_DO_MODIFY | MLX4_EN_WOL_ENABLED | - MLX4_EN_WOL_MAGIC; - } else { - config &= ~(MLX4_EN_WOL_ENABLED | MLX4_EN_WOL_MAGIC); - config |= MLX4_EN_WOL_DO_MODIFY; - } + /* Must redo promiscuous mode setup. */ + priv->flags &= ~(MLX4_EN_FLAG_PROMISC | MLX4_EN_FLAG_MC_PROMISC); - err = mlx4_wol_write(priv->mdev->dev, config, priv->port); - if (err) { - en_err(priv, "Failed to set WoL information\n"); - goto wol_err; - } + /* Schedule multicast task to populate multicast list */ + queue_work(mdev->workqueue, &priv->rx_mode_task); + mlx4_set_stats_bitmap(mdev->dev, priv->stats_bitmap); + priv->port_up = true; - /* Populate multicast list */ - mlx4_en_set_multicast(dev); + /* Enable the queues. */ + dev->if_drv_flags &= ~IFF_DRV_OACTIVE; + dev->if_drv_flags |= IFF_DRV_RUNNING; +#ifdef CONFIG_DEBUG_FS + mlx4_en_create_debug_files(priv); +#endif + callout_reset(&priv->watchdog_timer, MLX4_EN_WATCHDOG_TIMEOUT, + mlx4_en_watchdog_timeout, priv); - /* Enable the queues. */ - dev->if_drv_flags &= ~IFF_DRV_OACTIVE; - dev->if_drv_flags |= IFF_DRV_RUNNING; - callout_reset(&priv->watchdog_timer, MLX4_EN_WATCHDOG_TIMEOUT, - mlx4_en_watchdog_timeout, priv); - return 0; -wol_err: - /* close port*/ - mlx4_CLOSE_PORT(mdev->dev, priv->port); - -mac_err: - mlx4_unregister_mac(mdev->dev, priv->port, priv->mac_index); tx_err: while (tx_index--) { - mlx4_en_deactivate_tx_ring(priv, &priv->tx_ring[tx_index]); - mlx4_en_deactivate_cq(priv, &priv->tx_cq[tx_index]); + mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[tx_index]); + mlx4_en_deactivate_cq(priv, priv->tx_cq[tx_index]); } - + mlx4_en_destroy_drop_qp(priv); +rss_err: mlx4_en_release_rss_steer(priv); +mac_err: + mlx4_en_put_qp(priv); cq_err: while (rx_index--) - mlx4_en_deactivate_cq(priv, &priv->rx_cq[rx_index]); + mlx4_en_deactivate_cq(priv, priv->rx_cq[rx_index]); for (i = 0; i < priv->rx_ring_num; i++) - mlx4_en_deactivate_rx_ring(priv, &priv->rx_ring[i]); + mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]); return err; /* need to close devices */ } void mlx4_en_stop_port(struct net_device *dev) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_mc_list *mclist, *tmp; int i; + u8 mc_list[16] = {0}; if (!priv->port_up) { en_dbg(DRV, priv, "stop port called while port already down\n"); return; } +#ifdef CONFIG_DEBUG_FS + mlx4_en_delete_debug_files(priv); +#endif + + /* close port*/ + mlx4_CLOSE_PORT(mdev->dev, priv->port); + /* Set port as not active */ priv->port_up = false; + if (priv->counter_index != 0xff) { + mlx4_counter_free(mdev->dev, priv->port, priv->counter_index); + priv->counter_index = 0xff; + } - /* Unregister Mac address for the port */ - mlx4_unregister_mac(mdev->dev, priv->port, priv->mac_index); - mdev->mac_removed[priv->port] = 1; + /* Promsicuous mode */ + if (mdev->dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) { + priv->flags &= ~(MLX4_EN_FLAG_PROMISC | + MLX4_EN_FLAG_MC_PROMISC); + mlx4_flow_steer_promisc_remove(mdev->dev, + priv->port, + MLX4_FS_ALL_DEFAULT); + mlx4_flow_steer_promisc_remove(mdev->dev, + priv->port, + MLX4_FS_MC_DEFAULT); + } else if (priv->flags & MLX4_EN_FLAG_PROMISC) { + priv->flags &= ~MLX4_EN_FLAG_PROMISC; + /* Disable promiscouos mode */ + mlx4_unicast_promisc_remove(mdev->dev, priv->base_qpn, + priv->port); + + /* Disable Multicast promisc */ + if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { + mlx4_multicast_promisc_remove(mdev->dev, priv->base_qpn, + priv->port); + priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; + } + } + + /* Detach All multicasts */ + memset(&mc_list[10], 0xff, ETH_ALEN); + mc_list[5] = priv->port; /* needed for B0 steering support */ + mlx4_multicast_detach(mdev->dev, &priv->rss_map.indir_qp, mc_list, + MLX4_PROT_ETH, priv->broadcast_id); + list_for_each_entry(mclist, &priv->curr_list, list) { + memcpy(&mc_list[10], mclist->addr, ETH_ALEN); + mc_list[5] = priv->port; + mlx4_multicast_detach(mdev->dev, &priv->rss_map.indir_qp, + mc_list, MLX4_PROT_ETH, mclist->reg_id); + } + mlx4_en_clear_list(dev); + list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) { + list_del(&mclist->list); + kfree(mclist); + } + + /* Flush multicast filter */ + mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1, MLX4_MCAST_CONFIG); + mlx4_en_destroy_drop_qp(priv); + /* Free TX Rings */ for (i = 0; i < priv->tx_ring_num; i++) { - mlx4_en_deactivate_tx_ring(priv, &priv->tx_ring[i]); - mlx4_en_deactivate_cq(priv, &priv->tx_cq[i]); + mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[i]); + mlx4_en_deactivate_cq(priv, priv->tx_cq[i]); } msleep(10); for (i = 0; i < priv->tx_ring_num; i++) - mlx4_en_free_tx_buf(dev, &priv->tx_ring[i]); + mlx4_en_free_tx_buf(dev, priv->tx_ring[i]); /* Free RSS qps */ mlx4_en_release_rss_steer(priv); + /* Unregister Mac address for the port */ + mlx4_en_put_qp(priv); + mdev->mac_removed[priv->port] = 1; + /* Free RX Rings */ for (i = 0; i < priv->rx_ring_num; i++) { - mlx4_en_deactivate_rx_ring(priv, &priv->rx_ring[i]); - mlx4_en_deactivate_cq(priv, &priv->rx_cq[i]); + struct mlx4_en_cq *cq = priv->rx_cq[i]; + mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]); + mlx4_en_deactivate_cq(priv, cq); } - /* close port*/ - mlx4_CLOSE_PORT(mdev->dev, priv->port); + callout_stop(&priv->watchdog_timer); - callout_stop(&priv->watchdog_timer); - - dev->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); + dev->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); } static void mlx4_en_restart(struct work_struct *work) { struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, watchdog_task); struct mlx4_en_dev *mdev = priv->mdev; struct net_device *dev = priv->dev; struct mlx4_en_tx_ring *ring; int i; + if (priv->blocked == 0 || priv->port_up == 0) return; for (i = 0; i < priv->tx_ring_num; i++) { - ring = &priv->tx_ring[i]; + ring = priv->tx_ring[i]; if (ring->blocked && - ring->watchdog_time + MLX4_EN_WATCHDOG_TIMEOUT < ticks) + ring->watchdog_time + MLX4_EN_WATCHDOG_TIMEOUT < ticks) goto reset; } return; reset: priv->port_stats.tx_timeout++; en_dbg(DRV, priv, "Watchdog task called for port %d\n", priv->port); mutex_lock(&mdev->state_lock); if (priv->port_up) { mlx4_en_stop_port(dev); + //for (i = 0; i < priv->tx_ring_num; i++) + // netdev_tx_reset_queue(priv->tx_ring[i]->tx_queue); if (mlx4_en_start_port(dev)) en_err(priv, "Failed restarting port %d\n", priv->port); } mutex_unlock(&mdev->state_lock); } - -static void -mlx4_en_init(void *arg) +static void mlx4_en_clear_stats(struct net_device *dev) { - struct mlx4_en_priv *priv; - struct mlx4_en_dev *mdev; + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int i; - priv = arg; - mdev = priv->mdev; - mutex_lock(&mdev->state_lock); - mlx4_en_init_locked(priv); - mutex_unlock(&mdev->state_lock); + if (!mlx4_is_slave(mdev->dev)) + if (mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 1)) + en_dbg(HW, priv, "Failed dumping statistics\n"); + + memset(&priv->pstats, 0, sizeof(priv->pstats)); + memset(&priv->pkstats, 0, sizeof(priv->pkstats)); + memset(&priv->port_stats, 0, sizeof(priv->port_stats)); + memset(&priv->vport_stats, 0, sizeof(priv->vport_stats)); + + for (i = 0; i < priv->tx_ring_num; i++) { + priv->tx_ring[i]->bytes = 0; + priv->tx_ring[i]->packets = 0; + priv->tx_ring[i]->tx_csum = 0; + } + for (i = 0; i < priv->rx_ring_num; i++) { + priv->rx_ring[i]->bytes = 0; + priv->rx_ring[i]->packets = 0; + priv->rx_ring[i]->csum_ok = 0; + priv->rx_ring[i]->csum_none = 0; + } } -static void -mlx4_en_init_locked(struct mlx4_en_priv *priv) +static void mlx4_en_open(void* arg) { - struct mlx4_en_dev *mdev; - struct ifnet *dev; - int i; + struct mlx4_en_priv *priv; + struct mlx4_en_dev *mdev; + struct net_device *dev; + int err = 0; - dev = priv->dev; - mdev = priv->mdev; - if (dev->if_drv_flags & IFF_DRV_RUNNING) - mlx4_en_stop_port(dev); + priv = arg; + mdev = priv->mdev; + dev = priv->dev; + + mutex_lock(&mdev->state_lock); + if (!mdev->device_up) { en_err(priv, "Cannot open - device down/disabled\n"); - return; + goto out; } - /* Reset HW statistics and performance counters */ - if (mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 1)) - en_dbg(HW, priv, "Failed dumping statistics\n"); + /* Reset HW statistics and SW counters */ + mlx4_en_clear_stats(dev); - memset(&priv->pstats, 0, sizeof(priv->pstats)); - - for (i = 0; i < priv->tx_ring_num; i++) { - priv->tx_ring[i].bytes = 0; - priv->tx_ring[i].packets = 0; - } - for (i = 0; i < priv->rx_ring_num; i++) { - priv->rx_ring[i].bytes = 0; - priv->rx_ring[i].packets = 0; - } - - mlx4_en_set_default_moderation(priv); - if (mlx4_en_start_port(dev)) + err = mlx4_en_start_port(dev); + if (err) en_err(priv, "Failed starting port:%d\n", priv->port); + +out: + mutex_unlock(&mdev->state_lock); + return; } void mlx4_en_free_resources(struct mlx4_en_priv *priv) { int i; +#ifdef CONFIG_RFS_ACCEL + if (priv->dev->rx_cpu_rmap) { + free_irq_cpu_rmap(priv->dev->rx_cpu_rmap); + priv->dev->rx_cpu_rmap = NULL; + } +#endif + for (i = 0; i < priv->tx_ring_num; i++) { - if (priv->tx_ring[i].tx_info) + if (priv->tx_ring && priv->tx_ring[i]) mlx4_en_destroy_tx_ring(priv, &priv->tx_ring[i]); - if (priv->tx_cq[i].buf) + if (priv->tx_cq && priv->tx_cq[i]) mlx4_en_destroy_cq(priv, &priv->tx_cq[i]); } for (i = 0; i < priv->rx_ring_num; i++) { - if (priv->rx_ring[i].rx_info) - mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i]); - if (priv->rx_cq[i].buf) + if (priv->rx_ring[i]) + mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], + priv->prof->rx_ring_size, priv->stride); + if (priv->rx_cq[i]) mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); } - /* Free the stats tree when we resize the rings. */ + if (priv->sysctl) sysctl_ctx_free(&priv->stat_ctx); + } int mlx4_en_alloc_resources(struct mlx4_en_priv *priv) { struct mlx4_en_port_profile *prof = priv->prof; int i; + int node = 0; - /* Create tx Rings */ - for (i = 0; i < priv->tx_ring_num; i++) { - if (mlx4_en_create_cq(priv, &priv->tx_cq[i], - prof->tx_ring_size, i, TX)) - goto err; - - if (mlx4_en_create_tx_ring(priv, &priv->tx_ring[i], - prof->tx_ring_size, TXBB_SIZE)) - goto err; - } - /* Create rx Rings */ for (i = 0; i < priv->rx_ring_num; i++) { if (mlx4_en_create_cq(priv, &priv->rx_cq[i], - prof->rx_ring_size, i, RX)) + prof->rx_ring_size, i, RX, node)) goto err; if (mlx4_en_create_rx_ring(priv, &priv->rx_ring[i], - prof->rx_ring_size)) + prof->rx_ring_size, node)) goto err; } - /* Re-create stat sysctls in case the number of rings changed. */ - mlx4_en_sysctl_stat(priv); + /* Create tx Rings */ + for (i = 0; i < priv->tx_ring_num; i++) { + if (mlx4_en_create_cq(priv, &priv->tx_cq[i], + prof->tx_ring_size, i, TX, node)) + goto err; - /* Populate Tx priority mappings */ - mlx4_en_set_prio_map(priv, priv->tx_prio_map, - priv->tx_ring_num - MLX4_EN_NUM_HASH_RINGS); + if (mlx4_en_create_tx_ring(priv, &priv->tx_ring[i], + prof->tx_ring_size, TXBB_SIZE, node, i)) + goto err; + } +#ifdef CONFIG_RFS_ACCEL + priv->dev->rx_cpu_rmap = alloc_irq_cpu_rmap(priv->rx_ring_num); + if (!priv->dev->rx_cpu_rmap) + goto err; +#endif + /* Re-create stat sysctls in case the number of rings changed. */ + mlx4_en_sysctl_stat(priv); return 0; err: en_err(priv, "Failed to allocate NIC resources\n"); + for (i = 0; i < priv->rx_ring_num; i++) { + if (priv->rx_ring[i]) + mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], + prof->rx_ring_size, + priv->stride); + if (priv->rx_cq[i]) + mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); + } + for (i = 0; i < priv->tx_ring_num; i++) { + if (priv->tx_ring[i]) + mlx4_en_destroy_tx_ring(priv, &priv->tx_ring[i]); + if (priv->tx_cq[i]) + mlx4_en_destroy_cq(priv, &priv->tx_cq[i]); + } + priv->port_up = false; return -ENOMEM; } +struct en_port_attribute { + struct attribute attr; + ssize_t (*show)(struct en_port *, struct en_port_attribute *, char *buf); + ssize_t (*store)(struct en_port *, struct en_port_attribute *, char *buf, size_t count); +}; +#define PORT_ATTR_RO(_name) \ +struct en_port_attribute en_port_attr_##_name = __ATTR_RO(_name) + +#define EN_PORT_ATTR(_name, _mode, _show, _store) \ +struct en_port_attribute en_port_attr_##_name = __ATTR(_name, _mode, _show, _store) + void mlx4_en_destroy_netdev(struct net_device *dev) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_dev *mdev = priv->mdev; en_dbg(DRV, priv, "Destroying netdev on port:%d\n", priv->port); - if (priv->vlan_attach != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach); - if (priv->vlan_detach != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach); + if (priv->vlan_attach != NULL) + EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach); + if (priv->vlan_detach != NULL) + EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach); /* Unregister device - this will close the port if it was up */ if (priv->registered) ether_ifdetach(dev); if (priv->allocated) mlx4_free_hwq_res(mdev->dev, &priv->res, MLX4_EN_PAGE_SIZE); - if (priv->sysctl) - sysctl_ctx_free(&priv->conf_ctx); - mutex_lock(&mdev->state_lock); mlx4_en_stop_port(dev); mutex_unlock(&mdev->state_lock); + cancel_delayed_work(&priv->stats_task); + cancel_delayed_work(&priv->service_task); /* flush any pending task for this netdev */ flush_workqueue(mdev->workqueue); - callout_drain(&priv->watchdog_timer); + callout_drain(&priv->watchdog_timer); /* Detach the netdev so tasks would not attempt to access it */ mutex_lock(&mdev->state_lock); mdev->pndev[priv->port] = NULL; mutex_unlock(&mdev->state_lock); + mlx4_en_free_resources(priv); - mtx_destroy(&priv->stats_lock.m); - mtx_destroy(&priv->vlan_lock.m); - kfree(priv); - if_free(dev); + + /* freeing the sysctl conf cannot be called from within mlx4_en_free_resources */ + if (priv->sysctl) + sysctl_ctx_free(&priv->conf_ctx); + + kfree(priv->tx_ring); + kfree(priv->tx_cq); + + kfree(priv); + if_free(dev); + } static int mlx4_en_change_mtu(struct net_device *dev, int new_mtu) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_dev *mdev = priv->mdev; int err = 0; - en_dbg(DRV, priv, "Change MTU called - current:%ld new:%d\n", - dev->if_mtu, new_mtu); + en_dbg(DRV, priv, "Change MTU called - current:%u new:%u\n", + (unsigned)dev->if_mtu, (unsigned)new_mtu); if ((new_mtu < MLX4_EN_MIN_MTU) || (new_mtu > priv->max_mtu)) { en_err(priv, "Bad MTU size:%d.\n", new_mtu); return -EPERM; } mutex_lock(&mdev->state_lock); dev->if_mtu = new_mtu; if (dev->if_drv_flags & IFF_DRV_RUNNING) { if (!mdev->device_up) { /* NIC is probably restarting - let watchdog task reset - * the port */ + * * the port */ en_dbg(DRV, priv, "Change MTU called with card down!?\n"); } else { mlx4_en_stop_port(dev); - mlx4_en_set_default_moderation(priv); err = mlx4_en_start_port(dev); if (err) { en_err(priv, "Failed restarting port:%d\n", - priv->port); + priv->port); queue_work(mdev->workqueue, &priv->watchdog_task); } } } mutex_unlock(&mdev->state_lock); return 0; } static int mlx4_en_calc_media(struct mlx4_en_priv *priv) { int trans_type; int active; active = IFM_ETHER; if (priv->last_link_state == MLX4_DEV_EVENT_PORT_DOWN) return (active); - if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) - return (active); + /* + * [ShaharK] mlx4_en_QUERY_PORT sleeps and cannot be called under a + * non-sleepable lock. + * I moved it to the periodic mlx4_en_do_get_stats. + if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) + return (active); + */ active |= IFM_FDX; trans_type = priv->port_state.transciver; /* XXX I don't know all of the transceiver values. */ switch (priv->port_state.link_speed) { case 1000: active |= IFM_1000_T; break; case 10000: if (trans_type > 0 && trans_type <= 0xC) active |= IFM_10G_SR; else if (trans_type == 0x80 || trans_type == 0) active |= IFM_10G_CX4; break; case 40000: active |= IFM_40G_CR4; break; } if (priv->prof->tx_pause) active |= IFM_ETH_TXPAUSE; if (priv->prof->rx_pause) active |= IFM_ETH_RXPAUSE; return (active); } - static void mlx4_en_media_status(struct ifnet *dev, struct ifmediareq *ifmr) { struct mlx4_en_priv *priv; priv = dev->if_softc; ifmr->ifm_status = IFM_AVALID; if (priv->last_link_state != MLX4_DEV_EVENT_PORT_DOWN) ifmr->ifm_status |= IFM_ACTIVE; ifmr->ifm_active = mlx4_en_calc_media(priv); return; } static int mlx4_en_media_change(struct ifnet *dev) { struct mlx4_en_priv *priv; struct ifmedia *ifm; int rxpause; int txpause; int error; priv = dev->if_softc; ifm = &priv->media; rxpause = txpause = 0; error = 0; if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) return (EINVAL); switch (IFM_SUBTYPE(ifm->ifm_media)) { case IFM_AUTO: break; case IFM_10G_SR: case IFM_10G_CX4: case IFM_1000_T: - if (IFM_SUBTYPE(ifm->ifm_media) == - IFM_SUBTYPE(mlx4_en_calc_media(priv)) && - (ifm->ifm_media & IFM_FDX)) + case IFM_40G_CR4: + if ((IFM_SUBTYPE(ifm->ifm_media) + == IFM_SUBTYPE(mlx4_en_calc_media(priv))) + && (ifm->ifm_media & IFM_FDX)) break; /* Fallthrough */ default: printf("%s: Only auto media type\n", if_name(dev)); return (EINVAL); } /* Allow user to set/clear pause */ if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_RXPAUSE) rxpause = 1; if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE) txpause = 1; if (priv->prof->tx_pause != txpause || priv->prof->rx_pause != rxpause) { priv->prof->tx_pause = txpause; priv->prof->rx_pause = rxpause; error = -mlx4_SET_PORT_general(priv->mdev->dev, priv->port, priv->rx_mb_size + ETHER_CRC_LEN, priv->prof->tx_pause, priv->prof->tx_ppp, priv->prof->rx_pause, priv->prof->rx_ppp); } return (error); } static int mlx4_en_ioctl(struct ifnet *dev, u_long command, caddr_t data) { struct mlx4_en_priv *priv; struct mlx4_en_dev *mdev; struct ifreq *ifr; int error; int mask; error = 0; mask = 0; priv = dev->if_softc; mdev = priv->mdev; ifr = (struct ifreq *) data; switch (command) { + case SIOCSIFMTU: error = -mlx4_en_change_mtu(dev, ifr->ifr_mtu); break; case SIOCSIFFLAGS: mutex_lock(&mdev->state_lock); if (dev->if_flags & IFF_UP) { if ((dev->if_drv_flags & IFF_DRV_RUNNING) == 0) mlx4_en_start_port(dev); else - mlx4_en_set_multicast(dev); + mlx4_en_set_rx_mode(dev); } else { if (dev->if_drv_flags & IFF_DRV_RUNNING) { mlx4_en_stop_port(dev); - if_link_state_change(dev, LINK_STATE_DOWN); + if_link_state_change(dev, LINK_STATE_DOWN); } } mutex_unlock(&mdev->state_lock); break; case SIOCADDMULTI: case SIOCDELMULTI: - mutex_lock(&mdev->state_lock); - mlx4_en_set_multicast(dev); - mutex_unlock(&mdev->state_lock); + mlx4_en_set_rx_mode(dev); break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: error = ifmedia_ioctl(dev, ifr, &priv->media, command); break; case SIOCSIFCAP: mutex_lock(&mdev->state_lock); mask = ifr->ifr_reqcap ^ dev->if_capenable; if (mask & IFCAP_HWCSUM) dev->if_capenable ^= IFCAP_HWCSUM; if (mask & IFCAP_TSO4) dev->if_capenable ^= IFCAP_TSO4; + if (mask & IFCAP_TSO6) + dev->if_capenable ^= IFCAP_TSO6; if (mask & IFCAP_LRO) dev->if_capenable ^= IFCAP_LRO; if (mask & IFCAP_VLAN_HWTAGGING) dev->if_capenable ^= IFCAP_VLAN_HWTAGGING; if (mask & IFCAP_VLAN_HWFILTER) dev->if_capenable ^= IFCAP_VLAN_HWFILTER; if (mask & IFCAP_WOL_MAGIC) dev->if_capenable ^= IFCAP_WOL_MAGIC; if (dev->if_drv_flags & IFF_DRV_RUNNING) - mlx4_en_init_locked(priv); + mlx4_en_start_port(dev); mutex_unlock(&mdev->state_lock); VLAN_CAPABILITIES(dev); break; default: error = ether_ioctl(dev, command, data); break; } return (error); } -static int mlx4_en_set_ring_size(struct net_device *dev, - int rx_size, int tx_size) + +int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, + struct mlx4_en_port_profile *prof) { - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int port_up = 0; - int err = 0; + struct net_device *dev; + struct mlx4_en_priv *priv; + uint8_t dev_addr[ETHER_ADDR_LEN]; + int err; + int i; - rx_size = roundup_pow_of_two(rx_size); - rx_size = max_t(u32, rx_size, MLX4_EN_MIN_RX_SIZE); - rx_size = min_t(u32, rx_size, MLX4_EN_MAX_RX_SIZE); - tx_size = roundup_pow_of_two(tx_size); - tx_size = max_t(u32, tx_size, MLX4_EN_MIN_TX_SIZE); - tx_size = min_t(u32, tx_size, MLX4_EN_MAX_TX_SIZE); + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + dev = priv->dev = if_alloc(IFT_ETHER); + if (dev == NULL) { + en_err(priv, "Net device allocation failed\n"); + kfree(priv); + return -ENOMEM; + } + dev->if_softc = priv; + if_initname(dev, "mlxen", atomic_fetchadd_int(&mlx4_en_unit, 1)); + dev->if_mtu = ETHERMTU; + dev->if_baudrate = 1000000000; + dev->if_init = mlx4_en_open; + dev->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; + dev->if_ioctl = mlx4_en_ioctl; + dev->if_transmit = mlx4_en_transmit; + dev->if_qflush = mlx4_en_qflush; + dev->if_snd.ifq_maxlen = prof->tx_ring_size; - if (rx_size == (priv->port_up ? - priv->rx_ring[0].actual_size : priv->rx_ring[0].size) && - tx_size == priv->tx_ring[0].size) - return 0; + /* + * Initialize driver private data + */ + priv->counter_index = 0xff; + spin_lock_init(&priv->stats_lock); + INIT_WORK(&priv->rx_mode_task, mlx4_en_do_set_rx_mode); + INIT_WORK(&priv->watchdog_task, mlx4_en_restart); + INIT_WORK(&priv->linkstate_task, mlx4_en_linkstate); + INIT_DELAYED_WORK(&priv->stats_task, mlx4_en_do_get_stats); + INIT_DELAYED_WORK(&priv->service_task, mlx4_en_service_task); + callout_init(&priv->watchdog_timer, 1); +#ifdef CONFIG_RFS_ACCEL + INIT_LIST_HEAD(&priv->filters); + spin_lock_init(&priv->filters_lock); +#endif - mutex_lock(&mdev->state_lock); - if (priv->port_up) { - port_up = 1; - mlx4_en_stop_port(dev); + priv->msg_enable = MLX4_EN_MSG_LEVEL; + priv->dev = dev; + priv->mdev = mdev; + priv->ddev = &mdev->pdev->dev; + priv->prof = prof; + priv->port = port; + priv->port_up = false; + priv->flags = prof->flags; + priv->ctrl_flags = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | + MLX4_WQE_CTRL_SOLICITED); + + priv->num_tx_rings_p_up = mdev->profile.num_tx_rings_p_up; + priv->tx_ring_num = prof->tx_ring_num; + priv->tx_ring = kcalloc(MAX_TX_RINGS, + sizeof(struct mlx4_en_tx_ring *), GFP_KERNEL); + if (!priv->tx_ring) { + err = -ENOMEM; + goto out; } - mlx4_en_free_resources(priv); - priv->prof->tx_ring_size = tx_size; - priv->prof->rx_ring_size = rx_size; + priv->tx_cq = kcalloc(sizeof(struct mlx4_en_cq *), MAX_TX_RINGS, + GFP_KERNEL); + if (!priv->tx_cq) { + err = -ENOMEM; + goto out; + } + + priv->rx_ring_num = prof->rx_ring_num; + priv->cqe_factor = (mdev->dev->caps.cqe_size == 64) ? 1 : 0; + priv->mac_index = -1; + priv->last_ifq_jiffies = 0; + priv->if_counters_rx_errors = 0; + priv->if_counters_rx_no_buffer = 0; +#ifdef CONFIG_MLX4_EN_DCB + if (!mlx4_is_slave(priv->mdev->dev)) { + priv->dcbx_cap = DCB_CAP_DCBX_HOST; + priv->flags |= MLX4_EN_FLAG_DCB_ENABLED; + if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) { + dev->dcbnl_ops = &mlx4_en_dcbnl_ops; + } else { + en_info(priv, "QoS disabled - no HW support\n"); + dev->dcbnl_ops = &mlx4_en_dcbnl_pfc_ops; + } + } +#endif + + for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) + INIT_HLIST_HEAD(&priv->mac_hash[i]); + + + /* Query for default mac and max mtu */ + priv->max_mtu = mdev->dev->caps.eth_mtu_cap[priv->port]; + priv->mac = mdev->dev->caps.def_mac[priv->port]; + if (ILLEGAL_MAC(priv->mac)) { +#if BITS_PER_LONG == 64 + en_err(priv, "Port: %d, invalid mac burned: 0x%lx, quiting\n", + priv->port, priv->mac); +#elif BITS_PER_LONG == 32 + en_err(priv, "Port: %d, invalid mac burned: 0x%llx, quiting\n", + priv->port, priv->mac); +#endif + err = -EINVAL; + goto out; + } + + + + priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + + DS_SIZE); + + mlx4_en_sysctl_conf(priv); + err = mlx4_en_alloc_resources(priv); + if (err) + goto out; + + /* Allocate page for receive rings */ + err = mlx4_alloc_hwq_res(mdev->dev, &priv->res, + MLX4_EN_PAGE_SIZE, MLX4_EN_PAGE_SIZE); if (err) { - en_err(priv, "Failed reallocating port resources\n"); + en_err(priv, "Failed to allocate page for rx qps\n"); goto out; } - if (port_up) { - err = mlx4_en_start_port(dev); - if (err) - en_err(priv, "Failed starting port\n"); + priv->allocated = 1; + + /* + * Set driver features + */ + dev->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; + dev->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING; + dev->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER; + dev->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU; + dev->if_capabilities |= IFCAP_LRO; + + if (mdev->LSO_support) + dev->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6 | IFCAP_VLAN_HWTSO; +#if 0 + /* set TSO limits so that we don't have to drop TX packets */ + dev->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); + dev->if_hw_tsomaxsegcount = 16; + dev->if_hw_tsomaxsegsize = 65536; /* XXX can do up to 4GByte */ +#endif + dev->if_capenable = dev->if_capabilities; + + dev->if_hwassist = 0; + if (dev->if_capenable & (IFCAP_TSO4 | IFCAP_TSO6)) + dev->if_hwassist |= CSUM_TSO; + if (dev->if_capenable & IFCAP_TXCSUM) + dev->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP); + + + /* Register for VLAN events */ + priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, + mlx4_en_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST); + priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, + mlx4_en_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST); + + mdev->pndev[priv->port] = dev; + + priv->last_link_state = MLX4_DEV_EVENT_PORT_DOWN; + mlx4_en_set_default_moderation(priv); + + /* Set default MAC */ + for (i = 0; i < ETHER_ADDR_LEN; i++) + dev_addr[ETHER_ADDR_LEN - 1 - i] = (u8) (priv->mac >> (8 * i)); + + + ether_ifattach(dev, dev_addr); + if_link_state_change(dev, LINK_STATE_DOWN); + ifmedia_init(&priv->media, IFM_IMASK | IFM_ETH_FMASK, + mlx4_en_media_change, mlx4_en_media_status); + ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_1000_T, 0, NULL); + ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_10G_SR, 0, NULL); + ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_10G_CX4, 0, NULL); + ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_40G_CR4, 0, NULL); + ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO); + + en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num); + en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num); + + priv->registered = 1; + + en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num); + en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num); + + + priv->rx_mb_size = dev->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN; + err = mlx4_SET_PORT_general(mdev->dev, priv->port, + priv->rx_mb_size, + prof->tx_pause, prof->tx_ppp, + prof->rx_pause, prof->rx_ppp); + if (err) { + en_err(priv, "Failed setting port general configurations " + "for port %d, with error %d\n", priv->port, err); + goto out; } + + /* Init port */ + en_warn(priv, "Initializing port\n"); + err = mlx4_INIT_PORT(mdev->dev, priv->port); + if (err) { + en_err(priv, "Failed Initializing port\n"); + goto out; + } + + queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY); + + if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) + queue_delayed_work(mdev->workqueue, &priv->service_task, SERVICE_TASK_DELAY); + + + + return 0; + out: - mutex_unlock(&mdev->state_lock); + mlx4_en_destroy_netdev(dev); return err; } +static int mlx4_en_set_ring_size(struct net_device *dev, + int rx_size, int tx_size) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int port_up = 0; + int err = 0; + rx_size = roundup_pow_of_two(rx_size); + rx_size = max_t(u32, rx_size, MLX4_EN_MIN_RX_SIZE); + rx_size = min_t(u32, rx_size, MLX4_EN_MAX_RX_SIZE); + tx_size = roundup_pow_of_two(tx_size); + tx_size = max_t(u32, tx_size, MLX4_EN_MIN_TX_SIZE); + tx_size = min_t(u32, tx_size, MLX4_EN_MAX_TX_SIZE); + + if (rx_size == (priv->port_up ? + priv->rx_ring[0]->actual_size : priv->rx_ring[0]->size) && + tx_size == priv->tx_ring[0]->size) + return 0; + mutex_lock(&mdev->state_lock); + if (priv->port_up) { + port_up = 1; + mlx4_en_stop_port(dev); + } + mlx4_en_free_resources(priv); + priv->prof->tx_ring_size = tx_size; + priv->prof->rx_ring_size = rx_size; + err = mlx4_en_alloc_resources(priv); + if (err) { + en_err(priv, "Failed reallocating port resources\n"); + goto out; + } + if (port_up) { + err = mlx4_en_start_port(dev); + if (err) + en_err(priv, "Failed starting port\n"); + } +out: + mutex_unlock(&mdev->state_lock); + return err; +} static int mlx4_en_set_rx_ring_size(SYSCTL_HANDLER_ARGS) { - struct mlx4_en_priv *priv; - int size; - int error; + struct mlx4_en_priv *priv; + int size; + int error; - priv = arg1; - size = priv->prof->rx_ring_size; - error = sysctl_handle_int(oidp, &size, 0, req); - if (error || !req->newptr) - return (error); - error = -mlx4_en_set_ring_size(priv->dev, size, - priv->prof->tx_ring_size); - - return (error); + priv = arg1; + size = priv->prof->rx_ring_size; + error = sysctl_handle_int(oidp, &size, 0, req); + if (error || !req->newptr) + return (error); + error = -mlx4_en_set_ring_size(priv->dev, size, + priv->prof->tx_ring_size); + return (error); } static int mlx4_en_set_tx_ring_size(SYSCTL_HANDLER_ARGS) { - struct mlx4_en_priv *priv; - int size; - int error; + struct mlx4_en_priv *priv; + int size; + int error; - priv = arg1; - size = priv->prof->tx_ring_size; - error = sysctl_handle_int(oidp, &size, 0, req); - if (error || !req->newptr) - return (error); - error = -mlx4_en_set_ring_size(priv->dev, priv->prof->rx_ring_size, - size); + priv = arg1; + size = priv->prof->tx_ring_size; + error = sysctl_handle_int(oidp, &size, 0, req); + if (error || !req->newptr) + return (error); + error = -mlx4_en_set_ring_size(priv->dev, priv->prof->rx_ring_size, + size); - return (error); + return (error); } static int mlx4_en_set_tx_ppp(SYSCTL_HANDLER_ARGS) { - struct mlx4_en_priv *priv; - int ppp; - int error; + struct mlx4_en_priv *priv; + int ppp; + int error; - priv = arg1; - ppp = priv->prof->tx_ppp; - error = sysctl_handle_int(oidp, &ppp, 0, req); - if (error || !req->newptr) - return (error); - if (ppp > 0xff || ppp < 0) - return (-EINVAL); - priv->prof->tx_ppp = ppp; - error = -mlx4_SET_PORT_general(priv->mdev->dev, priv->port, - priv->rx_mb_size + ETHER_CRC_LEN, - priv->prof->tx_pause, - priv->prof->tx_ppp, - priv->prof->rx_pause, - priv->prof->rx_ppp); + priv = arg1; + ppp = priv->prof->tx_ppp; + error = sysctl_handle_int(oidp, &ppp, 0, req); + if (error || !req->newptr) + return (error); + if (ppp > 0xff || ppp < 0) + return (-EINVAL); + priv->prof->tx_ppp = ppp; + error = -mlx4_SET_PORT_general(priv->mdev->dev, priv->port, + priv->rx_mb_size + ETHER_CRC_LEN, + priv->prof->tx_pause, + priv->prof->tx_ppp, + priv->prof->rx_pause, + priv->prof->rx_ppp); - return (error); + return (error); } static int mlx4_en_set_rx_ppp(SYSCTL_HANDLER_ARGS) { - struct mlx4_en_priv *priv; - struct mlx4_en_dev *mdev; - int tx_ring_num; - int ppp; - int error; - int port_up; + struct mlx4_en_priv *priv; + struct mlx4_en_dev *mdev; + int ppp; + int error; + int port_up; - port_up = 0; - priv = arg1; - mdev = priv->mdev; - ppp = priv->prof->rx_ppp; - error = sysctl_handle_int(oidp, &ppp, 0, req); - if (error || !req->newptr) - return (error); - if (ppp > 0xff || ppp < 0) - return (-EINVAL); - /* See if we have to change the number of tx queues. */ - if (!ppp != !priv->prof->rx_ppp) { - tx_ring_num = MLX4_EN_NUM_HASH_RINGS + 1 + - (!!ppp) * MLX4_EN_NUM_PPP_RINGS; - mutex_lock(&mdev->state_lock); - if (priv->port_up) { - port_up = 1; - mlx4_en_stop_port(priv->dev); - } - mlx4_en_free_resources(priv); - priv->tx_ring_num = tx_ring_num; - priv->prof->rx_ppp = ppp; - error = -mlx4_en_alloc_resources(priv); - if (error) - en_err(priv, "Failed reallocating port resources\n"); - if (error == 0 && port_up) { - error = -mlx4_en_start_port(priv->dev); - if (error) - en_err(priv, "Failed starting port\n"); - } - mutex_unlock(&mdev->state_lock); - return (error); + port_up = 0; + priv = arg1; + mdev = priv->mdev; + ppp = priv->prof->rx_ppp; + error = sysctl_handle_int(oidp, &ppp, 0, req); + if (error || !req->newptr) + return (error); + if (ppp > 0xff || ppp < 0) + return (-EINVAL); + /* See if we have to change the number of tx queues. */ + if (!ppp != !priv->prof->rx_ppp) { + mutex_lock(&mdev->state_lock); + if (priv->port_up) { + port_up = 1; + mlx4_en_stop_port(priv->dev); + } + mlx4_en_free_resources(priv); + priv->prof->rx_ppp = ppp; + error = -mlx4_en_alloc_resources(priv); + if (error) + en_err(priv, "Failed reallocating port resources\n"); + if (error == 0 && port_up) { + error = -mlx4_en_start_port(priv->dev); + if (error) + en_err(priv, "Failed starting port\n"); + } + mutex_unlock(&mdev->state_lock); + return (error); - } - priv->prof->rx_ppp = ppp; - error = -mlx4_SET_PORT_general(priv->mdev->dev, priv->port, - priv->rx_mb_size + ETHER_CRC_LEN, - priv->prof->tx_pause, - priv->prof->tx_ppp, - priv->prof->rx_pause, - priv->prof->rx_ppp); + } + priv->prof->rx_ppp = ppp; + error = -mlx4_SET_PORT_general(priv->mdev->dev, priv->port, + priv->rx_mb_size + ETHER_CRC_LEN, + priv->prof->tx_pause, + priv->prof->tx_ppp, + priv->prof->rx_pause, + priv->prof->rx_ppp); - return (error); + return (error); } static void mlx4_en_sysctl_conf(struct mlx4_en_priv *priv) { - struct net_device *dev; - struct sysctl_ctx_list *ctx; - struct sysctl_oid *node; - struct sysctl_oid_list *node_list; - struct sysctl_oid *coal; - struct sysctl_oid_list *coal_list; + struct net_device *dev; + struct sysctl_ctx_list *ctx; + struct sysctl_oid *node; + struct sysctl_oid_list *node_list; + struct sysctl_oid *coal; + struct sysctl_oid_list *coal_list; - dev = priv->dev; - ctx = &priv->conf_ctx; + dev = priv->dev; + ctx = &priv->conf_ctx; - sysctl_ctx_init(ctx); - priv->sysctl = SYSCTL_ADD_NODE(ctx, SYSCTL_STATIC_CHILDREN(_hw), - OID_AUTO, dev->if_xname, CTLFLAG_RD, 0, "mlx4 10gig ethernet"); - node = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(priv->sysctl), OID_AUTO, - "conf", CTLFLAG_RD, NULL, "Configuration"); - node_list = SYSCTL_CHILDREN(node); + sysctl_ctx_init(ctx); + priv->sysctl = SYSCTL_ADD_NODE(ctx, SYSCTL_STATIC_CHILDREN(_hw), + OID_AUTO, dev->if_xname, CTLFLAG_RD, 0, "mlx4 10gig ethernet"); + node = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(priv->sysctl), OID_AUTO, + "conf", CTLFLAG_RD, NULL, "Configuration"); + node_list = SYSCTL_CHILDREN(node); - SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "msg_enable", - CTLFLAG_RW, &priv->msg_enable, 0, - "Driver message enable bitfield"); - SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "rx_rings", - CTLTYPE_INT | CTLFLAG_RD, &priv->rx_ring_num, 0, - "Number of receive rings"); - SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "tx_rings", - CTLTYPE_INT | CTLFLAG_RD, &priv->tx_ring_num, 0, - "Number of transmit rings"); - SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "rx_size", - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, - mlx4_en_set_rx_ring_size, "I", "Receive ring size"); - SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "tx_size", - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, - mlx4_en_set_tx_ring_size, "I", "Transmit ring size"); - SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "ip_reasm", - CTLFLAG_RW, &priv->ip_reasm, 0, - "Allow reassembly of IP fragments."); - SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "tx_ppp", - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, - mlx4_en_set_tx_ppp, "I", "TX Per-priority pause"); - SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "rx_ppp", - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, - mlx4_en_set_rx_ppp, "I", "RX Per-priority pause"); + SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "msg_enable", + CTLFLAG_RW, &priv->msg_enable, 0, + "Driver message enable bitfield"); + SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "rx_rings", + CTLTYPE_INT | CTLFLAG_RD, &priv->rx_ring_num, 0, + "Number of receive rings"); + SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "tx_rings", + CTLTYPE_INT | CTLFLAG_RD, &priv->tx_ring_num, 0, + "Number of transmit rings"); + SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "rx_size", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, + mlx4_en_set_rx_ring_size, "I", "Receive ring size"); + SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "tx_size", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, + mlx4_en_set_tx_ring_size, "I", "Transmit ring size"); + SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "tx_ppp", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, + mlx4_en_set_tx_ppp, "I", "TX Per-priority pause"); + SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "rx_ppp", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, + mlx4_en_set_rx_ppp, "I", "RX Per-priority pause"); - /* Add coalescer configuration. */ - coal = SYSCTL_ADD_NODE(ctx, node_list, OID_AUTO, - "coalesce", CTLFLAG_RD, NULL, "Interrupt coalesce configuration"); - coal_list = SYSCTL_CHILDREN(node); - SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "pkt_rate_low", - CTLFLAG_RW, &priv->pkt_rate_low, 0, - "Packets per-second for minimum delay"); - SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "rx_usecs_low", - CTLFLAG_RW, &priv->rx_usecs_low, 0, - "Minimum RX delay in micro-seconds"); - SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "pkt_rate_high", - CTLFLAG_RW, &priv->pkt_rate_high, 0, - "Packets per-second for maximum delay"); - SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "rx_usecs_high", - CTLFLAG_RW, &priv->rx_usecs_high, 0, - "Maximum RX delay in micro-seconds"); - SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "sample_interval", - CTLFLAG_RW, &priv->sample_interval, 0, - "adaptive frequency in units of HZ ticks"); - SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "adaptive_rx_coal", - CTLFLAG_RW, &priv->adaptive_rx_coal, 0, - "Enable adaptive rx coalescing"); + /* Add coalescer configuration. */ + coal = SYSCTL_ADD_NODE(ctx, node_list, OID_AUTO, + "coalesce", CTLFLAG_RD, NULL, "Interrupt coalesce configuration"); + coal_list = SYSCTL_CHILDREN(node); + SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "pkt_rate_low", + CTLFLAG_RW, &priv->pkt_rate_low, 0, + "Packets per-second for minimum delay"); + SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "rx_usecs_low", + CTLFLAG_RW, &priv->rx_usecs_low, 0, + "Minimum RX delay in micro-seconds"); + SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "pkt_rate_high", + CTLFLAG_RW, &priv->pkt_rate_high, 0, + "Packets per-second for maximum delay"); + SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "rx_usecs_high", + CTLFLAG_RW, &priv->rx_usecs_high, 0, + "Maximum RX delay in micro-seconds"); + SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "sample_interval", + CTLFLAG_RW, &priv->sample_interval, 0, + "adaptive frequency in units of HZ ticks"); + SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "adaptive_rx_coal", + CTLFLAG_RW, &priv->adaptive_rx_coal, 0, + "Enable adaptive rx coalescing"); } + static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv) { struct net_device *dev; struct sysctl_ctx_list *ctx; struct sysctl_oid *node; struct sysctl_oid_list *node_list; struct sysctl_oid *ring_node; struct sysctl_oid_list *ring_list; struct mlx4_en_tx_ring *tx_ring; struct mlx4_en_rx_ring *rx_ring; char namebuf[128]; int i; dev = priv->dev; ctx = &priv->stat_ctx; sysctl_ctx_init(ctx); node = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(priv->sysctl), OID_AUTO, "stat", CTLFLAG_RD, NULL, "Statistics"); node_list = SYSCTL_CHILDREN(node); #ifdef MLX4_EN_PERF_STAT SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "tx_poll", CTLFLAG_RD, &priv->pstats.tx_poll, "TX Poll calls"); SYSCTL_ADD_QUAD(ctx, node_list, OID_AUTO, "tx_pktsz_avg", CTLFLAG_RD, &priv->pstats.tx_pktsz_avg, "TX average packet size"); SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "inflight_avg", CTLFLAG_RD, &priv->pstats.inflight_avg, "TX average packets in-flight"); SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "tx_coal_avg", CTLFLAG_RD, &priv->pstats.tx_coal_avg, "TX average coalesced completions"); SYSCTL_ADD_UINT(ctx, node_list, OID_AUTO, "rx_coal_avg", CTLFLAG_RD, &priv->pstats.rx_coal_avg, "RX average coalesced completions"); #endif SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tso_packets", CTLFLAG_RD, &priv->port_stats.tso_packets, "TSO packets sent"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "queue_stopped", CTLFLAG_RD, &priv->port_stats.queue_stopped, "Queue full"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "wake_queue", CTLFLAG_RD, &priv->port_stats.wake_queue, "Queue resumed after full"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_timeout", CTLFLAG_RD, &priv->port_stats.tx_timeout, "Transmit timeouts"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_alloc_failed", CTLFLAG_RD, &priv->port_stats.rx_alloc_failed, "RX failed to allocate mbuf"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_chksum_good", CTLFLAG_RD, &priv->port_stats.rx_chksum_good, "RX checksum offload success"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_chksum_none", CTLFLAG_RD, &priv->port_stats.rx_chksum_none, "RX without checksum offload"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_chksum_offload", CTLFLAG_RD, &priv->port_stats.tx_chksum_offload, "TX checksum offloads"); /* Could strdup the names and add in a loop. This is simpler. */ - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "broadcast", CTLFLAG_RD, - &priv->pkstats.broadcast, "Broadcast packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_prio0", CTLFLAG_RD, - &priv->pkstats.tx_prio[0], "TX Priority 0 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_prio1", CTLFLAG_RD, - &priv->pkstats.tx_prio[1], "TX Priority 1 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_prio2", CTLFLAG_RD, - &priv->pkstats.tx_prio[2], "TX Priority 2 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_prio3", CTLFLAG_RD, - &priv->pkstats.tx_prio[3], "TX Priority 3 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_prio4", CTLFLAG_RD, - &priv->pkstats.tx_prio[4], "TX Priority 4 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_prio5", CTLFLAG_RD, - &priv->pkstats.tx_prio[5], "TX Priority 5 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_prio6", CTLFLAG_RD, - &priv->pkstats.tx_prio[6], "TX Priority 6 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_prio7", CTLFLAG_RD, - &priv->pkstats.tx_prio[7], "TX Priority 7 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_prio0", CTLFLAG_RD, - &priv->pkstats.rx_prio[0], "RX Priority 0 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_prio1", CTLFLAG_RD, - &priv->pkstats.rx_prio[1], "RX Priority 1 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_prio2", CTLFLAG_RD, - &priv->pkstats.rx_prio[2], "RX Priority 2 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_prio3", CTLFLAG_RD, - &priv->pkstats.rx_prio[3], "RX Priority 3 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_prio4", CTLFLAG_RD, - &priv->pkstats.rx_prio[4], "RX Priority 4 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_prio5", CTLFLAG_RD, - &priv->pkstats.rx_prio[5], "RX Priority 5 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_prio6", CTLFLAG_RD, - &priv->pkstats.rx_prio[6], "RX Priority 6 packets"); - SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_prio7", CTLFLAG_RD, - &priv->pkstats.rx_prio[7], "RX Priority 7 packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_bytes", CTLFLAG_RD, + &priv->pkstats.rx_bytes, "RX Bytes"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_packets", CTLFLAG_RD, + &priv->pkstats.rx_packets, "RX packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_multicast_packets", CTLFLAG_RD, + &priv->pkstats.rx_multicast_packets, "RX Multicast Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_broadcast_packets", CTLFLAG_RD, + &priv->pkstats.rx_broadcast_packets, "RX Broadcast Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_errors", CTLFLAG_RD, + &priv->pkstats.rx_errors, "RX Errors"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_dropped", CTLFLAG_RD, + &priv->pkstats.rx_dropped, "RX Dropped"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_length_errors", CTLFLAG_RD, + &priv->pkstats.rx_length_errors, "RX Length Errors"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_over_errors", CTLFLAG_RD, + &priv->pkstats.rx_over_errors, "RX Over Errors"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_crc_errors", CTLFLAG_RD, + &priv->pkstats.rx_crc_errors, "RX CRC Errors"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_jabbers", CTLFLAG_RD, + &priv->pkstats.rx_jabbers, "RX Jabbers"); + + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_in_range_length_error", CTLFLAG_RD, + &priv->pkstats.rx_in_range_length_error, "RX IN_Range Length Error"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_out_range_length_error", + CTLFLAG_RD, &priv->pkstats.rx_out_range_length_error, + "RX Out Range Length Error"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_lt_64_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_lt_64_bytes_packets, "RX Lt 64 Bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_127_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_127_bytes_packets, "RX 127 bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_255_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_255_bytes_packets, "RX 255 bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_511_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_511_bytes_packets, "RX 511 bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_1023_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_1023_bytes_packets, "RX 1023 bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_1518_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_1518_bytes_packets, "RX 1518 bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_1522_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_1522_bytes_packets, "RX 1522 bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_1548_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_1548_bytes_packets, "RX 1548 bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_gt_1548_bytes_packets", CTLFLAG_RD, + &priv->pkstats.rx_gt_1548_bytes_packets, + "RX Greater Then 1548 bytes Packets"); + +struct mlx4_en_pkt_stats { + unsigned long tx_packets; + unsigned long tx_bytes; + unsigned long tx_multicast_packets; + unsigned long tx_broadcast_packets; + unsigned long tx_errors; + unsigned long tx_dropped; + unsigned long tx_lt_64_bytes_packets; + unsigned long tx_127_bytes_packets; + unsigned long tx_255_bytes_packets; + unsigned long tx_511_bytes_packets; + unsigned long tx_1023_bytes_packets; + unsigned long tx_1518_bytes_packets; + unsigned long tx_1522_bytes_packets; + unsigned long tx_1548_bytes_packets; + unsigned long tx_gt_1548_bytes_packets; + unsigned long rx_prio[NUM_PRIORITIES][NUM_PRIORITY_STATS]; + unsigned long tx_prio[NUM_PRIORITIES][NUM_PRIORITY_STATS]; +#define NUM_PKT_STATS 72 +}; + + + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_packets", CTLFLAG_RD, + &priv->pkstats.tx_packets, "TX packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_bytes", CTLFLAG_RD, + &priv->pkstats.tx_packets, "TX Bytes"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_multicast_packets", CTLFLAG_RD, + &priv->pkstats.tx_multicast_packets, "TX Multicast Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_broadcast_packets", CTLFLAG_RD, + &priv->pkstats.tx_broadcast_packets, "TX Broadcast Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_errors", CTLFLAG_RD, + &priv->pkstats.tx_errors, "TX Errors"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_dropped", CTLFLAG_RD, + &priv->pkstats.tx_dropped, "TX Dropped"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_lt_64_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_lt_64_bytes_packets, "TX Less Then 64 Bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_127_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_127_bytes_packets, "TX 127 Bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_255_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_255_bytes_packets, "TX 255 Bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_511_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_511_bytes_packets, "TX 511 Bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_1023_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_1023_bytes_packets, "TX 1023 Bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_1518_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_1518_bytes_packets, "TX 1518 Bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_1522_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_1522_bytes_packets, "TX 1522 Bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_1548_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_1548_bytes_packets, "TX 1548 Bytes Packets"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_gt_1548_bytes_packets", CTLFLAG_RD, + &priv->pkstats.tx_gt_1548_bytes_packets, + "TX Greater Then 1548 Bytes Packets"); + + + for (i = 0; i < priv->tx_ring_num; i++) { - tx_ring = &priv->tx_ring[i]; + tx_ring = priv->tx_ring[i]; snprintf(namebuf, sizeof(namebuf), "tx_ring%d", i); ring_node = SYSCTL_ADD_NODE(ctx, node_list, OID_AUTO, namebuf, CTLFLAG_RD, NULL, "TX Ring"); ring_list = SYSCTL_CHILDREN(ring_node); SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "packets", CTLFLAG_RD, &tx_ring->packets, "TX packets"); SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "bytes", CTLFLAG_RD, &tx_ring->bytes, "TX bytes"); - SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "error", - CTLFLAG_RD, &tx_ring->errors, "TX soft errors"); } for (i = 0; i < priv->rx_ring_num; i++) { - rx_ring = &priv->rx_ring[i]; + rx_ring = priv->rx_ring[i]; snprintf(namebuf, sizeof(namebuf), "rx_ring%d", i); ring_node = SYSCTL_ADD_NODE(ctx, node_list, OID_AUTO, namebuf, CTLFLAG_RD, NULL, "RX Ring"); ring_list = SYSCTL_CHILDREN(ring_node); SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "packets", CTLFLAG_RD, &rx_ring->packets, "RX packets"); SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "bytes", CTLFLAG_RD, &rx_ring->bytes, "RX bytes"); SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "error", CTLFLAG_RD, &rx_ring->errors, "RX soft errors"); - SYSCTL_ADD_UINT(ctx, ring_list, OID_AUTO, "lro_queued", - CTLFLAG_RD, &rx_ring->lro.lro_queued, 0, "LRO Queued"); - SYSCTL_ADD_UINT(ctx, ring_list, OID_AUTO, "lro_flushed", - CTLFLAG_RD, &rx_ring->lro.lro_flushed, 0, "LRO Flushed"); } } - -int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, - struct mlx4_en_port_profile *prof) -{ - static volatile int mlx4_en_unit; - struct net_device *dev; - struct mlx4_en_priv *priv; - uint8_t dev_addr[ETHER_ADDR_LEN]; - int err; - int i; - - priv = kzalloc(sizeof(*priv), GFP_KERNEL); - dev = priv->dev = if_alloc(IFT_ETHER); - if (dev == NULL) { - mlx4_err(mdev, "Net device allocation failed\n"); - kfree(priv); - return -ENOMEM; - } - dev->if_softc = priv; - if_initname(dev, "mlxen", atomic_fetchadd_int(&mlx4_en_unit, 1)); - dev->if_mtu = ETHERMTU; - dev->if_baudrate = 1000000000; - dev->if_init = mlx4_en_init; - dev->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; - dev->if_ioctl = mlx4_en_ioctl; - dev->if_transmit = mlx4_en_transmit; - dev->if_qflush = mlx4_en_qflush; - dev->if_snd.ifq_maxlen = prof->tx_ring_size; - - /* - * Initialize driver private data - */ - priv->dev = dev; - priv->mdev = mdev; - priv->prof = prof; - priv->port = port; - priv->port_up = false; - priv->rx_csum = 1; - priv->flags = prof->flags; - priv->tx_ring_num = prof->tx_ring_num; - priv->rx_ring_num = prof->rx_ring_num; - priv->mac_index = -1; - priv->msg_enable = MLX4_EN_MSG_LEVEL; - priv->ip_reasm = priv->mdev->profile.ip_reasm; - mtx_init(&priv->stats_lock.m, "mlx4 stats", NULL, MTX_DEF); - mtx_init(&priv->vlan_lock.m, "mlx4 vlan", NULL, MTX_DEF); - INIT_WORK(&priv->mcast_task, mlx4_en_do_set_multicast); - INIT_WORK(&priv->watchdog_task, mlx4_en_restart); - INIT_WORK(&priv->linkstate_task, mlx4_en_linkstate); - INIT_DELAYED_WORK(&priv->stats_task, mlx4_en_do_get_stats); - callout_init(&priv->watchdog_timer, 1); - - /* Query for default mac and max mtu */ - priv->max_mtu = mdev->dev->caps.eth_mtu_cap[priv->port]; - priv->mac = mdev->dev->caps.def_mac[priv->port]; - - if (ILLEGAL_MAC(priv->mac)) { - en_err(priv, "Port: %d, invalid mac burned: 0x%llx, quiting\n", - priv->port, priv->mac); - err = -EINVAL; - goto out; - } - - mlx4_en_sysctl_conf(priv); - - err = mlx4_en_alloc_resources(priv); - if (err) - goto out; - - /* Allocate page for receive rings */ - err = mlx4_alloc_hwq_res(mdev->dev, &priv->res, - MLX4_EN_PAGE_SIZE, MLX4_EN_PAGE_SIZE); - if (err) { - en_err(priv, "Failed to allocate page for rx qps\n"); - goto out; - } - priv->allocated = 1; - - /* - * Set driver features - */ - dev->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; - dev->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING; - dev->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER; - dev->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU; - if (mdev->LSO_support) - dev->if_capabilities |= IFCAP_TSO4 | IFCAP_VLAN_HWTSO; - if (mdev->profile.num_lro) - dev->if_capabilities |= IFCAP_LRO; - dev->if_capenable = dev->if_capabilities; - /* - * Setup wake-on-lan. - */ - if (priv->mdev->dev->caps.wol) { - u64 config; - if (mlx4_wol_read(priv->mdev->dev, &config, priv->port) == 0) { - if (config & MLX4_EN_WOL_MAGIC) - dev->if_capabilities |= IFCAP_WOL_MAGIC; - if (config & MLX4_EN_WOL_ENABLED) - dev->if_capenable |= IFCAP_WOL_MAGIC; - } - } - - /* Register for VLAN events */ - priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, - mlx4_en_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST); - priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, - mlx4_en_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST); - - mdev->pndev[priv->port] = dev; - - priv->last_link_state = MLX4_DEV_EVENT_PORT_DOWN; - if_link_state_change(dev, LINK_STATE_DOWN); - - /* Set default MAC */ - for (i = 0; i < ETHER_ADDR_LEN; i++) - dev_addr[ETHER_ADDR_LEN - 1 - i] = (u8) (priv->mac >> (8 * i)); - - ether_ifattach(dev, dev_addr); - ifmedia_init(&priv->media, IFM_IMASK | IFM_ETH_FMASK, - mlx4_en_media_change, mlx4_en_media_status); - ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_1000_T, 0, NULL); - ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_10G_SR, 0, NULL); - ifmedia_add(&priv->media, IFM_ETHER | IFM_FDX | IFM_10G_CX4, 0, NULL); - ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO); - - en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num); - en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num); - - priv->registered = 1; - queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY); - - return 0; - -out: - mlx4_en_destroy_netdev(dev); - return err; -} - Index: stable/9/sys/ofed/drivers/net/mlx4/en_port.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/en_port.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/en_port.c (revision 273864) @@ -1,325 +1,573 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ - -#include "mlx4_en.h" - +#include #include #include #include +#include "en_port.h" +#include "mlx4_en.h" +#define EN_IFQ_MIN_INTERVAL 3000 -int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, - u64 mac, u64 clear, u8 mode) -{ - return mlx4_cmd(dev, (mac | (clear << 63)), port, mode, - MLX4_CMD_SET_MCAST_FLTR, MLX4_CMD_TIME_CLASS_B); -} -int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, u8 port, u32 *vlans) +int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, struct mlx4_en_priv *priv) { struct mlx4_cmd_mailbox *mailbox; struct mlx4_set_vlan_fltr_mbox *filter; - int i, j; + int i; + int j; + int index = 0; + u32 entry; int err = 0; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); filter = mailbox->buf; - memset(filter, 0, sizeof *filter); - if (vlans) - for (i = 0, j = VLAN_FLTR_SIZE - 1; i < VLAN_FLTR_SIZE; - i++, j--) - filter->entry[j] = cpu_to_be32(vlans[i]); - err = mlx4_cmd(dev, mailbox->dma, port, 0, MLX4_CMD_SET_VLAN_FLTR, - MLX4_CMD_TIME_CLASS_B); + memset(filter, 0, sizeof(*filter)); + for (i = VLAN_FLTR_SIZE - 1; i >= 0; i--) { + entry = 0; + for (j = 0; j < 32; j++) + if (test_bit(index, priv->active_vlans)) + entry |= 1 << j; + index++; + filter->entry[i] = cpu_to_be32(entry); + } + err = mlx4_cmd(dev, mailbox->dma, priv->port, 0, MLX4_CMD_SET_VLAN_FLTR, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); mlx4_free_cmd_mailbox(dev, mailbox); return err; } - -int mlx4_SET_PORT_general(struct mlx4_dev *dev, u8 port, int mtu, - u8 pptx, u8 pfctx, u8 pprx, u8 pfcrx) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_set_port_general_context *context; - int err; - u32 in_mod; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - context = mailbox->buf; - memset(context, 0, sizeof *context); - - context->flags = SET_PORT_GEN_ALL_VALID; - context->mtu = cpu_to_be16(mtu); - context->pptx = (pptx * (!pfctx)) << 7; - context->pfctx = pfctx; - context->pprx = (pprx * (!pfcrx)) << 7; - context->pfcrx = pfcrx; - - in_mod = MLX4_SET_PORT_GENERAL << 8 | port; - err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - -int mlx4_SET_PORT_qpn_calc(struct mlx4_dev *dev, u8 port, u32 base_qpn, - u8 promisc) -{ - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_set_port_rqp_calc_context *context; - int err; - u32 in_mod; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - context = mailbox->buf; - memset(context, 0, sizeof *context); - - context->base_qpn = cpu_to_be32(base_qpn); - context->promisc = cpu_to_be32(promisc << SET_PORT_PROMISC_EN_SHIFT | base_qpn); - context->mcast = cpu_to_be32((dev->caps.mc_promisc_mode << - SET_PORT_PROMISC_MODE_SHIFT) | base_qpn); - context->intra_no_vlan = 0; - context->no_vlan = MLX4_NO_VLAN_IDX; - context->intra_vlan_miss = 0; - context->vlan_miss = MLX4_VLAN_MISS_IDX; - - in_mod = MLX4_SET_PORT_RQP_CALC << 8 | port; - err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B); - - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} - int mlx4_en_QUERY_PORT(struct mlx4_en_dev *mdev, u8 port) { struct mlx4_en_query_port_context *qport_context; struct mlx4_en_priv *priv = netdev_priv(mdev->pndev[port]); struct mlx4_en_port_state *state = &priv->port_state; struct mlx4_cmd_mailbox *mailbox; int err; mailbox = mlx4_alloc_cmd_mailbox(mdev->dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); memset(mailbox->buf, 0, sizeof(*qport_context)); err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, port, 0, - MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B); + MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); if (err) goto out; qport_context = mailbox->buf; /* This command is always accessed from Ethtool context * already synchronized, no need in locking */ state->link_state = !!(qport_context->link_up & MLX4_EN_LINK_UP_MASK); switch (qport_context->link_speed & MLX4_EN_SPEED_MASK) { case MLX4_EN_1G_SPEED: state->link_speed = 1000; break; case MLX4_EN_10G_SPEED_XAUI: case MLX4_EN_10G_SPEED_XFI: state->link_speed = 10000; break; + case MLX4_EN_20G_SPEED: + state->link_speed = 20000; + break; case MLX4_EN_40G_SPEED: state->link_speed = 40000; break; + case MLX4_EN_56G_SPEED: + state->link_speed = 56000; + break; default: state->link_speed = -1; break; } state->transciver = qport_context->transceiver; - if (be32_to_cpu(qport_context->transceiver_code_hi) & 0x400) - state->transciver = 0x80; + state->autoneg = !!(qport_context->autoneg & MLX4_EN_AUTONEG_MASK); out: mlx4_free_cmd_mailbox(mdev->dev, mailbox); return err; } -static int read_iboe_counters(struct mlx4_dev *dev, int index, u64 counters[]) +int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset) { - struct mlx4_cmd_mailbox *mailbox; + struct mlx4_en_stat_out_mbox *mlx4_en_stats; + struct mlx4_en_stat_out_flow_control_mbox *flowstats; + struct mlx4_en_priv *priv = netdev_priv(mdev->pndev[port]); + struct mlx4_en_vport_stats *vport_stats = &priv->vport_stats; + struct mlx4_cmd_mailbox *mailbox = NULL; + struct mlx4_cmd_mailbox *mailbox_flow = NULL; + u64 in_mod = reset << 8 | port; int err; - int mode; - struct mlx4_counters_ext *ext; - struct mlx4_counters *reg; + int i; + int do_if_stat = 1; + unsigned long period = (unsigned long) (jiffies - priv->last_ifq_jiffies); + struct mlx4_en_vport_stats tmp_vport_stats; + struct net_device *dev; - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return -ENOMEM; + if (jiffies_to_msecs(period) < EN_IFQ_MIN_INTERVAL || + priv->counter_index == 0xff) + do_if_stat = 0; - err = mlx4_cmd_box(dev, 0, mailbox->dma, index, 0, - MLX4_CMD_QUERY_IF_STAT, MLX4_CMD_TIME_CLASS_C); - if (err) - goto out; + mailbox = mlx4_alloc_cmd_mailbox(mdev->dev); + if (IS_ERR(mailbox)) { + err = PTR_ERR(mailbox); + goto mailbox_out; + } - mode = be32_to_cpu(((struct mlx4_counters *)mailbox->buf)->counter_mode) & 0xf; - switch (mode) { - case 0: - reg = mailbox->buf; - counters[0] = be64_to_cpu(reg->rx_frames); - counters[1] = be64_to_cpu(reg->tx_frames); - counters[2] = be64_to_cpu(reg->rx_bytes); - counters[3] = be64_to_cpu(reg->tx_bytes); - break; - case 1: - ext = mailbox->buf; - counters[0] = be64_to_cpu(ext->rx_uni_frames); - counters[1] = be64_to_cpu(ext->tx_uni_frames); - counters[2] = be64_to_cpu(ext->rx_uni_bytes); - counters[3] = be64_to_cpu(ext->tx_uni_bytes); - break; - default: - err = -EINVAL; + mailbox_flow = mlx4_alloc_cmd_mailbox(mdev->dev); + if (IS_ERR(mailbox_flow)) { + mlx4_free_cmd_mailbox(mdev->dev, mailbox); + err = PTR_ERR(mailbox_flow); + goto mailbox_out; } -out: - mlx4_free_cmd_mailbox(dev, mailbox); - return err; -} + /* 0xffs indicates invalid value */ + memset(mailbox_flow->buf, 0xff, sizeof(*flowstats) * + MLX4_NUM_PRIORITIES); -int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset) -{ - struct mlx4_en_stat_out_mbox *mlx4_en_stats; - struct net_device *dev; - struct mlx4_en_priv *priv; - struct mlx4_cmd_mailbox *mailbox; - u64 in_mod = reset << 8 | port; - unsigned long oerror; - unsigned long ierror; - int err; - int i; - int counter; - u64 counters[4]; + if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) { + memset(mailbox_flow->buf, 0, sizeof(*flowstats)); + err = mlx4_cmd_box(mdev->dev, 0, mailbox_flow->dma, + in_mod | 1<<12, 0, MLX4_CMD_DUMP_ETH_STATS, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); - dev = mdev->pndev[port]; - priv = netdev_priv(dev); - memset(counters, 0, sizeof counters); - counter = mlx4_get_iboe_counter(priv->mdev->dev, port); - if (counter >= 0) - err = read_iboe_counters(priv->mdev->dev, counter, counters); + if (err) + goto out; + } - mailbox = mlx4_alloc_cmd_mailbox(mdev->dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - memset(mailbox->buf, 0, sizeof(*mlx4_en_stats)); err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, in_mod, 0, - MLX4_CMD_DUMP_ETH_STATS, MLX4_CMD_TIME_CLASS_B); + MLX4_CMD_DUMP_ETH_STATS, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); if (err) goto out; mlx4_en_stats = mailbox->buf; spin_lock(&priv->stats_lock); - oerror = ierror = 0; - dev->if_ipackets = counters[0]; - dev->if_ibytes = counters[2]; + priv->port_stats.rx_chksum_good = 0; + priv->port_stats.rx_chksum_none = 0; for (i = 0; i < priv->rx_ring_num; i++) { - dev->if_ipackets += priv->rx_ring[i].packets; - dev->if_ibytes += priv->rx_ring[i].bytes; - ierror += priv->rx_ring[i].errors; + priv->port_stats.rx_chksum_good += priv->rx_ring[i]->csum_ok; + priv->port_stats.rx_chksum_none += priv->rx_ring[i]->csum_none; } - dev->if_opackets = counters[1]; - dev->if_obytes = counters[3]; - for (i = 0; i <= priv->tx_ring_num; i++) { - dev->if_opackets += priv->tx_ring[i].packets; - dev->if_obytes += priv->tx_ring[i].bytes; - oerror += priv->tx_ring[i].errors; + + priv->port_stats.tx_chksum_offload = 0; + priv->port_stats.queue_stopped = 0; + priv->port_stats.wake_queue = 0; + for (i = 0; i < priv->tx_ring_num; i++) { + priv->port_stats.tx_chksum_offload += priv->tx_ring[i]->tx_csum; + priv->port_stats.queue_stopped += priv->tx_ring[i]->queue_stopped; + priv->port_stats.wake_queue += priv->tx_ring[i]->wake_queue; } + /* RX Statistics */ + priv->pkstats.rx_packets = be64_to_cpu(mlx4_en_stats->RTOT_prio_0) + + be64_to_cpu(mlx4_en_stats->RTOT_prio_1) + + be64_to_cpu(mlx4_en_stats->RTOT_prio_2) + + be64_to_cpu(mlx4_en_stats->RTOT_prio_3) + + be64_to_cpu(mlx4_en_stats->RTOT_prio_4) + + be64_to_cpu(mlx4_en_stats->RTOT_prio_5) + + be64_to_cpu(mlx4_en_stats->RTOT_prio_6) + + be64_to_cpu(mlx4_en_stats->RTOT_prio_7) + + be64_to_cpu(mlx4_en_stats->RTOT_novlan); + priv->pkstats.rx_bytes = be64_to_cpu(mlx4_en_stats->ROCT_prio_0) + + be64_to_cpu(mlx4_en_stats->ROCT_prio_1) + + be64_to_cpu(mlx4_en_stats->ROCT_prio_2) + + be64_to_cpu(mlx4_en_stats->ROCT_prio_3) + + be64_to_cpu(mlx4_en_stats->ROCT_prio_4) + + be64_to_cpu(mlx4_en_stats->ROCT_prio_5) + + be64_to_cpu(mlx4_en_stats->ROCT_prio_6) + + be64_to_cpu(mlx4_en_stats->ROCT_prio_7) + + be64_to_cpu(mlx4_en_stats->ROCT_novlan); + priv->pkstats.rx_multicast_packets = be64_to_cpu(mlx4_en_stats->MCAST_prio_0) + + be64_to_cpu(mlx4_en_stats->MCAST_prio_1) + + be64_to_cpu(mlx4_en_stats->MCAST_prio_2) + + be64_to_cpu(mlx4_en_stats->MCAST_prio_3) + + be64_to_cpu(mlx4_en_stats->MCAST_prio_4) + + be64_to_cpu(mlx4_en_stats->MCAST_prio_5) + + be64_to_cpu(mlx4_en_stats->MCAST_prio_6) + + be64_to_cpu(mlx4_en_stats->MCAST_prio_7) + + be64_to_cpu(mlx4_en_stats->MCAST_novlan); + priv->pkstats.rx_broadcast_packets = be64_to_cpu(mlx4_en_stats->RBCAST_prio_0) + + be64_to_cpu(mlx4_en_stats->RBCAST_prio_1) + + be64_to_cpu(mlx4_en_stats->RBCAST_prio_2) + + be64_to_cpu(mlx4_en_stats->RBCAST_prio_3) + + be64_to_cpu(mlx4_en_stats->RBCAST_prio_4) + + be64_to_cpu(mlx4_en_stats->RBCAST_prio_5) + + be64_to_cpu(mlx4_en_stats->RBCAST_prio_6) + + be64_to_cpu(mlx4_en_stats->RBCAST_prio_7) + + be64_to_cpu(mlx4_en_stats->RBCAST_novlan); + priv->pkstats.rx_errors = be64_to_cpu(mlx4_en_stats->PCS) + + be32_to_cpu(mlx4_en_stats->RJBBR) + + be32_to_cpu(mlx4_en_stats->RCRC) + + be32_to_cpu(mlx4_en_stats->RRUNT) + + be64_to_cpu(mlx4_en_stats->RInRangeLengthErr) + + be64_to_cpu(mlx4_en_stats->ROutRangeLengthErr) + + be32_to_cpu(mlx4_en_stats->RSHORT) + + be64_to_cpu(mlx4_en_stats->RGIANT_prio_0) + + be64_to_cpu(mlx4_en_stats->RGIANT_prio_1) + + be64_to_cpu(mlx4_en_stats->RGIANT_prio_2) + + be64_to_cpu(mlx4_en_stats->RGIANT_prio_3) + + be64_to_cpu(mlx4_en_stats->RGIANT_prio_4) + + be64_to_cpu(mlx4_en_stats->RGIANT_prio_5) + + be64_to_cpu(mlx4_en_stats->RGIANT_prio_6) + + be64_to_cpu(mlx4_en_stats->RGIANT_prio_7) + + be64_to_cpu(mlx4_en_stats->RGIANT_novlan); + priv->pkstats.rx_dropped = be32_to_cpu(mlx4_en_stats->RdropOvflw); + priv->pkstats.rx_length_errors = be32_to_cpu(mlx4_en_stats->RdropLength); + priv->pkstats.rx_over_errors = be32_to_cpu(mlx4_en_stats->RdropOvflw); + priv->pkstats.rx_crc_errors = be32_to_cpu(mlx4_en_stats->RCRC); + priv->pkstats.rx_jabbers = be32_to_cpu(mlx4_en_stats->RJBBR); + priv->pkstats.rx_in_range_length_error = be64_to_cpu(mlx4_en_stats->RInRangeLengthErr); + priv->pkstats.rx_out_range_length_error = be64_to_cpu(mlx4_en_stats->ROutRangeLengthErr); + priv->pkstats.rx_lt_64_bytes_packets = be64_to_cpu(mlx4_en_stats->R64_prio_0) + + be64_to_cpu(mlx4_en_stats->R64_prio_1) + + be64_to_cpu(mlx4_en_stats->R64_prio_2) + + be64_to_cpu(mlx4_en_stats->R64_prio_3) + + be64_to_cpu(mlx4_en_stats->R64_prio_4) + + be64_to_cpu(mlx4_en_stats->R64_prio_5) + + be64_to_cpu(mlx4_en_stats->R64_prio_6) + + be64_to_cpu(mlx4_en_stats->R64_prio_7) + + be64_to_cpu(mlx4_en_stats->R64_novlan); + priv->pkstats.rx_127_bytes_packets = be64_to_cpu(mlx4_en_stats->R127_prio_0) + + be64_to_cpu(mlx4_en_stats->R127_prio_1) + + be64_to_cpu(mlx4_en_stats->R127_prio_2) + + be64_to_cpu(mlx4_en_stats->R127_prio_3) + + be64_to_cpu(mlx4_en_stats->R127_prio_4) + + be64_to_cpu(mlx4_en_stats->R127_prio_5) + + be64_to_cpu(mlx4_en_stats->R127_prio_6) + + be64_to_cpu(mlx4_en_stats->R127_prio_7) + + be64_to_cpu(mlx4_en_stats->R127_novlan); + priv->pkstats.rx_255_bytes_packets = be64_to_cpu(mlx4_en_stats->R255_prio_0) + + be64_to_cpu(mlx4_en_stats->R255_prio_1) + + be64_to_cpu(mlx4_en_stats->R255_prio_2) + + be64_to_cpu(mlx4_en_stats->R255_prio_3) + + be64_to_cpu(mlx4_en_stats->R255_prio_4) + + be64_to_cpu(mlx4_en_stats->R255_prio_5) + + be64_to_cpu(mlx4_en_stats->R255_prio_6) + + be64_to_cpu(mlx4_en_stats->R255_prio_7) + + be64_to_cpu(mlx4_en_stats->R255_novlan); + priv->pkstats.rx_511_bytes_packets = be64_to_cpu(mlx4_en_stats->R511_prio_0) + + be64_to_cpu(mlx4_en_stats->R511_prio_1) + + be64_to_cpu(mlx4_en_stats->R511_prio_2) + + be64_to_cpu(mlx4_en_stats->R511_prio_3) + + be64_to_cpu(mlx4_en_stats->R511_prio_4) + + be64_to_cpu(mlx4_en_stats->R511_prio_5) + + be64_to_cpu(mlx4_en_stats->R511_prio_6) + + be64_to_cpu(mlx4_en_stats->R511_prio_7) + + be64_to_cpu(mlx4_en_stats->R511_novlan); + priv->pkstats.rx_1023_bytes_packets = be64_to_cpu(mlx4_en_stats->R1023_prio_0) + + be64_to_cpu(mlx4_en_stats->R1023_prio_1) + + be64_to_cpu(mlx4_en_stats->R1023_prio_2) + + be64_to_cpu(mlx4_en_stats->R1023_prio_3) + + be64_to_cpu(mlx4_en_stats->R1023_prio_4) + + be64_to_cpu(mlx4_en_stats->R1023_prio_5) + + be64_to_cpu(mlx4_en_stats->R1023_prio_6) + + be64_to_cpu(mlx4_en_stats->R1023_prio_7) + + be64_to_cpu(mlx4_en_stats->R1023_novlan); + priv->pkstats.rx_1518_bytes_packets = be64_to_cpu(mlx4_en_stats->R1518_prio_0) + + be64_to_cpu(mlx4_en_stats->R1518_prio_1) + + be64_to_cpu(mlx4_en_stats->R1518_prio_2) + + be64_to_cpu(mlx4_en_stats->R1518_prio_3) + + be64_to_cpu(mlx4_en_stats->R1518_prio_4) + + be64_to_cpu(mlx4_en_stats->R1518_prio_5) + + be64_to_cpu(mlx4_en_stats->R1518_prio_6) + + be64_to_cpu(mlx4_en_stats->R1518_prio_7) + + be64_to_cpu(mlx4_en_stats->R1518_novlan); + priv->pkstats.rx_1522_bytes_packets = be64_to_cpu(mlx4_en_stats->R1522_prio_0) + + be64_to_cpu(mlx4_en_stats->R1522_prio_1) + + be64_to_cpu(mlx4_en_stats->R1522_prio_2) + + be64_to_cpu(mlx4_en_stats->R1522_prio_3) + + be64_to_cpu(mlx4_en_stats->R1522_prio_4) + + be64_to_cpu(mlx4_en_stats->R1522_prio_5) + + be64_to_cpu(mlx4_en_stats->R1522_prio_6) + + be64_to_cpu(mlx4_en_stats->R1522_prio_7) + + be64_to_cpu(mlx4_en_stats->R1522_novlan); + priv->pkstats.rx_1548_bytes_packets = be64_to_cpu(mlx4_en_stats->R1548_prio_0) + + be64_to_cpu(mlx4_en_stats->R1548_prio_1) + + be64_to_cpu(mlx4_en_stats->R1548_prio_2) + + be64_to_cpu(mlx4_en_stats->R1548_prio_3) + + be64_to_cpu(mlx4_en_stats->R1548_prio_4) + + be64_to_cpu(mlx4_en_stats->R1548_prio_5) + + be64_to_cpu(mlx4_en_stats->R1548_prio_6) + + be64_to_cpu(mlx4_en_stats->R1548_prio_7) + + be64_to_cpu(mlx4_en_stats->R1548_novlan); + priv->pkstats.rx_gt_1548_bytes_packets = be64_to_cpu(mlx4_en_stats->R2MTU_prio_0) + + be64_to_cpu(mlx4_en_stats->R2MTU_prio_1) + + be64_to_cpu(mlx4_en_stats->R2MTU_prio_2) + + be64_to_cpu(mlx4_en_stats->R2MTU_prio_3) + + be64_to_cpu(mlx4_en_stats->R2MTU_prio_4) + + be64_to_cpu(mlx4_en_stats->R2MTU_prio_5) + + be64_to_cpu(mlx4_en_stats->R2MTU_prio_6) + + be64_to_cpu(mlx4_en_stats->R2MTU_prio_7) + + be64_to_cpu(mlx4_en_stats->R2MTU_novlan); - dev->if_ierrors = be32_to_cpu(mlx4_en_stats->RDROP) + ierror; - dev->if_oerrors = be32_to_cpu(mlx4_en_stats->TDROP) + oerror; - dev->if_imcasts = be64_to_cpu(mlx4_en_stats->MCAST_prio_0) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_1) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_2) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_3) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_4) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_5) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_6) + - be64_to_cpu(mlx4_en_stats->MCAST_prio_7) + - be64_to_cpu(mlx4_en_stats->MCAST_novlan); - dev->if_omcasts = be64_to_cpu(mlx4_en_stats->TMCAST_prio_0) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_1) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_2) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_3) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_4) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_5) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_6) + - be64_to_cpu(mlx4_en_stats->TMCAST_prio_7) + - be64_to_cpu(mlx4_en_stats->TMCAST_novlan); - dev->if_collisions = 0; + /* Tx Stats */ + priv->pkstats.tx_packets = be64_to_cpu(mlx4_en_stats->TTOT_prio_0) + + be64_to_cpu(mlx4_en_stats->TTOT_prio_1) + + be64_to_cpu(mlx4_en_stats->TTOT_prio_2) + + be64_to_cpu(mlx4_en_stats->TTOT_prio_3) + + be64_to_cpu(mlx4_en_stats->TTOT_prio_4) + + be64_to_cpu(mlx4_en_stats->TTOT_prio_5) + + be64_to_cpu(mlx4_en_stats->TTOT_prio_6) + + be64_to_cpu(mlx4_en_stats->TTOT_prio_7) + + be64_to_cpu(mlx4_en_stats->TTOT_novlan); + priv->pkstats.tx_bytes = be64_to_cpu(mlx4_en_stats->TOCT_prio_0) + + be64_to_cpu(mlx4_en_stats->TOCT_prio_1) + + be64_to_cpu(mlx4_en_stats->TOCT_prio_2) + + be64_to_cpu(mlx4_en_stats->TOCT_prio_3) + + be64_to_cpu(mlx4_en_stats->TOCT_prio_4) + + be64_to_cpu(mlx4_en_stats->TOCT_prio_5) + + be64_to_cpu(mlx4_en_stats->TOCT_prio_6) + + be64_to_cpu(mlx4_en_stats->TOCT_prio_7) + + be64_to_cpu(mlx4_en_stats->TOCT_novlan); + priv->pkstats.tx_multicast_packets = be64_to_cpu(mlx4_en_stats->TMCAST_prio_0) + + be64_to_cpu(mlx4_en_stats->TMCAST_prio_1) + + be64_to_cpu(mlx4_en_stats->TMCAST_prio_2) + + be64_to_cpu(mlx4_en_stats->TMCAST_prio_3) + + be64_to_cpu(mlx4_en_stats->TMCAST_prio_4) + + be64_to_cpu(mlx4_en_stats->TMCAST_prio_5) + + be64_to_cpu(mlx4_en_stats->TMCAST_prio_6) + + be64_to_cpu(mlx4_en_stats->TMCAST_prio_7) + + be64_to_cpu(mlx4_en_stats->TMCAST_novlan); + priv->pkstats.tx_broadcast_packets = be64_to_cpu(mlx4_en_stats->TBCAST_prio_0) + + be64_to_cpu(mlx4_en_stats->TBCAST_prio_1) + + be64_to_cpu(mlx4_en_stats->TBCAST_prio_2) + + be64_to_cpu(mlx4_en_stats->TBCAST_prio_3) + + be64_to_cpu(mlx4_en_stats->TBCAST_prio_4) + + be64_to_cpu(mlx4_en_stats->TBCAST_prio_5) + + be64_to_cpu(mlx4_en_stats->TBCAST_prio_6) + + be64_to_cpu(mlx4_en_stats->TBCAST_prio_7) + + be64_to_cpu(mlx4_en_stats->TBCAST_novlan); + priv->pkstats.tx_errors = be64_to_cpu(mlx4_en_stats->TGIANT_prio_0) + + be64_to_cpu(mlx4_en_stats->TGIANT_prio_1) + + be64_to_cpu(mlx4_en_stats->TGIANT_prio_2) + + be64_to_cpu(mlx4_en_stats->TGIANT_prio_3) + + be64_to_cpu(mlx4_en_stats->TGIANT_prio_4) + + be64_to_cpu(mlx4_en_stats->TGIANT_prio_5) + + be64_to_cpu(mlx4_en_stats->TGIANT_prio_6) + + be64_to_cpu(mlx4_en_stats->TGIANT_prio_7) + + be64_to_cpu(mlx4_en_stats->TGIANT_novlan); + priv->pkstats.tx_dropped = be32_to_cpu(mlx4_en_stats->TDROP) - + priv->pkstats.tx_errors; + priv->pkstats.tx_lt_64_bytes_packets = be64_to_cpu(mlx4_en_stats->T64_prio_0) + + be64_to_cpu(mlx4_en_stats->T64_prio_1) + + be64_to_cpu(mlx4_en_stats->T64_prio_2) + + be64_to_cpu(mlx4_en_stats->T64_prio_3) + + be64_to_cpu(mlx4_en_stats->T64_prio_4) + + be64_to_cpu(mlx4_en_stats->T64_prio_5) + + be64_to_cpu(mlx4_en_stats->T64_prio_6) + + be64_to_cpu(mlx4_en_stats->T64_prio_7) + + be64_to_cpu(mlx4_en_stats->T64_novlan); + priv->pkstats.tx_127_bytes_packets = be64_to_cpu(mlx4_en_stats->T127_prio_0) + + be64_to_cpu(mlx4_en_stats->T127_prio_1) + + be64_to_cpu(mlx4_en_stats->T127_prio_2) + + be64_to_cpu(mlx4_en_stats->T127_prio_3) + + be64_to_cpu(mlx4_en_stats->T127_prio_4) + + be64_to_cpu(mlx4_en_stats->T127_prio_5) + + be64_to_cpu(mlx4_en_stats->T127_prio_6) + + be64_to_cpu(mlx4_en_stats->T127_prio_7) + + be64_to_cpu(mlx4_en_stats->T127_novlan); + priv->pkstats.tx_255_bytes_packets = be64_to_cpu(mlx4_en_stats->T255_prio_0) + + be64_to_cpu(mlx4_en_stats->T255_prio_1) + + be64_to_cpu(mlx4_en_stats->T255_prio_2) + + be64_to_cpu(mlx4_en_stats->T255_prio_3) + + be64_to_cpu(mlx4_en_stats->T255_prio_4) + + be64_to_cpu(mlx4_en_stats->T255_prio_5) + + be64_to_cpu(mlx4_en_stats->T255_prio_6) + + be64_to_cpu(mlx4_en_stats->T255_prio_7) + + be64_to_cpu(mlx4_en_stats->T255_novlan); + priv->pkstats.tx_511_bytes_packets = be64_to_cpu(mlx4_en_stats->T511_prio_0) + + be64_to_cpu(mlx4_en_stats->T511_prio_1) + + be64_to_cpu(mlx4_en_stats->T511_prio_2) + + be64_to_cpu(mlx4_en_stats->T511_prio_3) + + be64_to_cpu(mlx4_en_stats->T511_prio_4) + + be64_to_cpu(mlx4_en_stats->T511_prio_5) + + be64_to_cpu(mlx4_en_stats->T511_prio_6) + + be64_to_cpu(mlx4_en_stats->T511_prio_7) + + be64_to_cpu(mlx4_en_stats->T511_novlan); + priv->pkstats.tx_1023_bytes_packets = be64_to_cpu(mlx4_en_stats->T1023_prio_0) + + be64_to_cpu(mlx4_en_stats->T1023_prio_1) + + be64_to_cpu(mlx4_en_stats->T1023_prio_2) + + be64_to_cpu(mlx4_en_stats->T1023_prio_3) + + be64_to_cpu(mlx4_en_stats->T1023_prio_4) + + be64_to_cpu(mlx4_en_stats->T1023_prio_5) + + be64_to_cpu(mlx4_en_stats->T1023_prio_6) + + be64_to_cpu(mlx4_en_stats->T1023_prio_7) + + be64_to_cpu(mlx4_en_stats->T1023_novlan); + priv->pkstats.tx_1518_bytes_packets = be64_to_cpu(mlx4_en_stats->T1518_prio_0) + + be64_to_cpu(mlx4_en_stats->T1518_prio_1) + + be64_to_cpu(mlx4_en_stats->T1518_prio_2) + + be64_to_cpu(mlx4_en_stats->T1518_prio_3) + + be64_to_cpu(mlx4_en_stats->T1518_prio_4) + + be64_to_cpu(mlx4_en_stats->T1518_prio_5) + + be64_to_cpu(mlx4_en_stats->T1518_prio_6) + + be64_to_cpu(mlx4_en_stats->T1518_prio_7) + + be64_to_cpu(mlx4_en_stats->T1518_novlan); + priv->pkstats.tx_1522_bytes_packets = be64_to_cpu(mlx4_en_stats->T1522_prio_0) + + be64_to_cpu(mlx4_en_stats->T1522_prio_1) + + be64_to_cpu(mlx4_en_stats->T1522_prio_2) + + be64_to_cpu(mlx4_en_stats->T1522_prio_3) + + be64_to_cpu(mlx4_en_stats->T1522_prio_4) + + be64_to_cpu(mlx4_en_stats->T1522_prio_5) + + be64_to_cpu(mlx4_en_stats->T1522_prio_6) + + be64_to_cpu(mlx4_en_stats->T1522_prio_7) + + be64_to_cpu(mlx4_en_stats->T1522_novlan); + priv->pkstats.tx_1548_bytes_packets = be64_to_cpu(mlx4_en_stats->T1548_prio_0) + + be64_to_cpu(mlx4_en_stats->T1548_prio_1) + + be64_to_cpu(mlx4_en_stats->T1548_prio_2) + + be64_to_cpu(mlx4_en_stats->T1548_prio_3) + + be64_to_cpu(mlx4_en_stats->T1548_prio_4) + + be64_to_cpu(mlx4_en_stats->T1548_prio_5) + + be64_to_cpu(mlx4_en_stats->T1548_prio_6) + + be64_to_cpu(mlx4_en_stats->T1548_prio_7) + + be64_to_cpu(mlx4_en_stats->T1548_novlan); + priv->pkstats.tx_gt_1548_bytes_packets = be64_to_cpu(mlx4_en_stats->T2MTU_prio_0) + + be64_to_cpu(mlx4_en_stats->T2MTU_prio_1) + + be64_to_cpu(mlx4_en_stats->T2MTU_prio_2) + + be64_to_cpu(mlx4_en_stats->T2MTU_prio_3) + + be64_to_cpu(mlx4_en_stats->T2MTU_prio_4) + + be64_to_cpu(mlx4_en_stats->T2MTU_prio_5) + + be64_to_cpu(mlx4_en_stats->T2MTU_prio_6) + + be64_to_cpu(mlx4_en_stats->T2MTU_prio_7) + + be64_to_cpu(mlx4_en_stats->T2MTU_novlan); - priv->pkstats.broadcast = - be64_to_cpu(mlx4_en_stats->RBCAST_prio_0) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_1) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_2) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_3) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_4) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_5) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_6) + - be64_to_cpu(mlx4_en_stats->RBCAST_prio_7) + - be64_to_cpu(mlx4_en_stats->RBCAST_novlan); - priv->pkstats.rx_prio[0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_0); - priv->pkstats.rx_prio[1] = be64_to_cpu(mlx4_en_stats->RTOT_prio_1); - priv->pkstats.rx_prio[2] = be64_to_cpu(mlx4_en_stats->RTOT_prio_2); - priv->pkstats.rx_prio[3] = be64_to_cpu(mlx4_en_stats->RTOT_prio_3); - priv->pkstats.rx_prio[4] = be64_to_cpu(mlx4_en_stats->RTOT_prio_4); - priv->pkstats.rx_prio[5] = be64_to_cpu(mlx4_en_stats->RTOT_prio_5); - priv->pkstats.rx_prio[6] = be64_to_cpu(mlx4_en_stats->RTOT_prio_6); - priv->pkstats.rx_prio[7] = be64_to_cpu(mlx4_en_stats->RTOT_prio_7); - priv->pkstats.tx_prio[0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_0); - priv->pkstats.tx_prio[1] = be64_to_cpu(mlx4_en_stats->TTOT_prio_1); - priv->pkstats.tx_prio[2] = be64_to_cpu(mlx4_en_stats->TTOT_prio_2); - priv->pkstats.tx_prio[3] = be64_to_cpu(mlx4_en_stats->TTOT_prio_3); - priv->pkstats.tx_prio[4] = be64_to_cpu(mlx4_en_stats->TTOT_prio_4); - priv->pkstats.tx_prio[5] = be64_to_cpu(mlx4_en_stats->TTOT_prio_5); - priv->pkstats.tx_prio[6] = be64_to_cpu(mlx4_en_stats->TTOT_prio_6); - priv->pkstats.tx_prio[7] = be64_to_cpu(mlx4_en_stats->TTOT_prio_7); + priv->pkstats.rx_prio[0][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_0); + priv->pkstats.rx_prio[0][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_0); + priv->pkstats.rx_prio[1][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_1); + priv->pkstats.rx_prio[1][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_1); + priv->pkstats.rx_prio[2][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_2); + priv->pkstats.rx_prio[2][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_2); + priv->pkstats.rx_prio[3][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_3); + priv->pkstats.rx_prio[3][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_3); + priv->pkstats.rx_prio[4][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_4); + priv->pkstats.rx_prio[4][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_4); + priv->pkstats.rx_prio[5][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_5); + priv->pkstats.rx_prio[5][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_5); + priv->pkstats.rx_prio[6][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_6); + priv->pkstats.rx_prio[6][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_6); + priv->pkstats.rx_prio[7][0] = be64_to_cpu(mlx4_en_stats->RTOT_prio_7); + priv->pkstats.rx_prio[7][1] = be64_to_cpu(mlx4_en_stats->ROCT_prio_7); + priv->pkstats.rx_prio[8][0] = be64_to_cpu(mlx4_en_stats->RTOT_novlan); + priv->pkstats.rx_prio[8][1] = be64_to_cpu(mlx4_en_stats->ROCT_novlan); + priv->pkstats.tx_prio[0][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_0); + priv->pkstats.tx_prio[0][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_0); + priv->pkstats.tx_prio[1][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_1); + priv->pkstats.tx_prio[1][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_1); + priv->pkstats.tx_prio[2][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_2); + priv->pkstats.tx_prio[2][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_2); + priv->pkstats.tx_prio[3][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_3); + priv->pkstats.tx_prio[3][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_3); + priv->pkstats.tx_prio[4][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_4); + priv->pkstats.tx_prio[4][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_4); + priv->pkstats.tx_prio[5][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_5); + priv->pkstats.tx_prio[5][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_5); + priv->pkstats.tx_prio[6][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_6); + priv->pkstats.tx_prio[6][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_6); + priv->pkstats.tx_prio[7][0] = be64_to_cpu(mlx4_en_stats->TTOT_prio_7); + priv->pkstats.tx_prio[7][1] = be64_to_cpu(mlx4_en_stats->TOCT_prio_7); + priv->pkstats.tx_prio[8][0] = be64_to_cpu(mlx4_en_stats->TTOT_novlan); + priv->pkstats.tx_prio[8][1] = be64_to_cpu(mlx4_en_stats->TOCT_novlan); + + flowstats = mailbox_flow->buf; + + for (i = 0; i < MLX4_NUM_PRIORITIES; i++) { + priv->flowstats[i].rx_pause = + be64_to_cpu(flowstats[i].rx_pause); + priv->flowstats[i].rx_pause_duration = + be64_to_cpu(flowstats[i].rx_pause_duration); + priv->flowstats[i].rx_pause_transition = + be64_to_cpu(flowstats[i].rx_pause_transition); + priv->flowstats[i].tx_pause = + be64_to_cpu(flowstats[i].tx_pause); + priv->flowstats[i].tx_pause_duration = + be64_to_cpu(flowstats[i].tx_pause_duration); + priv->flowstats[i].tx_pause_transition = + be64_to_cpu(flowstats[i].tx_pause_transition); + } + + memset(&tmp_vport_stats, 0, sizeof(tmp_vport_stats)); spin_unlock(&priv->stats_lock); + err = mlx4_get_vport_ethtool_stats(mdev->dev, port, + &tmp_vport_stats, reset); + spin_lock(&priv->stats_lock); + if (!err) { + /* ethtool stats format */ + vport_stats->rx_unicast_packets = tmp_vport_stats.rx_unicast_packets; + vport_stats->rx_unicast_bytes = tmp_vport_stats.rx_unicast_bytes; + vport_stats->rx_multicast_packets = tmp_vport_stats.rx_multicast_packets; + vport_stats->rx_multicast_bytes = tmp_vport_stats.rx_multicast_bytes; + vport_stats->rx_broadcast_packets = tmp_vport_stats.rx_broadcast_packets; + vport_stats->rx_broadcast_bytes = tmp_vport_stats.rx_broadcast_bytes; + vport_stats->rx_dropped = tmp_vport_stats.rx_dropped; + vport_stats->rx_errors = tmp_vport_stats.rx_errors; + vport_stats->tx_unicast_packets = tmp_vport_stats.tx_unicast_packets; + vport_stats->tx_unicast_bytes = tmp_vport_stats.tx_unicast_bytes; + vport_stats->tx_multicast_packets = tmp_vport_stats.tx_multicast_packets; + vport_stats->tx_multicast_bytes = tmp_vport_stats.tx_multicast_bytes; + vport_stats->tx_broadcast_packets = tmp_vport_stats.tx_broadcast_packets; + vport_stats->tx_broadcast_bytes = tmp_vport_stats.tx_broadcast_bytes; + vport_stats->tx_errors = tmp_vport_stats.tx_errors; + } + if (!mlx4_is_mfunc(mdev->dev)) { + /* netdevice stats format */ + dev = mdev->pndev[port]; + dev->if_ipackets = priv->pkstats.rx_packets; + dev->if_opackets = priv->pkstats.tx_packets; + dev->if_ibytes = priv->pkstats.rx_bytes; + dev->if_obytes = priv->pkstats.tx_bytes; + dev->if_ierrors = priv->pkstats.rx_errors; + dev->if_iqdrops = priv->pkstats.rx_dropped; + dev->if_imcasts = priv->pkstats.rx_multicast_packets; + dev->if_omcasts = priv->pkstats.tx_multicast_packets; + dev->if_collisions = 0; + } + + spin_unlock(&priv->stats_lock); + out: + mlx4_free_cmd_mailbox(mdev->dev, mailbox_flow); mlx4_free_cmd_mailbox(mdev->dev, mailbox); + +mailbox_out: + if (do_if_stat) + priv->last_ifq_jiffies = jiffies; + return err; } - Index: stable/9/sys/ofed/drivers/net/mlx4/en_port.h =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/en_port.h (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/en_port.h (revision 273864) @@ -1,598 +1,563 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ #ifndef _MLX4_EN_PORT_H_ #define _MLX4_EN_PORT_H_ #define SET_PORT_GEN_ALL_VALID 0x7 -#define SET_PORT_PROMISC_EN_SHIFT 31 -#define SET_PORT_PROMISC_MODE_SHIFT 30 +#define SET_PORT_PROMISC_SHIFT 31 +#define SET_PORT_MC_PROMISC_SHIFT 30 -enum { - MLX4_CMD_SET_VLAN_FLTR = 0x47, - MLX4_CMD_SET_MCAST_FLTR = 0x48, - MLX4_CMD_DUMP_ETH_STATS = 0x49, -}; +#define MLX4_EN_NUM_TC 8 -struct mlx4_set_port_general_context { - u8 reserved[3]; - u8 flags; - u16 reserved2; - __be16 mtu; - u8 pptx; - u8 pfctx; - u16 reserved3; - u8 pprx; - u8 pfcrx; - u16 reserved4; -}; - -struct mlx4_set_port_rqp_calc_context { - __be32 base_qpn; - __be32 flags; - u8 reserved[3]; - u8 mac_miss; - u8 intra_no_vlan; - u8 no_vlan; - u8 intra_vlan_miss; - u8 vlan_miss; - u8 reserved2[3]; - u8 no_vlan_prio; - __be32 promisc; - __be32 mcast; -}; - #define VLAN_FLTR_SIZE 128 struct mlx4_set_vlan_fltr_mbox { __be32 entry[VLAN_FLTR_SIZE]; }; enum { MLX4_MCAST_CONFIG = 0, MLX4_MCAST_DISABLE = 1, MLX4_MCAST_ENABLE = 2, }; enum { - MLX4_EN_1G_SPEED = 0x02, - MLX4_EN_10G_SPEED_XFI = 0x01, MLX4_EN_10G_SPEED_XAUI = 0x00, + MLX4_EN_10G_SPEED_XFI = 0x01, + MLX4_EN_1G_SPEED = 0x02, + MLX4_EN_20G_SPEED = 0x08, MLX4_EN_40G_SPEED = 0x40, + MLX4_EN_56G_SPEED = 0x20, MLX4_EN_OTHER_SPEED = 0x0f, }; struct mlx4_en_query_port_context { u8 link_up; #define MLX4_EN_LINK_UP_MASK 0x80 - u8 reserved; + u8 autoneg; +#define MLX4_EN_AUTONEG_MASK 0x80 __be16 mtu; u8 reserved2; u8 link_speed; -#define MLX4_EN_SPEED_MASK 0x43 +#define MLX4_EN_SPEED_MASK 0x6b u16 reserved3[5]; __be64 mac; u8 transceiver; - u8 reserved4[3]; - __be32 wavelenth; - u32 reserved5; - __be32 transceiver_code_hi; - __be32 transceiver_code_low; }; struct mlx4_en_stat_out_mbox { /* Received frames with a length of 64 octets */ __be64 R64_prio_0; __be64 R64_prio_1; __be64 R64_prio_2; __be64 R64_prio_3; __be64 R64_prio_4; __be64 R64_prio_5; __be64 R64_prio_6; __be64 R64_prio_7; __be64 R64_novlan; /* Received frames with a length of 127 octets */ __be64 R127_prio_0; __be64 R127_prio_1; __be64 R127_prio_2; __be64 R127_prio_3; __be64 R127_prio_4; __be64 R127_prio_5; __be64 R127_prio_6; __be64 R127_prio_7; __be64 R127_novlan; /* Received frames with a length of 255 octets */ __be64 R255_prio_0; __be64 R255_prio_1; __be64 R255_prio_2; __be64 R255_prio_3; __be64 R255_prio_4; __be64 R255_prio_5; __be64 R255_prio_6; __be64 R255_prio_7; __be64 R255_novlan; /* Received frames with a length of 511 octets */ __be64 R511_prio_0; __be64 R511_prio_1; __be64 R511_prio_2; __be64 R511_prio_3; __be64 R511_prio_4; __be64 R511_prio_5; __be64 R511_prio_6; __be64 R511_prio_7; __be64 R511_novlan; /* Received frames with a length of 1023 octets */ __be64 R1023_prio_0; __be64 R1023_prio_1; __be64 R1023_prio_2; __be64 R1023_prio_3; __be64 R1023_prio_4; __be64 R1023_prio_5; __be64 R1023_prio_6; __be64 R1023_prio_7; __be64 R1023_novlan; /* Received frames with a length of 1518 octets */ __be64 R1518_prio_0; __be64 R1518_prio_1; __be64 R1518_prio_2; __be64 R1518_prio_3; __be64 R1518_prio_4; __be64 R1518_prio_5; __be64 R1518_prio_6; __be64 R1518_prio_7; __be64 R1518_novlan; /* Received frames with a length of 1522 octets */ __be64 R1522_prio_0; __be64 R1522_prio_1; __be64 R1522_prio_2; __be64 R1522_prio_3; __be64 R1522_prio_4; __be64 R1522_prio_5; __be64 R1522_prio_6; __be64 R1522_prio_7; __be64 R1522_novlan; /* Received frames with a length of 1548 octets */ __be64 R1548_prio_0; __be64 R1548_prio_1; __be64 R1548_prio_2; __be64 R1548_prio_3; __be64 R1548_prio_4; __be64 R1548_prio_5; __be64 R1548_prio_6; __be64 R1548_prio_7; __be64 R1548_novlan; /* Received frames with a length of 1548 < octets < MTU */ __be64 R2MTU_prio_0; __be64 R2MTU_prio_1; __be64 R2MTU_prio_2; __be64 R2MTU_prio_3; __be64 R2MTU_prio_4; __be64 R2MTU_prio_5; __be64 R2MTU_prio_6; __be64 R2MTU_prio_7; __be64 R2MTU_novlan; /* Received frames with a length of MTU< octets and good CRC */ __be64 RGIANT_prio_0; __be64 RGIANT_prio_1; __be64 RGIANT_prio_2; __be64 RGIANT_prio_3; __be64 RGIANT_prio_4; __be64 RGIANT_prio_5; __be64 RGIANT_prio_6; __be64 RGIANT_prio_7; __be64 RGIANT_novlan; /* Received broadcast frames with good CRC */ __be64 RBCAST_prio_0; __be64 RBCAST_prio_1; __be64 RBCAST_prio_2; __be64 RBCAST_prio_3; __be64 RBCAST_prio_4; __be64 RBCAST_prio_5; __be64 RBCAST_prio_6; __be64 RBCAST_prio_7; __be64 RBCAST_novlan; /* Received multicast frames with good CRC */ __be64 MCAST_prio_0; __be64 MCAST_prio_1; __be64 MCAST_prio_2; __be64 MCAST_prio_3; __be64 MCAST_prio_4; __be64 MCAST_prio_5; __be64 MCAST_prio_6; __be64 MCAST_prio_7; __be64 MCAST_novlan; /* Received unicast not short or GIANT frames with good CRC */ __be64 RTOTG_prio_0; __be64 RTOTG_prio_1; __be64 RTOTG_prio_2; __be64 RTOTG_prio_3; __be64 RTOTG_prio_4; __be64 RTOTG_prio_5; __be64 RTOTG_prio_6; __be64 RTOTG_prio_7; __be64 RTOTG_novlan; /* Count of total octets of received frames, includes framing characters */ __be64 RTTLOCT_prio_0; /* Count of total octets of received frames, not including framing characters */ __be64 RTTLOCT_NOFRM_prio_0; /* Count of Total number of octets received (only for frames without errors) */ __be64 ROCT_prio_0; __be64 RTTLOCT_prio_1; __be64 RTTLOCT_NOFRM_prio_1; __be64 ROCT_prio_1; __be64 RTTLOCT_prio_2; __be64 RTTLOCT_NOFRM_prio_2; __be64 ROCT_prio_2; __be64 RTTLOCT_prio_3; __be64 RTTLOCT_NOFRM_prio_3; __be64 ROCT_prio_3; __be64 RTTLOCT_prio_4; __be64 RTTLOCT_NOFRM_prio_4; __be64 ROCT_prio_4; __be64 RTTLOCT_prio_5; __be64 RTTLOCT_NOFRM_prio_5; __be64 ROCT_prio_5; __be64 RTTLOCT_prio_6; __be64 RTTLOCT_NOFRM_prio_6; __be64 ROCT_prio_6; __be64 RTTLOCT_prio_7; __be64 RTTLOCT_NOFRM_prio_7; __be64 ROCT_prio_7; __be64 RTTLOCT_novlan; __be64 RTTLOCT_NOFRM_novlan; __be64 ROCT_novlan; /* Count of Total received frames including bad frames */ __be64 RTOT_prio_0; /* Count of Total number of received frames with 802.1Q encapsulation */ __be64 R1Q_prio_0; __be64 reserved1; __be64 RTOT_prio_1; __be64 R1Q_prio_1; __be64 reserved2; __be64 RTOT_prio_2; __be64 R1Q_prio_2; __be64 reserved3; __be64 RTOT_prio_3; __be64 R1Q_prio_3; __be64 reserved4; __be64 RTOT_prio_4; __be64 R1Q_prio_4; __be64 reserved5; __be64 RTOT_prio_5; __be64 R1Q_prio_5; __be64 reserved6; __be64 RTOT_prio_6; __be64 R1Q_prio_6; __be64 reserved7; __be64 RTOT_prio_7; __be64 R1Q_prio_7; __be64 reserved8; __be64 RTOT_novlan; __be64 R1Q_novlan; __be64 reserved9; /* Total number of Successfully Received Control Frames */ __be64 RCNTL; __be64 reserved10; __be64 reserved11; __be64 reserved12; /* Count of received frames with a length/type field value between 46 (42 for VLANtagged frames) and 1500 (also 1500 for VLAN-tagged frames), inclusive */ __be64 RInRangeLengthErr; /* Count of received frames with length/type field between 1501 and 1535 decimal, inclusive */ __be64 ROutRangeLengthErr; /* Count of received frames that are longer than max allowed size for 802.3 frames (1518/1522) */ __be64 RFrmTooLong; /* Count frames received with PCS error */ __be64 PCS; /* Transmit frames with a length of 64 octets */ __be64 T64_prio_0; __be64 T64_prio_1; __be64 T64_prio_2; __be64 T64_prio_3; __be64 T64_prio_4; __be64 T64_prio_5; __be64 T64_prio_6; __be64 T64_prio_7; __be64 T64_novlan; __be64 T64_loopbk; /* Transmit frames with a length of 65 to 127 octets. */ __be64 T127_prio_0; __be64 T127_prio_1; __be64 T127_prio_2; __be64 T127_prio_3; __be64 T127_prio_4; __be64 T127_prio_5; __be64 T127_prio_6; __be64 T127_prio_7; __be64 T127_novlan; __be64 T127_loopbk; /* Transmit frames with a length of 128 to 255 octets */ __be64 T255_prio_0; __be64 T255_prio_1; __be64 T255_prio_2; __be64 T255_prio_3; __be64 T255_prio_4; __be64 T255_prio_5; __be64 T255_prio_6; __be64 T255_prio_7; __be64 T255_novlan; __be64 T255_loopbk; /* Transmit frames with a length of 256 to 511 octets */ __be64 T511_prio_0; __be64 T511_prio_1; __be64 T511_prio_2; __be64 T511_prio_3; __be64 T511_prio_4; __be64 T511_prio_5; __be64 T511_prio_6; __be64 T511_prio_7; __be64 T511_novlan; __be64 T511_loopbk; /* Transmit frames with a length of 512 to 1023 octets */ __be64 T1023_prio_0; __be64 T1023_prio_1; __be64 T1023_prio_2; __be64 T1023_prio_3; __be64 T1023_prio_4; __be64 T1023_prio_5; __be64 T1023_prio_6; __be64 T1023_prio_7; __be64 T1023_novlan; __be64 T1023_loopbk; /* Transmit frames with a length of 1024 to 1518 octets */ __be64 T1518_prio_0; __be64 T1518_prio_1; __be64 T1518_prio_2; __be64 T1518_prio_3; __be64 T1518_prio_4; __be64 T1518_prio_5; __be64 T1518_prio_6; __be64 T1518_prio_7; __be64 T1518_novlan; __be64 T1518_loopbk; /* Counts transmit frames with a length of 1519 to 1522 bytes */ __be64 T1522_prio_0; __be64 T1522_prio_1; __be64 T1522_prio_2; __be64 T1522_prio_3; __be64 T1522_prio_4; __be64 T1522_prio_5; __be64 T1522_prio_6; __be64 T1522_prio_7; __be64 T1522_novlan; __be64 T1522_loopbk; /* Transmit frames with a length of 1523 to 1548 octets */ __be64 T1548_prio_0; __be64 T1548_prio_1; __be64 T1548_prio_2; __be64 T1548_prio_3; __be64 T1548_prio_4; __be64 T1548_prio_5; __be64 T1548_prio_6; __be64 T1548_prio_7; __be64 T1548_novlan; __be64 T1548_loopbk; /* Counts transmit frames with a length of 1549 to MTU bytes */ __be64 T2MTU_prio_0; __be64 T2MTU_prio_1; __be64 T2MTU_prio_2; __be64 T2MTU_prio_3; __be64 T2MTU_prio_4; __be64 T2MTU_prio_5; __be64 T2MTU_prio_6; __be64 T2MTU_prio_7; __be64 T2MTU_novlan; __be64 T2MTU_loopbk; /* Transmit frames with a length greater than MTU octets and a good CRC. */ __be64 TGIANT_prio_0; __be64 TGIANT_prio_1; __be64 TGIANT_prio_2; __be64 TGIANT_prio_3; __be64 TGIANT_prio_4; __be64 TGIANT_prio_5; __be64 TGIANT_prio_6; __be64 TGIANT_prio_7; __be64 TGIANT_novlan; __be64 TGIANT_loopbk; /* Transmit broadcast frames with a good CRC */ __be64 TBCAST_prio_0; __be64 TBCAST_prio_1; __be64 TBCAST_prio_2; __be64 TBCAST_prio_3; __be64 TBCAST_prio_4; __be64 TBCAST_prio_5; __be64 TBCAST_prio_6; __be64 TBCAST_prio_7; __be64 TBCAST_novlan; __be64 TBCAST_loopbk; /* Transmit multicast frames with a good CRC */ __be64 TMCAST_prio_0; __be64 TMCAST_prio_1; __be64 TMCAST_prio_2; __be64 TMCAST_prio_3; __be64 TMCAST_prio_4; __be64 TMCAST_prio_5; __be64 TMCAST_prio_6; __be64 TMCAST_prio_7; __be64 TMCAST_novlan; __be64 TMCAST_loopbk; /* Transmit good frames that are neither broadcast nor multicast */ __be64 TTOTG_prio_0; __be64 TTOTG_prio_1; __be64 TTOTG_prio_2; __be64 TTOTG_prio_3; __be64 TTOTG_prio_4; __be64 TTOTG_prio_5; __be64 TTOTG_prio_6; __be64 TTOTG_prio_7; __be64 TTOTG_novlan; __be64 TTOTG_loopbk; /* total octets of transmitted frames, including framing characters */ __be64 TTTLOCT_prio_0; /* total octets of transmitted frames, not including framing characters */ __be64 TTTLOCT_NOFRM_prio_0; /* ifOutOctets */ __be64 TOCT_prio_0; __be64 TTTLOCT_prio_1; __be64 TTTLOCT_NOFRM_prio_1; __be64 TOCT_prio_1; __be64 TTTLOCT_prio_2; __be64 TTTLOCT_NOFRM_prio_2; __be64 TOCT_prio_2; __be64 TTTLOCT_prio_3; __be64 TTTLOCT_NOFRM_prio_3; __be64 TOCT_prio_3; __be64 TTTLOCT_prio_4; __be64 TTTLOCT_NOFRM_prio_4; __be64 TOCT_prio_4; __be64 TTTLOCT_prio_5; __be64 TTTLOCT_NOFRM_prio_5; __be64 TOCT_prio_5; __be64 TTTLOCT_prio_6; __be64 TTTLOCT_NOFRM_prio_6; __be64 TOCT_prio_6; __be64 TTTLOCT_prio_7; __be64 TTTLOCT_NOFRM_prio_7; __be64 TOCT_prio_7; __be64 TTTLOCT_novlan; __be64 TTTLOCT_NOFRM_novlan; __be64 TOCT_novlan; __be64 TTTLOCT_loopbk; __be64 TTTLOCT_NOFRM_loopbk; __be64 TOCT_loopbk; /* Total frames transmitted with a good CRC that are not aborted */ __be64 TTOT_prio_0; /* Total number of frames transmitted with 802.1Q encapsulation */ __be64 T1Q_prio_0; __be64 reserved13; __be64 TTOT_prio_1; __be64 T1Q_prio_1; __be64 reserved14; __be64 TTOT_prio_2; __be64 T1Q_prio_2; __be64 reserved15; __be64 TTOT_prio_3; __be64 T1Q_prio_3; __be64 reserved16; __be64 TTOT_prio_4; __be64 T1Q_prio_4; __be64 reserved17; __be64 TTOT_prio_5; __be64 T1Q_prio_5; __be64 reserved18; __be64 TTOT_prio_6; __be64 T1Q_prio_6; __be64 reserved19; __be64 TTOT_prio_7; __be64 T1Q_prio_7; __be64 reserved20; __be64 TTOT_novlan; __be64 T1Q_novlan; __be64 reserved21; __be64 TTOT_loopbk; __be64 T1Q_loopbk; __be64 reserved22; /* Received frames with a length greater than MTU octets and a bad CRC */ __be32 RJBBR; /* Received frames with a bad CRC that are not runts, jabbers, or alignment errors */ __be32 RCRC; /* Received frames with SFD with a length of less than 64 octets and a bad CRC */ __be32 RRUNT; /* Received frames with a length less than 64 octets and a good CRC */ __be32 RSHORT; /* Total Number of Received Packets Dropped */ __be32 RDROP; /* Drop due to overflow */ __be32 RdropOvflw; /* Drop due to overflow */ __be32 RdropLength; /* Total of good frames. Does not include frames received with frame-too-long, FCS, or length errors */ __be32 RTOTFRMS; /* Total dropped Xmited packets */ __be32 TDROP; }; -enum mlx4_query_reply mlx4_en_query(void *endev_ptr, void *int_dev); #endif Index: stable/9/sys/ofed/drivers/net/mlx4/en_resources.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/en_resources.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/en_resources.c (revision 273864) @@ -1,103 +1,121 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ +#include #include #include #include "mlx4_en.h" + void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride, int is_tx, int rss, int qpn, int cqn, - struct mlx4_qp_context *context) + int user_prio, struct mlx4_qp_context *context) { struct mlx4_en_dev *mdev = priv->mdev; + struct net_device *dev = priv->dev; memset(context, 0, sizeof *context); - context->flags = cpu_to_be32(7 << 16 | rss << 13); + context->flags = cpu_to_be32(7 << 16 | rss << MLX4_RSS_QPC_FLAG_OFFSET); context->pd = cpu_to_be32(mdev->priv_pdn); context->mtu_msgmax = 0xff; - if (!is_tx && !rss) { + if (!is_tx && !rss) context->rq_size_stride = ilog2(size) << 3 | (ilog2(stride) - 4); - } if (is_tx) context->sq_size_stride = ilog2(size) << 3 | (ilog2(stride) - 4); else context->sq_size_stride = ilog2(TXBB_SIZE) - 4; context->usr_page = cpu_to_be32(mdev->priv_uar.index); context->local_qpn = cpu_to_be32(qpn); context->pri_path.ackto = 1 & 0x07; context->pri_path.sched_queue = 0x83 | (priv->port - 1) << 6; - context->pri_path.counter_index = 0xff; + if (user_prio >= 0) { + context->pri_path.sched_queue |= user_prio << 3; + context->pri_path.feup = 1 << 6; + } + context->pri_path.counter_index = (u8)(priv->counter_index); + if (!rss && + (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK) && + context->pri_path.counter_index != 0xFF) { + /* disable multicast loopback to qp with same counter */ + context->pri_path.fl |= MLX4_FL_ETH_SRC_CHECK_MC_LB; + context->pri_path.vlan_control |= + MLX4_VLAN_CTRL_ETH_SRC_CHECK_IF_COUNTER; + } + context->cqn_send = cpu_to_be32(cqn); context->cqn_recv = cpu_to_be32(cqn); context->db_rec_addr = cpu_to_be64(priv->res.db.dma << 2); + if (!(dev->if_capabilities & IFCAP_VLAN_HWCSUM)) + context->param3 |= cpu_to_be32(1 << 30); } int mlx4_en_map_buffer(struct mlx4_buf *buf) { struct page **pages; int i; + // if nbufs == 1 - there is no need to vmap + // if buf->direct.buf is not NULL it means that vmap was already done by mlx4_alloc_buff if (buf->direct.buf != NULL || buf->nbufs == 1) return 0; pages = kmalloc(sizeof *pages * buf->nbufs, GFP_KERNEL); if (!pages) return -ENOMEM; for (i = 0; i < buf->nbufs; ++i) pages[i] = virt_to_page(buf->page_list[i].buf); buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); kfree(pages); if (!buf->direct.buf) return -ENOMEM; return 0; } void mlx4_en_unmap_buffer(struct mlx4_buf *buf) { - if (buf->direct.buf != NULL || buf->nbufs == 1) + if (BITS_PER_LONG == 64 || buf->nbufs == 1) return; vunmap(buf->direct.buf); - buf->direct.buf = NULL; } void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event) { return; } Index: stable/9/sys/ofed/drivers/net/mlx4/en_rx.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/en_rx.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/en_rx.c (revision 273864) @@ -1,806 +1,901 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ - #include "opt_inet.h" -#include "mlx4_en.h" - #include +#include #include +#include +#include +#include +#include +#ifdef CONFIG_NET_RX_BUSY_POLL +#include +#endif -#include -#include -#include +#include "mlx4_en.h" -enum { - MIN_RX_ARM = 1024, -}; +static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring *ring, + int index) +{ + struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index; + int possible_frags; + int i; + + + /* Set size and memtype fields */ + for (i = 0; i < priv->num_frags; i++) { + rx_desc->data[i].byte_count = + cpu_to_be32(priv->frag_info[i].frag_size); + rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key); + } + + /* If the number of used fragments does not fill up the ring stride, + * * remaining (unused) fragments must be padded with null address/size + * * and a special memory key */ + possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; + for (i = priv->num_frags; i < possible_frags; i++) { + rx_desc->data[i].byte_count = 0; + rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); + rx_desc->data[i].addr = 0; + } + +} + static int mlx4_en_alloc_buf(struct mlx4_en_priv *priv, struct mlx4_en_rx_desc *rx_desc, struct mbuf **mb_list, int i) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_frag_info *frag_info = &priv->frag_info[i]; struct mbuf *mb; dma_addr_t dma; if (i == 0) mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, frag_info->frag_size); else mb = m_getjcl(M_NOWAIT, MT_DATA, 0, frag_info->frag_size); if (mb == NULL) { priv->port_stats.rx_alloc_failed++; return -ENOMEM; } dma = pci_map_single(mdev->pdev, mb->m_data, frag_info->frag_size, PCI_DMA_FROMDEVICE); rx_desc->data[i].addr = cpu_to_be64(dma); mb_list[i] = mb; return 0; } -static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring, int index) -{ - struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index; - int possible_frags; - int i; - /* Set size and memtype fields */ - for (i = 0; i < priv->num_frags; i++) { - rx_desc->data[i].byte_count = - cpu_to_be32(priv->frag_info[i].frag_size); - rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key); - } - - /* If the number of used fragments does not fill up the ring stride, - * remaining (unused) fragments must be padded with null address/size - * and a special memory key */ - possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; - for (i = priv->num_frags; i < possible_frags; i++) { - rx_desc->data[i].byte_count = 0; - rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); - rx_desc->data[i].addr = 0; - } -} - static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring, int index) + struct mlx4_en_rx_ring *ring, int index) { - struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride); - struct mbuf **mb_list = ring->rx_info + (index << priv->log_rx_info); - int i; + struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride); + struct mbuf **mb_list = ring->rx_info + (index << priv->log_rx_info); + int i; - for (i = 0; i < priv->num_frags; i++) - if (mlx4_en_alloc_buf(priv, rx_desc, mb_list, i)) - goto err; + for (i = 0; i < priv->num_frags; i++) + if (mlx4_en_alloc_buf(priv, rx_desc, mb_list, i)) + goto err; - return 0; + return 0; err: - while (i--) - m_free(mb_list[i]); - return -ENOMEM; + while (i--) + m_free(mb_list[i]); + return -ENOMEM; } static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) { *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); } static void mlx4_en_free_rx_desc(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, int index) { struct mlx4_en_frag_info *frag_info; struct mlx4_en_dev *mdev = priv->mdev; struct mbuf **mb_list; struct mlx4_en_rx_desc *rx_desc = ring->buf + (index << ring->log_stride); dma_addr_t dma; int nr; mb_list = ring->rx_info + (index << priv->log_rx_info); for (nr = 0; nr < priv->num_frags; nr++) { en_dbg(DRV, priv, "Freeing fragment:%d\n", nr); frag_info = &priv->frag_info[nr]; dma = be64_to_cpu(rx_desc->data[nr].addr); - en_dbg(DRV, priv, "Unmaping buffer at dma:0x%llx\n", (u64) dma); +#if BITS_PER_LONG == 64 + en_dbg(DRV, priv, "Unmaping buffer at dma:0x%lx\n", (u64) dma); +#elif BITS_PER_LONG == 32 + en_dbg(DRV, priv, "Unmaping buffer at dma:0x%llx\n", (u64) dma); +#endif pci_unmap_single(mdev->pdev, dma, frag_info->frag_size, PCI_DMA_FROMDEVICE); m_free(mb_list[nr]); } } static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv) { struct mlx4_en_rx_ring *ring; int ring_ind; int buf_ind; int new_size; int err; for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) { for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { - ring = &priv->rx_ring[ring_ind]; + ring = priv->rx_ring[ring_ind]; err = mlx4_en_prepare_rx_desc(priv, ring, ring->actual_size); if (err) { if (ring->actual_size == 0) { en_err(priv, "Failed to allocate " "enough rx buffers\n"); return -ENOMEM; } else { - new_size = rounddown_pow_of_two(ring->actual_size); + new_size = + rounddown_pow_of_two(ring->actual_size); en_warn(priv, "Only %d buffers allocated " "reducing ring size to %d\n", ring->actual_size, new_size); goto reduce_rings; } } ring->actual_size++; ring->prod++; } } return 0; reduce_rings: for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { - ring = &priv->rx_ring[ring_ind]; + ring = priv->rx_ring[ring_ind]; while (ring->actual_size > new_size) { ring->actual_size--; ring->prod--; mlx4_en_free_rx_desc(priv, ring, ring->actual_size); } } return 0; } static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring) { int index; en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n", ring->cons, ring->prod); /* Unmap and free Rx buffers */ BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size); while (ring->cons != ring->prod) { index = ring->cons & ring->size_mask; en_dbg(DRV, priv, "Processing descriptor:%d\n", index); mlx4_en_free_rx_desc(priv, ring, index); ++ring->cons; } } +#if MLX4_EN_MAX_RX_FRAGS == 3 +static int frag_sizes[] = { + FRAG_SZ0, + FRAG_SZ1, + FRAG_SZ2, +}; +#elif MLX4_EN_MAX_RX_FRAGS == 2 +static int frag_sizes[] = { + FRAG_SZ0, + FRAG_SZ1, +}; +#else +#error "Unknown MAX_RX_FRAGS" +#endif +void mlx4_en_calc_rx_buf(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + int eff_mtu = dev->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN; + int buf_size = 0; + int i, frag; + + for (i = 0, frag = 0; buf_size < eff_mtu; frag++, i++) { + /* + * Allocate small to large but only as much as is needed for + * the tail. + */ + while (i > 0 && eff_mtu - buf_size <= frag_sizes[i - 1]) + i--; + priv->frag_info[frag].frag_size = frag_sizes[i]; + priv->frag_info[frag].frag_prefix_size = buf_size; + buf_size += priv->frag_info[frag].frag_size; + } + + priv->num_frags = frag; + priv->rx_mb_size = eff_mtu; + priv->log_rx_info = + ROUNDUP_LOG2(priv->num_frags * sizeof(struct mbuf *)); + + en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d " + "num_frags:%d):\n", eff_mtu, priv->num_frags); + for (i = 0; i < priv->num_frags; i++) { + en_dbg(DRV, priv, " frag:%d - size:%d prefix:%d\n", i, + priv->frag_info[i].frag_size, + priv->frag_info[i].frag_prefix_size); + } +} + + int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring, u32 size) + struct mlx4_en_rx_ring **pring, + u32 size, int node) { struct mlx4_en_dev *mdev = priv->mdev; - int err; + struct mlx4_en_rx_ring *ring; + int err = -ENOMEM; int tmp; - + ring = kzalloc(sizeof(struct mlx4_en_rx_ring), GFP_KERNEL); + if (!ring) { + en_err(priv, "Failed to allocate RX ring structure\n"); + return -ENOMEM; + } + ring->prod = 0; ring->cons = 0; ring->size = size; ring->size_mask = size - 1; ring->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + - DS_SIZE * MLX4_EN_MAX_RX_FRAGS); + DS_SIZE * MLX4_EN_MAX_RX_FRAGS); ring->log_stride = ffs(ring->stride) - 1; ring->buf_size = ring->size * ring->stride + TXBB_SIZE; tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS * - sizeof(struct mbuf *)); + sizeof(struct mbuf *)); - ring->rx_info = kmalloc(tmp, GFP_KERNEL); - if (!ring->rx_info) { - en_err(priv, "Failed allocating rx_info ring\n"); - return -ENOMEM; - } - en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d stride:%d (%d)\n", - ring->rx_info, tmp, ring->stride, ring->log_stride); + ring->rx_info = kmalloc(tmp, GFP_KERNEL); + if (!ring->rx_info) { + err = -ENOMEM; + goto err_ring; + } + en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n", + ring->rx_info, tmp); + err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size, 2 * PAGE_SIZE); if (err) - goto err_ring; + goto err_info; err = mlx4_en_map_buffer(&ring->wqres.buf); if (err) { en_err(priv, "Failed to map RX buffer\n"); goto err_hwq; } ring->buf = ring->wqres.buf.direct.buf; - + *pring = ring; return 0; - mlx4_en_unmap_buffer(&ring->wqres.buf); err_hwq: mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); +err_info: + vfree(ring->rx_info); err_ring: - kfree(ring->rx_info); - ring->rx_info = NULL; + kfree(ring); + return err; } + int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) { struct mlx4_en_rx_ring *ring; int i; int ring_ind; int err; int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + - DS_SIZE * priv->num_frags); + DS_SIZE * priv->num_frags); for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { - ring = &priv->rx_ring[ring_ind]; + ring = priv->rx_ring[ring_ind]; ring->prod = 0; ring->cons = 0; ring->actual_size = 0; - ring->cqn = priv->rx_cq[ring_ind].mcq.cqn; + ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn; + ring->rx_alloc_order = priv->rx_alloc_order; + ring->rx_alloc_size = priv->rx_alloc_size; + ring->rx_buf_size = priv->rx_buf_size; + ring->rx_mb_size = priv->rx_mb_size; + ring->stride = stride; if (ring->stride <= TXBB_SIZE) ring->buf += TXBB_SIZE; ring->log_stride = ffs(ring->stride) - 1; ring->buf_size = ring->size * ring->stride; memset(ring->buf, 0, ring->buf_size); mlx4_en_update_rx_prod_db(ring); - /* Initailize all descriptors */ + /* Initialize all descriptors */ for (i = 0; i < ring->size; i++) mlx4_en_init_rx_desc(priv, ring, i); + #ifdef INET /* Configure lro mngr */ if (priv->dev->if_capenable & IFCAP_LRO) { if (tcp_lro_init(&ring->lro)) priv->dev->if_capenable &= ~IFCAP_LRO; else ring->lro.ifp = priv->dev; } #endif } + + err = mlx4_en_fill_rx_buffers(priv); if (err) goto err_buffers; for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { - ring = &priv->rx_ring[ring_ind]; + ring = priv->rx_ring[ring_ind]; ring->size_mask = ring->actual_size - 1; mlx4_en_update_rx_prod_db(ring); } - return 0; err_buffers: for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) - mlx4_en_free_rx_buf(priv, &priv->rx_ring[ring_ind]); + mlx4_en_free_rx_buf(priv, priv->rx_ring[ring_ind]); + ring_ind = priv->rx_ring_num - 1; + + while (ring_ind >= 0) { + ring = priv->rx_ring[ring_ind]; + if (ring->stride <= TXBB_SIZE) + ring->buf -= TXBB_SIZE; + ring_ind--; + } + return err; } + void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring) + struct mlx4_en_rx_ring **pring, + u32 size, u16 stride) { struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_rx_ring *ring = *pring; mlx4_en_unmap_buffer(&ring->wqres.buf); - mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size + TXBB_SIZE); - kfree(ring->rx_info); - ring->rx_info = NULL; + mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); + vfree(ring->rx_info); + kfree(ring); + *pring = NULL; +#ifdef CONFIG_RFS_ACCEL + mlx4_en_cleanup_filters(priv, ring); +#endif } + void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring) { #ifdef INET tcp_lro_free(&ring->lro); #endif mlx4_en_free_rx_buf(priv, ring); if (ring->stride <= TXBB_SIZE) ring->buf -= TXBB_SIZE; } +static void validate_loopback(struct mlx4_en_priv *priv, struct mbuf *mb) +{ + int i; + int offset = ETHER_HDR_LEN; + + for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) { + if (*(mb->m_data + offset) != (unsigned char) (i & 0xff)) + goto out_loopback; + } + /* Loopback found */ + priv->loopback_ok = 1; + +out_loopback: + m_freem(mb); +} + + +static inline int invalid_cqe(struct mlx4_en_priv *priv, + struct mlx4_cqe *cqe) +{ + /* Drop packet on bad receive or bad checksum */ + if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == + MLX4_CQE_OPCODE_ERROR)) { + en_err(priv, "CQE completed in error - vendor syndrom:%d syndrom:%d\n", + ((struct mlx4_err_cqe *)cqe)->vendor_err_syndrome, + ((struct mlx4_err_cqe *)cqe)->syndrome); + return 1; + } + if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) { + en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n"); + return 1; + } + + return 0; +} + + /* Unmap a completed descriptor and free unused pages */ static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv, struct mlx4_en_rx_desc *rx_desc, struct mbuf **mb_list, int length) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_frag_info *frag_info; dma_addr_t dma; struct mbuf *mb; int nr; mb = mb_list[0]; mb->m_pkthdr.len = length; /* Collect used fragments while replacing them in the HW descirptors */ for (nr = 0; nr < priv->num_frags; nr++) { frag_info = &priv->frag_info[nr]; if (length <= frag_info->frag_prefix_size) break; - if (nr) + if (nr) mb->m_next = mb_list[nr]; mb = mb_list[nr]; mb->m_len = frag_info[nr].frag_size; dma = be64_to_cpu(rx_desc->data[nr].addr); - /* Allocate a replacement page */ - if (mlx4_en_alloc_buf(priv, rx_desc, mb_list, nr)) - goto fail; + /* Allocate a replacement page */ + if (mlx4_en_alloc_buf(priv, rx_desc, mb_list, nr)) + goto fail; /* Unmap buffer */ pci_unmap_single(mdev->pdev, dma, frag_info[nr].frag_size, PCI_DMA_FROMDEVICE); } /* Adjust size of last fragment to match actual length */ mb->m_len = length - priv->frag_info[nr - 1].frag_prefix_size; mb->m_next = NULL; return 0; fail: - /* Drop all accumulated fragments (which have already been replaced in - * the descriptor) of this packet; remaining fragments are reused... */ - while (nr > 0) { - nr--; - m_free(mb_list[nr]); - } - return -ENOMEM; -} + /* Drop all accumulated fragments (which have already been replaced in + * the descriptor) of this packet; remaining fragments are reused... */ + while (nr > 0) { + nr--; + m_free(mb_list[nr]); + } + return -ENOMEM; - -static inline int invalid_cqe(struct mlx4_en_priv *priv, - struct mlx4_cqe *cqe) -{ - /* Drop packet on bad receive or bad checksum */ - if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == - MLX4_CQE_OPCODE_ERROR)) { - en_err(priv, "CQE completed in error - vendor " - "syndrom:%d syndrom:%d\n", - ((struct mlx4_err_cqe *) cqe)->vendor_err_syndrome, - ((struct mlx4_err_cqe *) cqe)->syndrome); - return 1; - } - if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) { - en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n"); - return 1; - } - - return 0; } -static void validate_loopback(struct mlx4_en_priv *priv, struct mbuf *mb) -{ - int i; - int offset = ETHER_HDR_LEN; - - for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) { - if (*(mb->m_data + offset) != (unsigned char) (i & 0xff)) - goto out_loopback; - } - /* Loopback found */ - priv->loopback_ok = 1; - -out_loopback: - m_freem(mb); -} - static struct mbuf *mlx4_en_rx_mb(struct mlx4_en_priv *priv, struct mlx4_en_rx_desc *rx_desc, struct mbuf **mb_list, unsigned int length) { struct mbuf *mb; mb = mb_list[0]; /* Move relevant fragments to mb */ if (unlikely(mlx4_en_complete_rx_desc(priv, rx_desc, mb_list, length))) return NULL; return mb; } - +/* For cpu arch with cache line of 64B the performance is better when cqe size==64B + * To enlarge cqe size from 32B to 64B --> 32B of garbage (i.e. 0xccccccc) + * was added in the beginning of each cqe (the real data is in the corresponding 32B). + * The following calc ensures that when factor==1, it means we are alligned to 64B + * and we get the real cqe data*/ +#define CQE_FACTOR_INDEX(index, factor) ((index << factor) + factor) int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_cqe *cqe; - struct mlx4_en_rx_ring *ring = &priv->rx_ring[cq->ring]; + struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring]; struct mbuf **mb_list; struct mlx4_en_rx_desc *rx_desc; struct mbuf *mb; + struct mlx4_cq *mcq = &cq->mcq; + struct mlx4_cqe *buf = cq->buf; #ifdef INET struct lro_entry *queued; #endif int index; unsigned int length; int polled = 0; + u32 cons_index = mcq->cons_index; + u32 size_mask = ring->size_mask; + int size = cq->size; + int factor = priv->cqe_factor; if (!priv->port_up) return 0; /* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx - * descriptor offset can be deduced from the CQE index instead of + * descriptor offset can be deducted from the CQE index instead of * reading 'cqe->index' */ - index = cq->mcq.cons_index & ring->size_mask; - cqe = &cq->buf[index]; + index = cons_index & size_mask; + cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; /* Process all completed CQEs */ while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, - cq->mcq.cons_index & cq->size)) { - + cons_index & size)) { mb_list = ring->rx_info + (index << priv->log_rx_info); rx_desc = ring->buf + (index << ring->log_stride); /* * make sure we read the CQE after we read the ownership bit */ rmb(); - if (invalid_cqe(priv, cqe)) + if (invalid_cqe(priv, cqe)) { goto next; - + } /* * Packet is OK - process it. */ length = be32_to_cpu(cqe->byte_cnt); + length -= ring->fcs_del; mb = mlx4_en_rx_mb(priv, rx_desc, mb_list, length); if (!mb) { ring->errors++; goto next; } ring->bytes += length; ring->packets++; - if (unlikely(priv->validate_loopback)) { + if (unlikely(priv->validate_loopback)) { validate_loopback(priv, mb); goto next; } mb->m_pkthdr.flowid = cq->ring; mb->m_flags |= M_FLOWID; mb->m_pkthdr.rcvif = dev; if (be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_VLAN_PRESENT_MASK) { mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->sl_vid); mb->m_flags |= M_VLANTAG; } - if (likely(priv->rx_csum) && + if (likely(dev->if_capabilities & IFCAP_RXCSUM) && (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) && (cqe->checksum == cpu_to_be16(0xffff))) { priv->port_stats.rx_chksum_good++; - mb->m_pkthdr.csum_flags = + mb->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR; mb->m_pkthdr.csum_data = htons(0xffff); /* This packet is eligible for LRO if it is: * - DIX Ethernet (type interpretation) * - TCP/IP (v4) * - without IP options * - not an IP fragment */ #ifdef INET if (mlx4_en_can_lro(cqe->status) && - (dev->if_capenable & IFCAP_LRO)) { + (dev->if_capenable & IFCAP_LRO)) { if (ring->lro.lro_cnt != 0 && - tcp_lro_rx(&ring->lro, mb, 0) == 0) + tcp_lro_rx(&ring->lro, mb, 0) == 0) goto next; } -#endif +#endif /* LRO not possible, complete processing here */ INC_PERF_COUNTER(priv->pstats.lro_misses); } else { mb->m_pkthdr.csum_flags = 0; priv->port_stats.rx_chksum_none++; -#ifdef INET - if (priv->ip_reasm && - cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV4) && - !mlx4_en_rx_frags(priv, ring, mb, cqe)) - goto next; -#endif } /* Push it up the stack */ dev->if_input(dev, mb); next: - ++cq->mcq.cons_index; - index = (cq->mcq.cons_index) & ring->size_mask; - cqe = &cq->buf[index]; + ++cons_index; + index = cons_index & size_mask; + cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; if (++polled == budget) goto out; } /* Flush all pending IP reassembly sessions */ out: #ifdef INET - mlx4_en_flush_frags(priv, ring); while ((queued = SLIST_FIRST(&ring->lro.lro_active)) != NULL) { SLIST_REMOVE_HEAD(&ring->lro.lro_active, next); tcp_lro_flush(&ring->lro, queued); } #endif AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled); - mlx4_cq_set_ci(&cq->mcq); + mcq->cons_index = cons_index; + mlx4_cq_set_ci(mcq); wmb(); /* ensure HW sees CQ consumer before we post new buffers */ - ring->cons = cq->mcq.cons_index; + ring->cons = mcq->cons_index; ring->prod += polled; /* Polled descriptors were realocated in place */ mlx4_en_update_rx_prod_db(ring); return polled; + } - /* Rx CQ polling - called by NAPI */ static int mlx4_en_poll_rx_cq(struct mlx4_en_cq *cq, int budget) { - struct net_device *dev = cq->dev; - int done; + struct net_device *dev = cq->dev; + int done; - done = mlx4_en_process_rx_cq(dev, cq, budget); - cq->tot_rx += done; + done = mlx4_en_process_rx_cq(dev, cq, budget); + cq->tot_rx += done; - return done; -} + return done; -void mlx4_en_rx_que(void *context, int pending) -{ - struct mlx4_en_cq *cq; - - cq = context; - while (mlx4_en_poll_rx_cq(cq, MLX4_EN_MAX_RX_POLL) - == MLX4_EN_MAX_RX_POLL); - mlx4_en_arm_cq(cq->dev->if_softc, cq); } - void mlx4_en_rx_irq(struct mlx4_cq *mcq) { struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); struct mlx4_en_priv *priv = netdev_priv(cq->dev); - int done; + int done; - done = mlx4_en_poll_rx_cq(cq, MLX4_EN_MAX_RX_POLL); - if (done == MLX4_EN_MAX_RX_POLL) + // Shoot one within the irq context + // Because there is no NAPI in freeBSD + done = mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET); + if (priv->port_up && (done == MLX4_EN_RX_BUDGET) ) { taskqueue_enqueue(cq->tq, &cq->cq_task); - else + } + else { mlx4_en_arm_cq(priv, cq); + } } - -#if MLX4_EN_MAX_RX_FRAGS == 3 -static int frag_sizes[] = { - FRAG_SZ0, - FRAG_SZ1, - FRAG_SZ2, -}; -#elif MLX4_EN_MAX_RX_FRAGS == 2 -static int frag_sizes[] = { - FRAG_SZ0, - FRAG_SZ1, -}; -#else -#error "Unknown MAX_RX_FRAGS" -#endif - -void mlx4_en_calc_rx_buf(struct net_device *dev) +void mlx4_en_rx_que(void *context, int pending) { - struct mlx4_en_priv *priv = netdev_priv(dev); - int eff_mtu = dev->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETH_LLC_SNAP_SIZE; - int buf_size = 0; - int i, frag; + struct mlx4_en_cq *cq; - for (i = 0, frag = 0; buf_size < eff_mtu; frag++, i++) { - /* - * Allocate small to large but only as much as is needed for - * the tail. - */ - while (i > 0 && eff_mtu - buf_size <= frag_sizes[i - 1]) - i--; - priv->frag_info[frag].frag_size = frag_sizes[i]; - priv->frag_info[frag].frag_prefix_size = buf_size; - buf_size += priv->frag_info[frag].frag_size; - } - - priv->num_frags = frag; - priv->rx_mb_size = eff_mtu; - priv->log_rx_info = - ROUNDUP_LOG2(priv->num_frags * sizeof(struct mbuf *)); - - en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d " - "num_frags:%d):\n", eff_mtu, priv->num_frags); - for (i = 0; i < priv->num_frags; i++) { - en_dbg(DRV, priv, " frag:%d - size:%d prefix:%d\n", i, - priv->frag_info[i].frag_size, - priv->frag_info[i].frag_prefix_size) - } + cq = context; + while (mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET) + == MLX4_EN_RX_BUDGET); + mlx4_en_arm_cq(cq->dev->if_softc, cq); } + /* RSS related functions */ static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn, struct mlx4_en_rx_ring *ring, enum mlx4_qp_state *state, struct mlx4_qp *qp) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_qp_context *context; int err = 0; context = kmalloc(sizeof *context , GFP_KERNEL); if (!context) { en_err(priv, "Failed to allocate qp context\n"); return -ENOMEM; } err = mlx4_qp_alloc(mdev->dev, qpn, qp); if (err) { en_err(priv, "Failed to allocate qp #%x\n", qpn); goto out; } qp->event = mlx4_en_sqp_event; memset(context, 0, sizeof *context); mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0, - qpn, ring->cqn, context); + qpn, ring->cqn, -1, context); context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma); + /* Cancel FCS removal if FW allows */ + if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) { + context->param3 |= cpu_to_be32(1 << 29); + ring->fcs_del = ETH_FCS_LEN; + } else + ring->fcs_del = 0; + err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state); if (err) { mlx4_qp_remove(mdev->dev, qp); mlx4_qp_free(mdev->dev, qp); } mlx4_en_update_rx_prod_db(ring); out: kfree(context); return err; } +int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv) +{ + int err; + u32 qpn; + + err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn, 0); + if (err) { + en_err(priv, "Failed reserving drop qpn\n"); + return err; + } + err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp); + if (err) { + en_err(priv, "Failed allocating drop qp\n"); + mlx4_qp_release_range(priv->mdev->dev, qpn, 1); + return err; + } + + return 0; +} + +void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv) +{ + u32 qpn; + + qpn = priv->drop_qp.qpn; + mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp); + mlx4_qp_free(priv->mdev->dev, &priv->drop_qp); + mlx4_qp_release_range(priv->mdev->dev, qpn, 1); +} + /* Allocate rx qp's and configure them according to rss map */ int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_rss_map *rss_map = &priv->rss_map; struct mlx4_qp_context context; - struct mlx4_en_rss_context *rss_context; + struct mlx4_rss_context *rss_context; + int rss_rings; void *ptr; - u8 rss_mask; - int i, qpn; + u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 | + MLX4_RSS_TCP_IPV6); + int i; int err = 0; int good_qps = 0; + static const u32 rsskey[10] = { 0xD181C62C, 0xF7F4DB5B, 0x1983A2FC, + 0x943E1ADB, 0xD9389E6B, 0xD1039C2C, 0xA74499AD, + 0x593D56D9, 0xF3253C06, 0x2ADC1FFC}; - if (mdev->profile.udp_rss) - rss_mask = 0x3f; - else - rss_mask = 0x14; en_dbg(DRV, priv, "Configuring rss steering\n"); err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num, - roundup_pow_of_two(priv->rx_ring_num), - &rss_map->base_qpn); + priv->rx_ring_num, + &rss_map->base_qpn, 0); if (err) { en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num); return err; } for (i = 0; i < priv->rx_ring_num; i++) { - qpn = rss_map->base_qpn + i; - err = mlx4_en_config_rss_qp(priv, qpn, - &priv->rx_ring[i], + priv->rx_ring[i]->qpn = rss_map->base_qpn + i; + err = mlx4_en_config_rss_qp(priv, priv->rx_ring[i]->qpn, + priv->rx_ring[i], &rss_map->state[i], &rss_map->qps[i]); if (err) goto rss_err; ++good_qps; } /* Configure RSS indirection qp */ - err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &priv->base_qpn); - if (err) { - en_err(priv, "Failed to reserve range for RSS " - "indirection qp\n"); - goto rss_err; - } err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp); if (err) { en_err(priv, "Failed to allocate RSS indirection QP\n"); - goto reserve_err; + goto rss_err; } rss_map->indir_qp.event = mlx4_en_sqp_event; mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn, - priv->rx_ring[0].cqn, &context); + priv->rx_ring[0]->cqn, -1, &context); - ptr = ((void *) &context) + 0x3c; - rss_context = (struct mlx4_en_rss_context *) ptr; - rss_context->base_qpn = cpu_to_be32(ilog2(priv->rx_ring_num) << 24 | + if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num) + rss_rings = priv->rx_ring_num; + else + rss_rings = priv->prof->rss_rings; + + ptr = ((void *) &context) + offsetof(struct mlx4_qp_context, pri_path) + + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH; + rss_context = ptr; + rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 | (rss_map->base_qpn)); rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn); + if (priv->mdev->profile.udp_rss) { + rss_mask |= MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6; + rss_context->base_qpn_udp = rss_context->default_qpn; + } rss_context->flags = rss_mask; - rss_context->base_qpn_udp = rss_context->default_qpn; + rss_context->hash_fn = MLX4_RSS_HASH_TOP; + for (i = 0; i < 10; i++) + rss_context->rss_key[i] = cpu_to_be32(rsskey[i]); err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context, &rss_map->indir_qp, &rss_map->indir_state); if (err) goto indir_err; return 0; indir_err: mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); mlx4_qp_free(mdev->dev, &rss_map->indir_qp); -reserve_err: - mlx4_qp_release_range(mdev->dev, priv->base_qpn, 1); rss_err: for (i = 0; i < good_qps; i++) { mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); mlx4_qp_free(mdev->dev, &rss_map->qps[i]); } mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); return err; } void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_rss_map *rss_map = &priv->rss_map; int i; mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); mlx4_qp_free(mdev->dev, &rss_map->indir_qp); - mlx4_qp_release_range(mdev->dev, priv->base_qpn, 1); for (i = 0; i < priv->rx_ring_num; i++) { mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); mlx4_qp_free(mdev->dev, &rss_map->qps[i]); } mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); } + Index: stable/9/sys/ofed/drivers/net/mlx4/en_selftest.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/en_selftest.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/en_selftest.c (revision 273864) @@ -1,179 +1,179 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ -#include "mlx4_en.h" - #include #include #include #include #include +#include "mlx4_en.h" + static int mlx4_en_test_registers(struct mlx4_en_priv *priv) { return mlx4_cmd(priv->mdev->dev, 0, 0, 0, MLX4_CMD_HW_HEALTH_CHECK, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); } static int mlx4_en_test_loopback_xmit(struct mlx4_en_priv *priv) { - struct mbuf *mb; + struct sk_buff *skb; struct ethhdr *ethh; unsigned char *packet; unsigned int packet_size = MLX4_LOOPBACK_TEST_PAYLOAD; unsigned int i; int err; /* build the pkt before xmit */ - mb = netdev_alloc_mb(priv->dev, MLX4_LOOPBACK_TEST_PAYLOAD + ETH_HLEN + NET_IP_ALIGN); - if (!mb) { - en_err(priv, "-LOOPBACK_TEST_XMIT- failed to create mb for xmit\n"); + skb = netdev_alloc_skb(priv->dev, MLX4_LOOPBACK_TEST_PAYLOAD + ETH_HLEN + NET_IP_ALIGN); + if (!skb) { + en_err(priv, "-LOOPBACK_TEST_XMIT- failed to create skb for xmit\n"); return -ENOMEM; } - mb_reserve(mb, NET_IP_ALIGN); + skb_reserve(skb, NET_IP_ALIGN); - ethh = (struct ethhdr *)mb_put(mb, sizeof(struct ethhdr)); - packet = (unsigned char *)mb_put(mb, packet_size); + ethh = (struct ethhdr *)skb_put(skb, sizeof(struct ethhdr)); + packet = (unsigned char *)skb_put(skb, packet_size); memcpy(ethh->h_dest, priv->dev->dev_addr, ETH_ALEN); memset(ethh->h_source, 0, ETH_ALEN); ethh->h_proto = htons(ETH_P_ARP); - mb_set_mac_header(mb, 0); + skb_set_mac_header(skb, 0); for (i = 0; i < packet_size; ++i) /* fill our packet */ packet[i] = (unsigned char)(i & 0xff); /* xmit the pkt */ - err = mlx4_en_xmit(mb, priv->dev); + err = mlx4_en_xmit(skb, priv->dev); return err; } static int mlx4_en_test_loopback(struct mlx4_en_priv *priv) { u32 loopback_ok = 0; int i; priv->loopback_ok = 0; priv->validate_loopback = 1; + mlx4_en_update_loopback_state(priv->dev, priv->dev->features); + /* xmit */ if (mlx4_en_test_loopback_xmit(priv)) { en_err(priv, "Transmitting loopback packet failed\n"); goto mlx4_en_test_loopback_exit; } /* polling for result */ for (i = 0; i < MLX4_EN_LOOPBACK_RETRIES; ++i) { msleep(MLX4_EN_LOOPBACK_TIMEOUT); if (priv->loopback_ok) { loopback_ok = 1; break; } } if (!loopback_ok) en_err(priv, "Loopback packet didn't arrive\n"); mlx4_en_test_loopback_exit: priv->validate_loopback = 0; - return (!loopback_ok); + mlx4_en_update_loopback_state(priv->dev, priv->dev->features); + return !loopback_ok; } static int mlx4_en_test_link(struct mlx4_en_priv *priv) { if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) return -ENOMEM; if (priv->port_state.link_state == 1) return 0; else return 1; } static int mlx4_en_test_speed(struct mlx4_en_priv *priv) { if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) return -ENOMEM; - /* The device currently only supports 10G speed */ - if (priv->port_state.link_speed != SPEED_10000) + /* The device supports 1G, 10G and 40G speed */ + if (priv->port_state.link_speed != MLX4_EN_LINK_SPEED_1G && + priv->port_state.link_speed != MLX4_EN_LINK_SPEED_10G && + priv->port_state.link_speed != MLX4_EN_LINK_SPEED_40G) return priv->port_state.link_speed; return 0; } void mlx4_en_ex_selftest(struct net_device *dev, u32 *flags, u64 *buf) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_tx_ring *tx_ring; int i, carrier_ok; memset(buf, 0, sizeof(u64) * MLX4_EN_NUM_SELF_TEST); if (*flags & ETH_TEST_FL_OFFLINE) { /* disable the interface */ carrier_ok = netif_carrier_ok(dev); netif_carrier_off(dev); -retry_tx: - /* Wait untill all tx queues are empty. + /* Wait until all tx queues are empty. * there should not be any additional incoming traffic * since we turned the carrier off */ msleep(200); - for (i = 0; i < priv->tx_ring_num && carrier_ok; i++) { - tx_ring = &priv->tx_ring[i]; - if (tx_ring->prod != (tx_ring->cons + tx_ring->last_nr_txbb)) - goto retry_tx; - } - if (priv->mdev->dev->caps.loopback_support){ + if (priv->mdev->dev->caps.flags & + MLX4_DEV_CAP_FLAG_UC_LOOPBACK) { buf[3] = mlx4_en_test_registers(priv); - buf[4] = mlx4_en_test_loopback(priv); + if (priv->port_up) + buf[4] = mlx4_en_test_loopback(priv); } if (carrier_ok) netif_carrier_on(dev); } buf[0] = mlx4_test_interrupts(mdev->dev); buf[1] = mlx4_en_test_link(priv); buf[2] = mlx4_en_test_speed(priv); for (i = 0; i < MLX4_EN_NUM_SELF_TEST; i++) { if (buf[i]) *flags |= ETH_TEST_FL_FAILED; } } Index: stable/9/sys/ofed/drivers/net/mlx4/en_tx.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/en_tx.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/en_tx.c (revision 273864) @@ -1,1039 +1,1115 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ -#include "mlx4_en.h" - +#include #include +#include #include +#include #include +#include -#include -#include -#include - #include #include #include #include #include #include #include #include +#include "mlx4_en.h" +#include "utils.h" + enum { MAX_INLINE = 104, /* 128 - 16 - 4 - 4 */ MAX_BF = 256, + MIN_PKT_LEN = 17, }; -static int inline_thold = MAX_INLINE; +static int inline_thold __read_mostly = MAX_INLINE; -module_param_named(inline_thold, inline_thold, int, 0444); -MODULE_PARM_DESC(inline_thold, "treshold for using inline data"); +module_param_named(inline_thold, inline_thold, uint, 0444); +MODULE_PARM_DESC(inline_thold, "threshold for using inline data"); int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring, u32 size, - u16 stride) + struct mlx4_en_tx_ring **pring, u32 size, + u16 stride, int node, int queue_idx) { struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_tx_ring *ring; int tmp; int err; + ring = kzalloc_node(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL, node); + if (!ring) { + ring = kzalloc(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL); + if (!ring) { + en_err(priv, "Failed allocating TX ring\n"); + return -ENOMEM; + } + } + ring->size = size; ring->size_mask = size - 1; ring->stride = stride; - - inline_thold = min(inline_thold, MAX_INLINE); - + ring->full_size = ring->size - HEADROOM - MAX_DESC_TXBBS; + ring->inline_thold = min(inline_thold, MAX_INLINE); mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF); mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF); /* Allocate the buf ring */ ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF, - M_WAITOK, &ring->tx_lock.m); + M_WAITOK, &ring->tx_lock.m); if (ring->br == NULL) { en_err(priv, "Failed allocating tx_info ring\n"); return -ENOMEM; } tmp = size * sizeof(struct mlx4_en_tx_info); - ring->tx_info = kmalloc(tmp, GFP_KERNEL); + ring->tx_info = vmalloc_node(tmp, node); if (!ring->tx_info) { - en_err(priv, "Failed allocating tx_info ring\n"); - err = -ENOMEM; - goto err_tx; + ring->tx_info = vmalloc(tmp); + if (!ring->tx_info) { + err = -ENOMEM; + goto err_ring; + } } + en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n", ring->tx_info, tmp); - ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL); + ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, GFP_KERNEL, node); if (!ring->bounce_buf) { - en_err(priv, "Failed allocating bounce buffer\n"); - err = -ENOMEM; - goto err_tx; + ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL); + if (!ring->bounce_buf) { + err = -ENOMEM; + goto err_info; + } } ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE); + /* Allocate HW buffers on provided NUMA node */ err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size, 2 * PAGE_SIZE); if (err) { en_err(priv, "Failed allocating hwq resources\n"); goto err_bounce; } err = mlx4_en_map_buffer(&ring->wqres.buf); if (err) { en_err(priv, "Failed to map TX buffer\n"); goto err_hwq_res; } ring->buf = ring->wqres.buf.direct.buf; en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d " "buf_size:%d dma:%llx\n", ring, ring->buf, ring->size, ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map); - err = mlx4_qp_reserve_range(mdev->dev, 1, 256, &ring->qpn); + err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn, + MLX4_RESERVE_BF_QP); if (err) { - en_err(priv, "Failed reserving qp for tx ring.\n"); + en_err(priv, "failed reserving qp for TX ring\n"); goto err_map; } err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp); if (err) { en_err(priv, "Failed allocating qp %d\n", ring->qpn); goto err_reserve; } ring->qp.event = mlx4_en_sqp_event; - err = mlx4_bf_alloc(mdev->dev, &ring->bf); + err = mlx4_bf_alloc(mdev->dev, &ring->bf, node); if (err) { + en_dbg(DRV, priv, "working without blueflame (%d)", err); ring->bf.uar = &mdev->priv_uar; ring->bf.uar->map = mdev->uar_map; ring->bf_enabled = false; } else ring->bf_enabled = true; + ring->queue_index = queue_idx; + if (queue_idx < priv->num_tx_rings_p_up ) + CPU_SET(queue_idx, &ring->affinity_mask); + *pring = ring; return 0; err_reserve: mlx4_qp_release_range(mdev->dev, ring->qpn, 1); err_map: mlx4_en_unmap_buffer(&ring->wqres.buf); err_hwq_res: mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); err_bounce: kfree(ring->bounce_buf); - ring->bounce_buf = NULL; -err_tx: +err_info: + vfree(ring->tx_info); +err_ring: buf_ring_free(ring->br, M_DEVBUF); - kfree(ring->tx_info); - ring->tx_info = NULL; + kfree(ring); return err; } void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring) + struct mlx4_en_tx_ring **pring) { struct mlx4_en_dev *mdev = priv->mdev; + struct mlx4_en_tx_ring *ring = *pring; en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn); buf_ring_free(ring->br, M_DEVBUF); if (ring->bf_enabled) mlx4_bf_free(mdev->dev, &ring->bf); mlx4_qp_remove(mdev->dev, &ring->qp); mlx4_qp_free(mdev->dev, &ring->qp); - mlx4_qp_release_range(mdev->dev, ring->qpn, 1); + mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1); mlx4_en_unmap_buffer(&ring->wqres.buf); mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); kfree(ring->bounce_buf); - ring->bounce_buf = NULL; - kfree(ring->tx_info); - ring->tx_info = NULL; + vfree(ring->tx_info); mtx_destroy(&ring->tx_lock.m); mtx_destroy(&ring->comp_lock.m); + kfree(ring); + *pring = NULL; } int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring, - int cq) + int cq, int user_prio) { struct mlx4_en_dev *mdev = priv->mdev; int err; ring->cqn = cq; ring->prod = 0; ring->cons = 0xffffffff; ring->last_nr_txbb = 1; ring->poll_cnt = 0; ring->blocked = 0; memset(ring->tx_info, 0, ring->size * sizeof(struct mlx4_en_tx_info)); memset(ring->buf, 0, ring->buf_size); ring->qp_state = MLX4_QP_STATE_RST; - ring->doorbell_qpn = swab32(ring->qp.qpn << 8); + ring->doorbell_qpn = ring->qp.qpn << 8; mlx4_en_fill_qp_context(priv, ring->size, ring->stride, 1, 0, ring->qpn, - ring->cqn, &ring->context); + ring->cqn, user_prio, &ring->context); if (ring->bf_enabled) ring->context.usr_page = cpu_to_be32(ring->bf.uar->index); err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, &ring->context, &ring->qp, &ring->qp_state); - return err; } void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring) { struct mlx4_en_dev *mdev = priv->mdev; mlx4_qp_modify(mdev->dev, NULL, ring->qp_state, MLX4_QP_STATE_RST, NULL, 0, 0, &ring->qp); } +static void mlx4_en_stamp_wqe(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring *ring, + int index, u8 owner) +{ + struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; + struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE; + void *end = ring->buf + ring->buf_size; + __be32 *ptr = (__be32 *)tx_desc; + __be32 stamp = cpu_to_be32(STAMP_VAL | (!!owner << STAMP_SHIFT)); + int i; + /* Optimize the common case when there are no wraparounds */ + if (likely((void *)tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) + /* Stamp the freed descriptor */ + for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { + *ptr = stamp; + ptr += STAMP_DWORDS; + } + else + /* Stamp the freed descriptor */ + for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { + *ptr = stamp; + ptr += STAMP_DWORDS; + if ((void *)ptr >= end) { + ptr = ring->buf; + stamp ^= cpu_to_be32(0x80000000); + } + } +} + static u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring, - int index, u8 owner) + int index, u8 owner, u64 timestamp) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE; struct mlx4_wqe_data_seg *data = (void *) tx_desc + tx_info->data_offset; - struct mbuf *mb = tx_info->mb; + struct mbuf *mb = tx_info->mb; void *end = ring->buf + ring->buf_size; - int frags = tx_info->nr_segs; + int frags = tx_info->nr_segs;; int i; - __be32 *ptr = (__be32 *)tx_desc; - __be32 stamp = cpu_to_be32(STAMP_VAL | (!!owner << STAMP_SHIFT)); /* Optimize the common case when there are no wraparounds */ if (likely((void *) tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) { if (!tx_info->inl) { + if (tx_info->linear) { + dma_unmap_single(priv->ddev, + (dma_addr_t) be64_to_cpu(data->addr), + be32_to_cpu(data->byte_count), + PCI_DMA_TODEVICE); + ++data; + } + for (i = 0; i < frags; i++) { - pci_unmap_single(mdev->pdev, - (dma_addr_t) be64_to_cpu(data[i].addr), - data[i].byte_count, PCI_DMA_TODEVICE); + pci_unmap_single(mdev->pdev, + (dma_addr_t) be64_to_cpu(data[i].addr), + data[i].byte_count, PCI_DMA_TODEVICE); } } - /* Stamp the freed descriptor */ - for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { - *ptr = stamp; - ptr += STAMP_DWORDS; - } - } else { if (!tx_info->inl) { + if ((void *) data >= end) { + data = ring->buf + ((void *)data - end); + } + + if (tx_info->linear) { + dma_unmap_single(priv->ddev, + (dma_addr_t) be64_to_cpu(data->addr), + be32_to_cpu(data->byte_count), + PCI_DMA_TODEVICE); + ++data; + } + for (i = 0; i < frags; i++) { /* Check for wraparound before unmapping */ if ((void *) data >= end) - data = (struct mlx4_wqe_data_seg *) ring->buf; - pci_unmap_single(mdev->pdev, - (dma_addr_t) be64_to_cpu(data->addr), - data->byte_count, PCI_DMA_TODEVICE); + data = ring->buf; + pci_unmap_single(mdev->pdev, + (dma_addr_t) be64_to_cpu(data->addr), + data->byte_count, PCI_DMA_TODEVICE); ++data; } } - /* Stamp the freed descriptor */ - for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { - *ptr = stamp; - ptr += STAMP_DWORDS; - if ((void *) ptr >= end) { - ptr = ring->buf; - stamp ^= cpu_to_be32(0x80000000); - } - } - } - m_freem(mb); + /* Send a copy of the frame to the BPF listener */ + if (priv->dev && priv->dev->if_bpf) + ETHER_BPF_MTAP(priv->dev, mb); + m_freem(mb); return tx_info->nr_txbb; } - int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring) { struct mlx4_en_priv *priv = netdev_priv(dev); int cnt = 0; /* Skip last polled descriptor */ ring->cons += ring->last_nr_txbb; en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n", ring->cons, ring->prod); if ((u32) (ring->prod - ring->cons) > ring->size) { - en_warn(priv, "Tx consumer passed producer!\n"); + en_warn(priv, "Tx consumer passed producer!\n"); return 0; } while (ring->cons != ring->prod) { ring->last_nr_txbb = mlx4_en_free_tx_desc(priv, ring, ring->cons & ring->size_mask, - !!(ring->cons & ring->size)); + !!(ring->cons & ring->size), 0); ring->cons += ring->last_nr_txbb; cnt++; } if (cnt) en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt); return cnt; } -void mlx4_en_set_prio_map(struct mlx4_en_priv *priv, u16 *prio_map, u32 ring_num) +static int mlx4_en_process_tx_cq(struct net_device *dev, + struct mlx4_en_cq *cq) { - int block = 8 / ring_num; - int extra = 8 - (block * ring_num); - int num = 0; - u16 ring = 1; - int prio; - - if (ring_num == 1) { - for (prio = 0; prio < 8; prio++) - prio_map[prio] = 0; - return; - } - - for (prio = 0; prio < 8; prio++) { - if (extra && (num == block + 1)) { - ring++; - num = 0; - extra--; - } else if (!extra && (num == block)) { - ring++; - num = 0; - } - prio_map[prio] = ring; - en_dbg(DRV, priv, " prio:%d --> ring:%d\n", prio, ring); - num++; - } -} - -static void mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq) -{ struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_cq *mcq = &cq->mcq; - struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring]; - struct mlx4_cqe *cqe = cq->buf; + struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; + struct mlx4_cqe *cqe; u16 index; - u16 new_index; + u16 new_index, ring_index, stamp_index; u32 txbbs_skipped = 0; - u32 cq_last_sav; + u32 txbbs_stamp = 0; + u32 cons_index = mcq->cons_index; + int size = cq->size; + u32 size_mask = ring->size_mask; + struct mlx4_cqe *buf = cq->buf; + u32 packets = 0; + u32 bytes = 0; + int factor = priv->cqe_factor; + u64 timestamp = 0; + int done = 0; - /* index always points to the first TXBB of the last polled descriptor */ - index = ring->cons & ring->size_mask; - new_index = be16_to_cpu(cqe->wqe_index) & ring->size_mask; - if (index == new_index) - return; if (!priv->port_up) - return; + return 0; - /* - * We use a two-stage loop: - * - the first samples the HW-updated CQE - * - the second frees TXBBs until the last sample - * This lets us amortize CQE cache misses, while still polling the CQ - * until is quiescent. - */ - cq_last_sav = mcq->cons_index; - do { + index = cons_index & size_mask; + cqe = &buf[(index << factor) + factor]; + ring_index = ring->cons & size_mask; + stamp_index = ring_index; + + /* Process all completed CQEs */ + while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, + cons_index & size)) { + /* + * make sure we read the CQE after we read the + * ownership bit + */ + rmb(); + + if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == + MLX4_CQE_OPCODE_ERROR)) { + en_err(priv, "CQE completed in error - vendor syndrom: 0x%x syndrom: 0x%x\n", + ((struct mlx4_err_cqe *)cqe)-> + vendor_err_syndrome, + ((struct mlx4_err_cqe *)cqe)->syndrome); + } + + /* Skip over last polled CQE */ + new_index = be16_to_cpu(cqe->wqe_index) & size_mask; + do { - /* Skip over last polled CQE */ - index = (index + ring->last_nr_txbb) & ring->size_mask; txbbs_skipped += ring->last_nr_txbb; - - /* Poll next CQE */ + ring_index = (ring_index + ring->last_nr_txbb) & size_mask; + /* free next descriptor */ ring->last_nr_txbb = mlx4_en_free_tx_desc( - priv, ring, index, - !!((ring->cons + txbbs_skipped) & - ring->size)); - ++mcq->cons_index; + priv, ring, ring_index, + !!((ring->cons + txbbs_skipped) & + ring->size), timestamp); + mlx4_en_stamp_wqe(priv, ring, stamp_index, + !!((ring->cons + txbbs_stamp) & + ring->size)); + stamp_index = ring_index; + txbbs_stamp = txbbs_skipped; + packets++; + bytes += ring->tx_info[ring_index].nr_bytes; + } while (ring_index != new_index); - } while (index != new_index); + ++cons_index; + index = cons_index & size_mask; + cqe = &buf[(index << factor) + factor]; + } - new_index = be16_to_cpu(cqe->wqe_index) & ring->size_mask; - } while (index != new_index); - AVG_PERF_COUNTER(priv->pstats.tx_coal_avg, - (u32) (mcq->cons_index - cq_last_sav)); /* * To prevent CQ overflow we first update CQ consumer and only then * the ring consumer. */ + mcq->cons_index = cons_index; mlx4_cq_set_ci(mcq); wmb(); ring->cons += txbbs_skipped; - /* Wakeup Tx queue if this ring stopped it */ - if (unlikely(ring->blocked)) { - if ((u32) (ring->prod - ring->cons) <= - ring->size - HEADROOM - MAX_DESC_TXBBS) { - ring->blocked = 0; - if (atomic_fetchadd_int(&priv->blocked, -1) == 1) - atomic_clear_int(&dev->if_drv_flags, - IFF_DRV_OACTIVE); - priv->port_stats.wake_queue++; - } + /* Wakeup Tx queue if it was stopped and ring is not full */ + if (unlikely(ring->blocked) && + (ring->prod - ring->cons) <= ring->full_size) { + ring->blocked = 0; + if (atomic_fetchadd_int(&priv->blocked, -1) == 1) + atomic_clear_int(&dev->if_drv_flags ,IFF_DRV_OACTIVE); + ring->wake_queue++; + priv->port_stats.wake_queue++; } + return done; } void mlx4_en_tx_irq(struct mlx4_cq *mcq) { struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); struct mlx4_en_priv *priv = netdev_priv(cq->dev); - struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring]; + struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; if (!spin_trylock(&ring->comp_lock)) return; mlx4_en_process_tx_cq(cq->dev, cq); mod_timer(&cq->timer, jiffies + 1); spin_unlock(&ring->comp_lock); } - void mlx4_en_poll_tx_cq(unsigned long data) { struct mlx4_en_cq *cq = (struct mlx4_en_cq *) data; struct mlx4_en_priv *priv = netdev_priv(cq->dev); - struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring]; + struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; u32 inflight; INC_PERF_COUNTER(priv->pstats.tx_poll); if (!spin_trylock(&ring->comp_lock)) { mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); return; } mlx4_en_process_tx_cq(cq->dev, cq); inflight = (u32) (ring->prod - ring->cons - ring->last_nr_txbb); /* If there are still packets in flight and the timer has not already * been scheduled by the Tx routine then schedule it here to guarantee * completion processing of these packets */ if (inflight && priv->port_up) mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); spin_unlock(&ring->comp_lock); } static struct mlx4_en_tx_desc *mlx4_en_bounce_to_desc(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring, u32 index, unsigned int desc_size) { u32 copy = (ring->size - index) * TXBB_SIZE; int i; for (i = desc_size - copy - 4; i >= 0; i -= 4) { if ((i & (TXBB_SIZE - 1)) == 0) wmb(); *((u32 *) (ring->buf + i)) = *((u32 *) (ring->bounce_buf + copy + i)); } for (i = copy - 4; i >= 4 ; i -= 4) { if ((i & (TXBB_SIZE - 1)) == 0) wmb(); *((u32 *) (ring->buf + index * TXBB_SIZE + i)) = *((u32 *) (ring->bounce_buf + i)); } /* Return real descriptor location */ return ring->buf + index * TXBB_SIZE; } static inline void mlx4_en_xmit_poll(struct mlx4_en_priv *priv, int tx_ind) { - struct mlx4_en_cq *cq = &priv->tx_cq[tx_ind]; - struct mlx4_en_tx_ring *ring = &priv->tx_ring[tx_ind]; + struct mlx4_en_cq *cq = priv->tx_cq[tx_ind]; + struct mlx4_en_tx_ring *ring = priv->tx_ring[tx_ind]; /* If we don't have a pending timer, set one up to catch our recent post in case the interface becomes idle */ if (!timer_pending(&cq->timer)) mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); /* Poll the CQ every mlx4_en_TX_MODER_POLL packets */ if ((++ring->poll_cnt & (MLX4_EN_TX_POLL_MODER - 1)) == 0) if (spin_trylock(&ring->comp_lock)) { mlx4_en_process_tx_cq(priv->dev, cq); spin_unlock(&ring->comp_lock); } } -static int is_inline(struct mbuf *mb) +static int is_inline(struct mbuf *mb, int thold) { - - if (inline_thold && mb->m_pkthdr.len <= inline_thold && - (mb->m_pkthdr.csum_flags & CSUM_TSO) == 0) + if (thold && mb->m_pkthdr.len <= thold && + (mb->m_pkthdr.csum_flags & CSUM_TSO) == 0) return 1; - return 0; + return 0; } static int inline_size(struct mbuf *mb) { int len; len = mb->m_pkthdr.len; if (len + CTRL_SIZE + sizeof(struct mlx4_wqe_inline_seg) <= MLX4_INLINE_ALIGN) return ALIGN(len + CTRL_SIZE + sizeof(struct mlx4_wqe_inline_seg), 16); else return ALIGN(len + CTRL_SIZE + 2 * sizeof(struct mlx4_wqe_inline_seg), 16); } static int get_head_size(struct mbuf *mb) { - struct tcphdr *th; - struct ip *ip; - int ip_hlen, tcp_hlen; - int len; + struct ether_vlan_header *eh; + struct tcphdr *th; + struct ip *ip; + int ip_hlen, tcp_hlen; + struct ip6_hdr *ip6; + uint16_t eth_type; + int eth_hdr_len; - len = ETHER_HDR_LEN; - if (mb->m_len < len + sizeof(struct ip)) + eh = mtod(mb, struct ether_vlan_header *); + if (mb->m_len < ETHER_HDR_LEN) return (0); - ip = (struct ip *)(mtod(mb, char *) + len); - if (ip->ip_p != IPPROTO_TCP) + if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + eth_type = ntohs(eh->evl_proto); + eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + } else { + eth_type = ntohs(eh->evl_encap_proto); + eth_hdr_len = ETHER_HDR_LEN; + } + if (mb->m_len < eth_hdr_len) return (0); - ip_hlen = ip->ip_hl << 2; - len += ip_hlen; - if (mb->m_len < len + sizeof(struct tcphdr)) + switch (eth_type) { + case ETHERTYPE_IP: + ip = (struct ip *)(mb->m_data + eth_hdr_len); + if (mb->m_len < eth_hdr_len + sizeof(*ip)) + return (0); + if (ip->ip_p != IPPROTO_TCP) + return (0); + ip_hlen = ip->ip_hl << 2; + eth_hdr_len += ip_hlen; + break; + case ETHERTYPE_IPV6: + ip6 = (struct ip6_hdr *)(mb->m_data + eth_hdr_len); + if (mb->m_len < eth_hdr_len + sizeof(*ip6)) + return (0); + if (ip6->ip6_nxt != IPPROTO_TCP) + return (0); + eth_hdr_len += sizeof(*ip6); + break; + default: return (0); - th = (struct tcphdr *)(mtod(mb, char *) + len); + } + if (mb->m_len < eth_hdr_len + sizeof(*th)) + return (0); + th = (struct tcphdr *)(mb->m_data + eth_hdr_len); tcp_hlen = th->th_off << 2; - len += tcp_hlen; - if (mb->m_len < len) + eth_hdr_len += tcp_hlen; + if (mb->m_len < eth_hdr_len) return (0); - return (len); + return (eth_hdr_len); } -static int get_real_size(struct mbuf *mb, struct net_device *dev, int *segsp, - int *lso_header_size) +static int get_real_size(struct mbuf *mb, struct net_device *dev, int *p_n_segs, + int *lso_header_size, int inl) { - struct mbuf *m; - int nr_segs; + struct mbuf *m; + int nr_segs = 0; - nr_segs = 0; - for (m = mb; m != NULL; m = m->m_next) - if (m->m_len) - nr_segs++; + for (m = mb; m != NULL; m = m->m_next) + if (m->m_len) + nr_segs++; - if (mb->m_pkthdr.csum_flags & CSUM_TSO) { - *lso_header_size = get_head_size(mb); - if (*lso_header_size) { - if (mb->m_len == *lso_header_size) - nr_segs--; - *segsp = nr_segs; - return CTRL_SIZE + nr_segs * DS_SIZE + - ALIGN(*lso_header_size + 4, DS_SIZE); - } - } else - *lso_header_size = 0; - *segsp = nr_segs; - if (is_inline(mb)) - return inline_size(mb); - return (CTRL_SIZE + nr_segs * DS_SIZE); + if (mb->m_pkthdr.csum_flags & CSUM_TSO) { + *lso_header_size = get_head_size(mb); + if (*lso_header_size) { + if (mb->m_len == *lso_header_size) + nr_segs--; + *p_n_segs = nr_segs; + return CTRL_SIZE + nr_segs * DS_SIZE + + ALIGN(*lso_header_size + 4, DS_SIZE); + } + } else + *lso_header_size = 0; + *p_n_segs = nr_segs; + if (inl) + return inline_size(mb); + return (CTRL_SIZE + nr_segs * DS_SIZE); } static struct mbuf *mb_copy(struct mbuf *mb, int *offp, char *data, int len) { - int bytes; - int off; + int bytes; + int off; - off = *offp; - while (len) { - bytes = min(mb->m_len - off, len); - if (bytes) - memcpy(data, mb->m_data + off, bytes); - len -= bytes; - data += bytes; - off += bytes; - if (off == mb->m_len) { - off = 0; - mb = mb->m_next; - } - } - *offp = off; - return (mb); + off = *offp; + while (len) { + bytes = min(mb->m_len - off, len); + if (bytes) + memcpy(data, mb->m_data + off, bytes); + len -= bytes; + data += bytes; + off += bytes; + if (off == mb->m_len) { + off = 0; + mb = mb->m_next; + } + } + *offp = off; + return (mb); } static void build_inline_wqe(struct mlx4_en_tx_desc *tx_desc, struct mbuf *mb, - int real_size, u16 *vlan_tag, int tx_ind) + int real_size, u16 *vlan_tag, int tx_ind) { struct mlx4_wqe_inline_seg *inl = &tx_desc->inl; int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - sizeof *inl; int len; int off; off = 0; len = mb->m_pkthdr.len; if (len <= spc) { - inl->byte_count = cpu_to_be32(1 << 31 | len); + inl->byte_count = cpu_to_be32(1 << 31 | + (max_t(typeof(len), len, MIN_PKT_LEN))); mb_copy(mb, &off, (void *)(inl + 1), len); + if (len < MIN_PKT_LEN) + memset(((void *)(inl + 1)) + len, 0, + MIN_PKT_LEN - len); } else { inl->byte_count = cpu_to_be32(1 << 31 | spc); mb = mb_copy(mb, &off, (void *)(inl + 1), spc); inl = (void *) (inl + 1) + spc; mb_copy(mb, &off, (void *)(inl + 1), len - spc); wmb(); inl->byte_count = cpu_to_be32(1 << 31 | (len - spc)); } tx_desc->ctrl.vlan_tag = cpu_to_be16(*vlan_tag); tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN * !!(*vlan_tag); tx_desc->ctrl.fence_size = (real_size / 16) & 0x3f; } +static unsigned long hashrandom; +static void hashrandom_init(void *arg) +{ + hashrandom = random(); +} +SYSINIT(hashrandom_init, SI_SUB_KLD, SI_ORDER_SECOND, &hashrandom_init, NULL); + u16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb) { struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_tx_hash_entry *entry; - struct ether_header *eth; - struct tcphdr *th; - struct ip *iph; - u32 hash_index; - int tx_ind = 0; - u16 vlan_tag = 0; - int len; + u32 rings_p_up = priv->num_tx_rings_p_up; + u32 vlan_tag = 0; + u32 up = 0; + u32 queue_index; /* Obtain VLAN information if present */ if (mb->m_flags & M_VLANTAG) { vlan_tag = mb->m_pkthdr.ether_vtag; - /* Set the Tx ring to use according to vlan priority */ - tx_ind = priv->tx_prio_map[vlan_tag >> 13]; - if (tx_ind) - return tx_ind; + up = (vlan_tag >> 13); } - if (mb->m_len < - ETHER_HDR_LEN + sizeof(struct ip) + sizeof(struct tcphdr)) - return MLX4_EN_NUM_HASH_RINGS; - eth = mtod(mb, struct ether_header *); - /* Hashing is only done for TCP/IP or UDP/IP packets */ - if (be16_to_cpu(eth->ether_type) != ETHERTYPE_IP) - return MLX4_EN_NUM_HASH_RINGS; - len = ETHER_HDR_LEN; - iph = (struct ip *)(mtod(mb, char *) + len); - len += iph->ip_hl << 2; - th = (struct tcphdr *)(mtod(mb, char *) + len); - hash_index = be32_to_cpu(iph->ip_dst.s_addr) & MLX4_EN_TX_HASH_MASK; - switch(iph->ip_p) { - case IPPROTO_UDP: - break; - case IPPROTO_TCP: - if (mb->m_len < len + sizeof(struct tcphdr)) - return MLX4_EN_NUM_HASH_RINGS; - hash_index = - (hash_index ^ be16_to_cpu(th->th_dport ^ th->th_sport)) & - MLX4_EN_TX_HASH_MASK; - break; - default: - return MLX4_EN_NUM_HASH_RINGS; - } - entry = &priv->tx_hash[hash_index]; - if(unlikely(!entry->cnt)) { - tx_ind = hash_index & (MLX4_EN_NUM_HASH_RINGS / 2 - 1); - if (2 * entry->small_pkts > entry->big_pkts) - tx_ind += MLX4_EN_NUM_HASH_RINGS / 2; - entry->small_pkts = entry->big_pkts = 0; - entry->ring = tx_ind; - } + /* hash mbuf */ + queue_index = mlx4_en_hashmbuf(MLX4_F_HASHL3 | MLX4_F_HASHL4, mb, hashrandom); - entry->cnt++; - if (mb->m_pkthdr.len > MLX4_EN_SMALL_PKT_SIZE) - entry->big_pkts++; - else - entry->small_pkts++; - return entry->ring; + return ((queue_index % rings_p_up) + (up * rings_p_up)); } -static void mlx4_bf_copy(unsigned long *dst, unsigned long *src, unsigned bytecnt) +static void mlx4_bf_copy(void __iomem *dst, unsigned long *src, unsigned bytecnt) { __iowrite64_copy(dst, src, bytecnt / 8); } +static u64 mlx4_en_mac_to_u64(u8 *addr) +{ + u64 mac = 0; + int i; + + for (i = 0; i < ETHER_ADDR_LEN; i++) { + mac <<= 8; + mac |= addr[i]; + } + return mac; +} + static int mlx4_en_xmit(struct net_device *dev, int tx_ind, struct mbuf **mbp) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_tx_ring *ring; struct mlx4_en_cq *cq; struct mlx4_en_tx_desc *tx_desc; struct mlx4_wqe_data_seg *data; struct mlx4_en_tx_info *tx_info; struct mbuf *m; int nr_txbb; int nr_segs; int desc_size; int real_size; dma_addr_t dma; - u32 index, bf_index; + u32 index, bf_index, ring_size; __be32 op_own; u16 vlan_tag = 0; int i; int lso_header_size; bool bounce = false; + bool inl = false; struct mbuf *mb; + mb = *mbp; int defrag = 1; - ring = &priv->tx_ring[tx_ind]; - mb = *mbp; if (!priv->port_up) goto tx_drop; + ring = priv->tx_ring[tx_ind]; + ring_size = ring->size; + inl = is_inline(mb, ring->inline_thold); + retry: - real_size = get_real_size(mb, dev, &nr_segs, &lso_header_size); + real_size = get_real_size(mb, dev, &nr_segs, &lso_header_size, inl); if (unlikely(!real_size)) goto tx_drop; - /* Allign descriptor to TXBB size */ + /* Align descriptor to TXBB size */ desc_size = ALIGN(real_size, TXBB_SIZE); nr_txbb = desc_size / TXBB_SIZE; if (unlikely(nr_txbb > MAX_DESC_TXBBS)) { if (defrag) { - mb = m_defrag(*mbp, M_DONTWAIT); - if (mb == NULL) { - mb = *mbp; - goto tx_drop; - } - *mbp = mb; - defrag = 0; - goto retry; - } + mb = m_defrag(*mbp, M_NOWAIT); + if (mb == NULL) { + mb = *mbp; + goto tx_drop; + } + *mbp = mb; + defrag = 0; + goto retry; + } + en_warn(priv, "Oversized header or SG list\n"); goto tx_drop; } - /* Check available TXBBs And 2K spare for prefetch */ - if (unlikely(((int)(ring->prod - ring->cons)) > - ring->size - HEADROOM - MAX_DESC_TXBBS)) { + /* Obtain VLAN information if present */ + if (mb->m_flags & M_VLANTAG) { + vlan_tag = mb->m_pkthdr.ether_vtag; + } + + /* Check available TXBBs and 2K spare for prefetch + * Even if netif_tx_stop_queue() will be called + * driver will send current packet to ensure + * that at least one completion will be issued after + * stopping the queue + */ + if (unlikely((int)(ring->prod - ring->cons) > ring->full_size)) { /* every full Tx ring stops queue */ if (ring->blocked == 0) - atomic_add_int(&priv->blocked, 1); + atomic_add_int(&priv->blocked, 1); + /* Set HW-queue-is-full flag */ atomic_set_int(&dev->if_drv_flags, IFF_DRV_OACTIVE); ring->blocked = 1; priv->port_stats.queue_stopped++; + ring->queue_stopped++; /* Use interrupts to find out when queue opened */ - cq = &priv->tx_cq[tx_ind]; + cq = priv->tx_cq[tx_ind]; mlx4_en_arm_cq(priv, cq); return EBUSY; - } + } /* Track current inflight packets for performance analysis */ AVG_PERF_COUNTER(priv->pstats.inflight_avg, (u32) (ring->prod - ring->cons - 1)); /* Packet is good - grab an index and transmit it */ index = ring->prod & ring->size_mask; bf_index = ring->prod; /* See if we have enough space for whole descriptor TXBB for setting * SW ownership on next descriptor; if not, use a bounce buffer. */ - if (likely(index + nr_txbb <= ring->size)) + if (likely(index + nr_txbb <= ring_size)) tx_desc = ring->buf + index * TXBB_SIZE; else { tx_desc = (struct mlx4_en_tx_desc *) ring->bounce_buf; bounce = true; } + /* Save mb in tx_info ring */ + tx_info = &ring->tx_info[index]; + tx_info->mb = mb; + tx_info->nr_txbb = nr_txbb; + tx_info->nr_segs = nr_segs; + + if (lso_header_size) { + memcpy(tx_desc->lso.header, mb->m_data, lso_header_size); + data = ((void *)&tx_desc->lso + ALIGN(lso_header_size + 4, + DS_SIZE)); + /* lso header is part of m_data. + * need to omit when mapping DMA */ + mb->m_data += lso_header_size; + mb->m_len -= lso_header_size; + } + else + data = &tx_desc->data; + + /* valid only for none inline segments */ + tx_info->data_offset = (void *)data - (void *)tx_desc; + + if (inl) { + tx_info->inl = 1; + } else { + for (i = 0, m = mb; i < nr_segs; i++, m = m->m_next) { + if (m->m_len == 0) { + i--; + continue; + } + dma = pci_map_single(mdev->dev->pdev, m->m_data, + m->m_len, PCI_DMA_TODEVICE); + data->addr = cpu_to_be64(dma); + data->lkey = cpu_to_be32(mdev->mr.key); + wmb(); + data->byte_count = cpu_to_be32(m->m_len); + data++; + } + if (lso_header_size) { + mb->m_data -= lso_header_size; + mb->m_len += lso_header_size; + } + tx_info->inl = 0; + } + + /* Prepare ctrl segement apart opcode+ownership, which depends on * whether LSO is used */ - if (mb->m_flags & M_VLANTAG) - vlan_tag = mb->m_pkthdr.ether_vtag; tx_desc->ctrl.vlan_tag = cpu_to_be16(vlan_tag); - tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN * !!vlan_tag; + tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN * + !!vlan_tag; tx_desc->ctrl.fence_size = (real_size / 16) & 0x3f; - tx_desc->ctrl.srcrb_flags = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | - MLX4_WQE_CTRL_SOLICITED); - if (mb->m_pkthdr.csum_flags & (CSUM_IP|CSUM_TCP|CSUM_UDP)) { - if (mb->m_pkthdr.csum_flags & CSUM_IP) - tx_desc->ctrl.srcrb_flags |= - cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM); - if (mb->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_UDP)) - tx_desc->ctrl.srcrb_flags |= - cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM); + tx_desc->ctrl.srcrb_flags = priv->ctrl_flags; + if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO | + CSUM_TCP | CSUM_UDP | CSUM_TCP_IPV6 | CSUM_UDP_IPV6)) { + if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) + tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM); + if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | + CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) + tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM); priv->port_stats.tx_chksum_offload++; - } + ring->tx_csum++; + } if (unlikely(priv->validate_loopback)) { /* Copy dst mac address to wqe */ - struct ether_header *ethh; - u64 mac; - u32 mac_l, mac_h; + struct ether_header *ethh; + u64 mac; + u32 mac_l, mac_h; - ethh = mtod(mb, struct ether_header *); - mac = mlx4_en_mac_to_u64(ethh->ether_dhost); - if (mac) { - mac_h = (u32) ((mac & 0xffff00000000ULL) >> 16); - mac_l = (u32) (mac & 0xffffffff); - tx_desc->ctrl.srcrb_flags |= cpu_to_be32(mac_h); - tx_desc->ctrl.imm = cpu_to_be32(mac_l); - } + ethh = mtod(mb, struct ether_header *); + mac = mlx4_en_mac_to_u64(ethh->ether_dhost); + if (mac) { + mac_h = (u32) ((mac & 0xffff00000000ULL) >> 16); + mac_l = (u32) (mac & 0xffffffff); + tx_desc->ctrl.srcrb_flags |= cpu_to_be32(mac_h); + tx_desc->ctrl.imm = cpu_to_be32(mac_l); + } } /* Handle LSO (TSO) packets */ if (lso_header_size) { int segsz; - /* Mark opcode as LSO */ op_own = cpu_to_be32(MLX4_OPCODE_LSO | (1 << 6)) | - ((ring->prod & ring->size) ? + ((ring->prod & ring_size) ? cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0); /* Fill in the LSO prefix */ tx_desc->lso.mss_hdr_size = cpu_to_be32( mb->m_pkthdr.tso_segsz << 16 | lso_header_size); - /* Copy headers; - * note that we already verified that it is linear */ - memcpy(tx_desc->lso.header, mb->m_data, lso_header_size); - data = ((void *) &tx_desc->lso + - ALIGN(lso_header_size + 4, DS_SIZE)); - - priv->port_stats.tso_packets++; - segsz = mb->m_pkthdr.tso_segsz; - i = ((mb->m_pkthdr.len - lso_header_size) / segsz) + - !!((mb->m_pkthdr.len - lso_header_size) % segsz); - ring->bytes += mb->m_pkthdr.len + (i - 1) * lso_header_size; - ring->packets += i; - mb->m_data += lso_header_size; - mb->m_len -= lso_header_size; + priv->port_stats.tso_packets++; + segsz = mb->m_pkthdr.tso_segsz; + i = ((mb->m_pkthdr.len - lso_header_size + segsz - 1) / segsz); + tx_info->nr_bytes= mb->m_pkthdr.len + (i - 1) * lso_header_size; + ring->packets += i; } else { /* Normal (Non LSO) packet */ op_own = cpu_to_be32(MLX4_OPCODE_SEND) | - ((ring->prod & ring->size) ? + ((ring->prod & ring_size) ? cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0); - data = &tx_desc->data; - ring->bytes += max(mb->m_pkthdr.len, - (unsigned int)ETHER_MIN_LEN - ETHER_CRC_LEN); + tx_info->nr_bytes = max(mb->m_pkthdr.len, + (unsigned int)ETHER_MIN_LEN - ETHER_CRC_LEN); ring->packets++; } + ring->bytes += tx_info->nr_bytes; AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, mb->m_pkthdr.len); - /* Save mb in tx_info ring */ - tx_info = &ring->tx_info[index]; - tx_info->mb = mb; - tx_info->nr_txbb = nr_txbb; - tx_info->nr_segs = nr_segs; - /* valid only for non inline segments */ - tx_info->data_offset = (void *) data - (void *) tx_desc; - - if (!is_inline(mb)) { - for (i = 0, m = mb; i < nr_segs; i++, m = m->m_next) { - if (m->m_len == 0) { - i--; - continue; - } - dma = pci_map_single(mdev->dev->pdev, m->m_data, - m->m_len, PCI_DMA_TODEVICE); - data->addr = cpu_to_be64(dma); - data->lkey = cpu_to_be32(mdev->mr.key); - wmb(); - data->byte_count = cpu_to_be32(m->m_len); - data++; - } - if (lso_header_size) { - mb->m_data -= lso_header_size; - mb->m_len += lso_header_size; - } - tx_info->inl = 0; - } else { + if (tx_info->inl) { build_inline_wqe(tx_desc, mb, real_size, &vlan_tag, tx_ind); tx_info->inl = 1; } ring->prod += nr_txbb; + /* If we used a bounce buffer then copy descriptor back into place */ - if (bounce) + if (unlikely(bounce)) tx_desc = mlx4_en_bounce_to_desc(priv, ring, index, desc_size); - if (ring->bf_enabled && desc_size <= MAX_BF && !bounce && !vlan_tag) { - *(u32 *) (&tx_desc->ctrl.vlan_tag) |= ring->doorbell_qpn; + *(__be32 *) (&tx_desc->ctrl.vlan_tag) |= cpu_to_be32(ring->doorbell_qpn); op_own |= htonl((bf_index & 0xffff) << 8); /* Ensure new descirptor hits memory * before setting ownership of this descriptor to HW */ wmb(); tx_desc->ctrl.owner_opcode = op_own; wmb(); mlx4_bf_copy(ring->bf.reg + ring->bf.offset, (unsigned long *) &tx_desc->ctrl, desc_size); wmb(); ring->bf.offset ^= ring->bf.buf_size; } else { /* Ensure new descirptor hits memory * before setting ownership of this descriptor to HW */ wmb(); tx_desc->ctrl.owner_opcode = op_own; wmb(); - writel(ring->doorbell_qpn, ring->bf.uar->map + MLX4_SEND_DOORBELL); + writel(cpu_to_be32(ring->doorbell_qpn), ring->bf.uar->map + MLX4_SEND_DOORBELL); } return 0; - tx_drop: *mbp = NULL; m_freem(mb); - ring->errors++; return EINVAL; } - static int mlx4_en_transmit_locked(struct ifnet *dev, int tx_ind, struct mbuf *m) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_tx_ring *ring; struct mbuf *next; int enqueued, err = 0; - ring = &priv->tx_ring[tx_ind]; + ring = priv->tx_ring[tx_ind]; if ((dev->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING || priv->port_up == 0) { if (m != NULL) err = drbr_enqueue(dev, ring->br, m); - return (err); + return (err); } enqueued = 0; if (m != NULL) { if ((err = drbr_enqueue(dev, ring->br, m)) != 0) return (err); } /* Process the queue */ while ((next = drbr_peek(dev, ring->br)) != NULL) { if ((err = mlx4_en_xmit(dev, tx_ind, &next)) != 0) { if (next == NULL) { drbr_advance(dev, ring->br); } else { drbr_putback(dev, ring->br, next); } break; } drbr_advance(dev, ring->br); enqueued++; dev->if_obytes += next->m_pkthdr.len; if (next->m_flags & M_MCAST) dev->if_omcasts++; - /* Send a copy of the frame to the BPF listener */ - ETHER_BPF_MTAP(dev, next); if ((dev->if_drv_flags & IFF_DRV_RUNNING) == 0) break; } if (enqueued > 0) ring->watchdog_time = ticks; return (err); } void mlx4_en_tx_que(void *context, int pending) { struct mlx4_en_tx_ring *ring; struct mlx4_en_priv *priv; struct net_device *dev; struct mlx4_en_cq *cq; int tx_ind; - cq = context; dev = cq->dev; priv = dev->if_softc; tx_ind = cq->ring; - ring = &priv->tx_ring[tx_ind]; + ring = priv->tx_ring[tx_ind]; if (dev->if_drv_flags & IFF_DRV_RUNNING) { mlx4_en_xmit_poll(priv, tx_ind); spin_lock(&ring->tx_lock); if (!drbr_empty(dev, ring->br)) mlx4_en_transmit_locked(dev, tx_ind, NULL); spin_unlock(&ring->tx_lock); } } int mlx4_en_transmit(struct ifnet *dev, struct mbuf *m) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_tx_ring *ring; struct mlx4_en_cq *cq; int i = 0, err = 0; /* Which queue to use */ - if ((m->m_flags & (M_FLOWID | M_VLANTAG)) == M_FLOWID) - i = m->m_pkthdr.flowid % (MLX4_EN_NUM_HASH_RINGS - 1); - else + if ((m->m_flags & (M_FLOWID | M_VLANTAG)) == M_FLOWID) { + i = m->m_pkthdr.flowid % (priv->tx_ring_num - 1); + } + else { i = mlx4_en_select_queue(dev, m); + } + ring = priv->tx_ring[i]; - ring = &priv->tx_ring[i]; - if (spin_trylock(&ring->tx_lock)) { err = mlx4_en_transmit_locked(dev, i, m); spin_unlock(&ring->tx_lock); /* Poll CQ here */ mlx4_en_xmit_poll(priv, i); } else { err = drbr_enqueue(dev, ring->br, m); - cq = &priv->tx_cq[i]; + cq = priv->tx_cq[i]; taskqueue_enqueue(cq->tq, &cq->cq_task); } return (err); } /* * Flush ring buffers. */ void mlx4_en_qflush(struct ifnet *dev) { struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_tx_ring *ring = priv->tx_ring; + struct mlx4_en_tx_ring *ring; struct mbuf *m; - for (int i = 0; i < priv->tx_ring_num; i++, ring++) { + for (int i = 0; i < priv->tx_ring_num; i++) { + ring = priv->tx_ring[i]; spin_lock(&ring->tx_lock); while ((m = buf_ring_dequeue_sc(ring->br)) != NULL) m_freem(m); spin_unlock(&ring->tx_lock); } if_qflush(dev); } Index: stable/9/sys/ofed/drivers/net/mlx4/eq.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/eq.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/eq.c (revision 273864) @@ -1,727 +1,1406 @@ /* - * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include #include +#include +#include #include #include #include #include "mlx4.h" #include "fw.h" enum { + MLX4_IRQNAME_SIZE = 32 +}; + +enum { MLX4_NUM_ASYNC_EQE = 0x100, MLX4_NUM_SPARE_EQE = 0x80, MLX4_EQ_ENTRY_SIZE = 0x20 }; -/* - * Must be packed because start is 64 bits but only aligned to 32 bits. - */ -struct mlx4_eq_context { - __be32 flags; - u16 reserved1[3]; - __be16 page_offset; - u8 log_eq_size; - u8 reserved2[4]; - u8 eq_period; - u8 reserved3; - u8 eq_max_count; - u8 reserved4[3]; - u8 intr; - u8 log_page_size; - u8 reserved5[2]; - u8 mtt_base_addr_h; - __be32 mtt_base_addr_l; - u32 reserved6[2]; - __be32 consumer_index; - __be32 producer_index; - u32 reserved7[4]; -}; - #define MLX4_EQ_STATUS_OK ( 0 << 28) #define MLX4_EQ_STATUS_WRITE_FAIL (10 << 28) #define MLX4_EQ_OWNER_SW ( 0 << 24) #define MLX4_EQ_OWNER_HW ( 1 << 24) #define MLX4_EQ_FLAG_EC ( 1 << 18) #define MLX4_EQ_FLAG_OI ( 1 << 17) #define MLX4_EQ_STATE_ARMED ( 9 << 8) #define MLX4_EQ_STATE_FIRED (10 << 8) #define MLX4_EQ_STATE_ALWAYS_ARMED (11 << 8) #define MLX4_ASYNC_EVENT_MASK ((1ull << MLX4_EVENT_TYPE_PATH_MIG) | \ (1ull << MLX4_EVENT_TYPE_COMM_EST) | \ (1ull << MLX4_EVENT_TYPE_SQ_DRAINED) | \ (1ull << MLX4_EVENT_TYPE_CQ_ERROR) | \ (1ull << MLX4_EVENT_TYPE_WQ_CATAS_ERROR) | \ (1ull << MLX4_EVENT_TYPE_EEC_CATAS_ERROR) | \ (1ull << MLX4_EVENT_TYPE_PATH_MIG_FAILED) | \ (1ull << MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \ (1ull << MLX4_EVENT_TYPE_WQ_ACCESS_ERROR) | \ (1ull << MLX4_EVENT_TYPE_PORT_CHANGE) | \ (1ull << MLX4_EVENT_TYPE_ECC_DETECT) | \ (1ull << MLX4_EVENT_TYPE_SRQ_CATAS_ERROR) | \ (1ull << MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE) | \ (1ull << MLX4_EVENT_TYPE_SRQ_LIMIT) | \ - (1ull << MLX4_EVENT_TYPE_CMD)) + (1ull << MLX4_EVENT_TYPE_CMD) | \ + (1ull << MLX4_EVENT_TYPE_OP_REQUIRED) | \ + (1ull << MLX4_EVENT_TYPE_COMM_CHANNEL) | \ + (1ull << MLX4_EVENT_TYPE_FLR_EVENT) | \ + (1ull << MLX4_EVENT_TYPE_FATAL_WARNING)) -struct mlx4_eqe { - u8 reserved1; - u8 type; - u8 reserved2; - u8 subtype; - union { - u32 raw[6]; - struct { - __be32 cqn; - } __attribute__((packed)) comp; - struct { - u16 reserved1; - __be16 token; - u32 reserved2; - u8 reserved3[3]; - u8 status; - __be64 out_param; - } __attribute__((packed)) cmd; - struct { - __be32 qpn; - } __attribute__((packed)) qp; - struct { - __be32 srqn; - } __attribute__((packed)) srq; - struct { - __be32 cqn; - u32 reserved1; - u8 reserved2[3]; - u8 syndrome; - } __attribute__((packed)) cq_err; - struct { - u32 reserved1[2]; - __be32 port; - } __attribute__((packed)) port_change; - } event; - u8 reserved3[3]; - u8 owner; -} __attribute__((packed)); +static u64 get_async_ev_mask(struct mlx4_dev *dev) +{ + u64 async_ev_mask = MLX4_ASYNC_EVENT_MASK; + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV) + async_ev_mask |= (1ull << MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT); + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT) + async_ev_mask |= (1ull << MLX4_EVENT_TYPE_RECOVERABLE_ERROR_EVENT); + return async_ev_mask; +} + static void eq_set_ci(struct mlx4_eq *eq, int req_not) { __raw_writel((__force u32) cpu_to_be32((eq->cons_index & 0xffffff) | req_not << 31), eq->doorbell); /* We still want ordering, just not swabbing, so add a barrier */ mb(); } -static struct mlx4_eqe *get_eqe(struct mlx4_eq *eq, u32 entry) +static struct mlx4_eqe *get_eqe(struct mlx4_eq *eq, u32 entry, u8 eqe_factor) { - unsigned long off = (entry & (eq->nent - 1)) * MLX4_EQ_ENTRY_SIZE; - return eq->page_list[off / PAGE_SIZE].buf + off % PAGE_SIZE; + /* (entry & (eq->nent - 1)) gives us a cyclic array */ + unsigned long offset = (entry & (eq->nent - 1)) * (MLX4_EQ_ENTRY_SIZE << eqe_factor); + /* CX3 is capable of extending the EQE from 32 to 64 bytes. + * When this feature is enabled, the first (in the lower addresses) + * 32 bytes in the 64 byte EQE are reserved and the next 32 bytes + * contain the legacy EQE information. + */ + return eq->page_list[offset / PAGE_SIZE].buf + (offset + (eqe_factor ? MLX4_EQ_ENTRY_SIZE : 0)) % PAGE_SIZE; } -static struct mlx4_eqe *next_eqe_sw(struct mlx4_eq *eq) +static struct mlx4_eqe *next_eqe_sw(struct mlx4_eq *eq, u8 eqe_factor) { - struct mlx4_eqe *eqe = get_eqe(eq, eq->cons_index); + struct mlx4_eqe *eqe = get_eqe(eq, eq->cons_index, eqe_factor); return !!(eqe->owner & 0x80) ^ !!(eq->cons_index & eq->nent) ? NULL : eqe; } +static struct mlx4_eqe *next_slave_event_eqe(struct mlx4_slave_event_eq *slave_eq) +{ + struct mlx4_eqe *eqe = + &slave_eq->event_eqe[slave_eq->cons & (SLAVE_EVENT_EQ_SIZE - 1)]; + return (!!(eqe->owner & 0x80) ^ + !!(slave_eq->cons & SLAVE_EVENT_EQ_SIZE)) ? + eqe : NULL; +} + +void mlx4_gen_slave_eqe(struct work_struct *work) +{ + struct mlx4_mfunc_master_ctx *master = + container_of(work, struct mlx4_mfunc_master_ctx, + slave_event_work); + struct mlx4_mfunc *mfunc = + container_of(master, struct mlx4_mfunc, master); + struct mlx4_priv *priv = container_of(mfunc, struct mlx4_priv, mfunc); + struct mlx4_dev *dev = &priv->dev; + struct mlx4_slave_event_eq *slave_eq = &mfunc->master.slave_eq; + struct mlx4_eqe *eqe; + u8 slave; + int i; + + for (eqe = next_slave_event_eqe(slave_eq); eqe; + eqe = next_slave_event_eqe(slave_eq)) { + slave = eqe->slave_id; + + /* All active slaves need to receive the event */ + if (slave == ALL_SLAVES) { + for (i = 0; i < dev->num_slaves; i++) { + if (mlx4_GEN_EQE(dev, i, eqe)) + mlx4_warn(dev, "Failed to generate " + "event for slave %d\n", i); + } + } else { + if (mlx4_GEN_EQE(dev, slave, eqe)) + mlx4_warn(dev, "Failed to generate event " + "for slave %d\n", slave); + } + ++slave_eq->cons; + } +} + + +static void slave_event(struct mlx4_dev *dev, u8 slave, struct mlx4_eqe *eqe) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_event_eq *slave_eq = &priv->mfunc.master.slave_eq; + struct mlx4_eqe *s_eqe; + unsigned long flags; + + spin_lock_irqsave(&slave_eq->event_lock, flags); + s_eqe = &slave_eq->event_eqe[slave_eq->prod & (SLAVE_EVENT_EQ_SIZE - 1)]; + if ((!!(s_eqe->owner & 0x80)) ^ + (!!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE))) { + mlx4_warn(dev, "Master failed to generate an EQE for slave: %d. " + "No free EQE on slave events queue\n", slave); + spin_unlock_irqrestore(&slave_eq->event_lock, flags); + return; + } + + memcpy(s_eqe, eqe, dev->caps.eqe_size - 1); + s_eqe->slave_id = slave; + /* ensure all information is written before setting the ownersip bit */ + wmb(); + s_eqe->owner = !!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE) ? 0x0 : 0x80; + ++slave_eq->prod; + + queue_work(priv->mfunc.master.comm_wq, + &priv->mfunc.master.slave_event_work); + spin_unlock_irqrestore(&slave_eq->event_lock, flags); +} + +static void mlx4_slave_event(struct mlx4_dev *dev, int slave, + struct mlx4_eqe *eqe) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + if (slave < 0 || slave >= dev->num_slaves || + slave == dev->caps.function) + return; + + if (!priv->mfunc.master.slave_state[slave].active) + return; + + slave_event(dev, slave, eqe); +} + +int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, u8 port) +{ + struct mlx4_eqe eqe; + + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *s_slave = &priv->mfunc.master.slave_state[slave]; + + if (!s_slave->active) + return 0; + + memset(&eqe, 0, sizeof eqe); + + eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT; + eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE; + eqe.event.port_mgmt_change.port = port; + + return mlx4_GEN_EQE(dev, slave, &eqe); +} +EXPORT_SYMBOL(mlx4_gen_pkey_eqe); + +int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, u8 port) +{ + struct mlx4_eqe eqe; + + /*don't send if we don't have the that slave */ + if (dev->num_vfs < slave) + return 0; + memset(&eqe, 0, sizeof eqe); + + eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT; + eqe.subtype = MLX4_DEV_PMC_SUBTYPE_GUID_INFO; + eqe.event.port_mgmt_change.port = port; + + return mlx4_GEN_EQE(dev, slave, &eqe); +} +EXPORT_SYMBOL(mlx4_gen_guid_change_eqe); + +int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port, + u8 port_subtype_change) +{ + struct mlx4_eqe eqe; + + /*don't send if we don't have the that slave */ + if (dev->num_vfs < slave) + return 0; + memset(&eqe, 0, sizeof eqe); + + eqe.type = MLX4_EVENT_TYPE_PORT_CHANGE; + eqe.subtype = port_subtype_change; + eqe.event.port_change.port = cpu_to_be32(port << 28); + + mlx4_dbg(dev, "%s: sending: %d to slave: %d on port: %d\n", __func__, + port_subtype_change, slave, port); + return mlx4_GEN_EQE(dev, slave, &eqe); +} +EXPORT_SYMBOL(mlx4_gen_port_state_change_eqe); + +enum slave_port_state mlx4_get_slave_port_state(struct mlx4_dev *dev, int slave, u8 port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state; + if (slave >= dev->num_slaves || port > MLX4_MAX_PORTS) { + pr_err("%s: Error: asking for slave:%d, port:%d\n", + __func__, slave, port); + return SLAVE_PORT_DOWN; + } + return s_state[slave].port_state[port]; +} +EXPORT_SYMBOL(mlx4_get_slave_port_state); + +static int mlx4_set_slave_port_state(struct mlx4_dev *dev, int slave, u8 port, + enum slave_port_state state) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state; + + if (slave >= dev->num_slaves || port > MLX4_MAX_PORTS || port == 0) { + pr_err("%s: Error: asking for slave:%d, port:%d\n", + __func__, slave, port); + return -1; + } + s_state[slave].port_state[port] = state; + + return 0; +} + +static void set_all_slave_state(struct mlx4_dev *dev, u8 port, int event) +{ + int i; + enum slave_port_gen_event gen_event; + + for (i = 0; i < dev->num_slaves; i++) + set_and_calc_slave_port_state(dev, i, port, event, &gen_event); +} +/************************************************************************** + The function get as input the new event to that port, + and according to the prev state change the slave's port state. + The events are: + MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN, + MLX4_PORT_STATE_DEV_EVENT_PORT_UP + MLX4_PORT_STATE_IB_EVENT_GID_VALID + MLX4_PORT_STATE_IB_EVENT_GID_INVALID +***************************************************************************/ +int set_and_calc_slave_port_state(struct mlx4_dev *dev, int slave, + u8 port, int event, + enum slave_port_gen_event *gen_event) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *ctx = NULL; + unsigned long flags; + int ret = -1; + enum slave_port_state cur_state = + mlx4_get_slave_port_state(dev, slave, port); + + *gen_event = SLAVE_PORT_GEN_EVENT_NONE; + + if (slave >= dev->num_slaves || port > MLX4_MAX_PORTS || port == 0) { + pr_err("%s: Error: asking for slave:%d, port:%d\n", + __func__, slave, port); + return ret; + } + + ctx = &priv->mfunc.master.slave_state[slave]; + spin_lock_irqsave(&ctx->lock, flags); + + switch (cur_state) { + case SLAVE_PORT_DOWN: + if (MLX4_PORT_STATE_DEV_EVENT_PORT_UP == event) + mlx4_set_slave_port_state(dev, slave, port, + SLAVE_PENDING_UP); + break; + case SLAVE_PENDING_UP: + if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event) + mlx4_set_slave_port_state(dev, slave, port, + SLAVE_PORT_DOWN); + else if (MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID == event) { + mlx4_set_slave_port_state(dev, slave, port, + SLAVE_PORT_UP); + *gen_event = SLAVE_PORT_GEN_EVENT_UP; + } + break; + case SLAVE_PORT_UP: + if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event) { + mlx4_set_slave_port_state(dev, slave, port, + SLAVE_PORT_DOWN); + *gen_event = SLAVE_PORT_GEN_EVENT_DOWN; + } else if (MLX4_PORT_STATE_IB_EVENT_GID_INVALID == + event) { + mlx4_set_slave_port_state(dev, slave, port, + SLAVE_PENDING_UP); + *gen_event = SLAVE_PORT_GEN_EVENT_DOWN; + } + break; + default: + pr_err("%s: BUG!!! UNKNOWN state: " + "slave:%d, port:%d\n", __func__, slave, port); + goto out; + } + ret = mlx4_get_slave_port_state(dev, slave, port); + +out: + spin_unlock_irqrestore(&ctx->lock, flags); + return ret; +} + +EXPORT_SYMBOL(set_and_calc_slave_port_state); + +int mlx4_gen_slaves_port_mgt_ev(struct mlx4_dev *dev, u8 port, int attr, u16 sm_lid, u8 sm_sl) +{ + struct mlx4_eqe eqe; + + memset(&eqe, 0, sizeof eqe); + + eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT; + eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PORT_INFO; + eqe.event.port_mgmt_change.port = port; + eqe.event.port_mgmt_change.params.port_info.changed_attr = + cpu_to_be32((u32) attr); + if (attr & MSTR_SM_CHANGE_MASK) { + eqe.event.port_mgmt_change.params.port_info.mstr_sm_lid = + cpu_to_be16(sm_lid); + eqe.event.port_mgmt_change.params.port_info.mstr_sm_sl = + sm_sl; + } + + slave_event(dev, ALL_SLAVES, &eqe); + return 0; +} +EXPORT_SYMBOL(mlx4_gen_slaves_port_mgt_ev); + +void mlx4_master_handle_slave_flr(struct work_struct *work) +{ + struct mlx4_mfunc_master_ctx *master = + container_of(work, struct mlx4_mfunc_master_ctx, + slave_flr_event_work); + struct mlx4_mfunc *mfunc = + container_of(master, struct mlx4_mfunc, master); + struct mlx4_priv *priv = + container_of(mfunc, struct mlx4_priv, mfunc); + struct mlx4_dev *dev = &priv->dev; + struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; + int i; + int err; + unsigned long flags; + + mlx4_dbg(dev, "mlx4_handle_slave_flr\n"); + + for (i = 0 ; i < dev->num_slaves; i++) { + + if (MLX4_COMM_CMD_FLR == slave_state[i].last_cmd) { + mlx4_dbg(dev, "mlx4_handle_slave_flr: " + "clean slave: %d\n", i); + + mlx4_delete_all_resources_for_slave(dev, i); + /*return the slave to running mode*/ + spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); + slave_state[i].last_cmd = MLX4_COMM_CMD_RESET; + slave_state[i].is_slave_going_down = 0; + spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); + /*notify the FW:*/ + err = mlx4_cmd(dev, 0, i, 0, MLX4_CMD_INFORM_FLR_DONE, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + mlx4_warn(dev, "Failed to notify FW on " + "FLR done (slave:%d)\n", i); + } + } +} + static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq) { + struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_eqe *eqe; int cqn; int eqes_found = 0; int set_ci = 0; int port; + int slave = 0; + int ret; + u32 flr_slave; + u8 update_slave_state; + int i; + enum slave_port_gen_event gen_event; + unsigned long flags; + struct mlx4_vport_state *s_info; - while ((eqe = next_eqe_sw(eq))) { + while ((eqe = next_eqe_sw(eq, dev->caps.eqe_factor))) { /* * Make sure we read EQ entry contents after we've * checked the ownership bit. */ rmb(); switch (eqe->type) { case MLX4_EVENT_TYPE_COMP: cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff; mlx4_cq_completion(dev, cqn); break; case MLX4_EVENT_TYPE_PATH_MIG: case MLX4_EVENT_TYPE_COMM_EST: case MLX4_EVENT_TYPE_SQ_DRAINED: case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE: case MLX4_EVENT_TYPE_WQ_CATAS_ERROR: case MLX4_EVENT_TYPE_PATH_MIG_FAILED: case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR: case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR: - mlx4_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, - eqe->type); + mlx4_dbg(dev, "event %d arrived\n", eqe->type); + if (mlx4_is_master(dev)) { + /* forward only to slave owning the QP */ + ret = mlx4_get_slave_from_resource_id(dev, + RES_QP, + be32_to_cpu(eqe->event.qp.qpn) + & 0xffffff, &slave); + if (ret && ret != -ENOENT) { + mlx4_dbg(dev, "QP event %02x(%02x) on " + "EQ %d at index %u: could " + "not get slave id (%d)\n", + eqe->type, eqe->subtype, + eq->eqn, eq->cons_index, ret); + break; + } + + if (!ret && slave != dev->caps.function) { + mlx4_slave_event(dev, slave, eqe); + break; + } + + } + mlx4_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & + 0xffffff, eqe->type); break; case MLX4_EVENT_TYPE_SRQ_LIMIT: + mlx4_dbg(dev, "%s: MLX4_EVENT_TYPE_SRQ_LIMIT\n", + __func__); + /* fall through */ case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR: - mlx4_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) & 0xffffff, - eqe->type); + if (mlx4_is_master(dev)) { + /* forward only to slave owning the SRQ */ + ret = mlx4_get_slave_from_resource_id(dev, + RES_SRQ, + be32_to_cpu(eqe->event.srq.srqn) + & 0xffffff, + &slave); + if (ret && ret != -ENOENT) { + mlx4_warn(dev, "SRQ event %02x(%02x) " + "on EQ %d at index %u: could" + " not get slave id (%d)\n", + eqe->type, eqe->subtype, + eq->eqn, eq->cons_index, ret); + break; + } + mlx4_dbg(dev, "%s: slave:%d, srq_no:0x%x, event: %02x(%02x)\n", + __func__, slave, + be32_to_cpu(eqe->event.srq.srqn), + eqe->type, eqe->subtype); + + if (!ret && slave != dev->caps.function) { + mlx4_dbg(dev, "%s: sending event %02x(%02x) to slave:%d\n", + __func__, eqe->type, + eqe->subtype, slave); + mlx4_slave_event(dev, slave, eqe); + break; + } + } + mlx4_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) & + 0xffffff, eqe->type); break; case MLX4_EVENT_TYPE_CMD: mlx4_cmd_event(dev, be16_to_cpu(eqe->event.cmd.token), eqe->event.cmd.status, be64_to_cpu(eqe->event.cmd.out_param)); break; case MLX4_EVENT_TYPE_PORT_CHANGE: port = be32_to_cpu(eqe->event.port_change.port) >> 28; if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) { mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_DOWN, port); mlx4_priv(dev)->sense.do_sense_port[port] = 1; + if (!mlx4_is_master(dev)) + break; + for (i = 0; i < dev->num_slaves; i++) { + if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) { + if (i == mlx4_master_func_num(dev)) + continue; + mlx4_dbg(dev, "%s: Sending MLX4_PORT_CHANGE_SUBTYPE_DOWN" + " to slave: %d, port:%d\n", + __func__, i, port); + s_info = &priv->mfunc.master.vf_oper[slave].vport[port].state; + if (IFLA_VF_LINK_STATE_AUTO == s_info->link_state) + mlx4_slave_event(dev, i, eqe); + } else { /* IB port */ + set_and_calc_slave_port_state(dev, i, port, + MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN, + &gen_event); + /*we can be in pending state, then do not send port_down event*/ + if (SLAVE_PORT_GEN_EVENT_DOWN == gen_event) { + if (i == mlx4_master_func_num(dev)) + continue; + mlx4_slave_event(dev, i, eqe); + } + } + } } else { - mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_UP, - port); + mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_UP, port); + mlx4_priv(dev)->sense.do_sense_port[port] = 0; + + if (!mlx4_is_master(dev)) + break; + if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) + for (i = 0; i < dev->num_slaves; i++) { + if (i == mlx4_master_func_num(dev)) + continue; + s_info = &priv->mfunc.master.vf_oper[slave].vport[port].state; + if (IFLA_VF_LINK_STATE_AUTO == s_info->link_state) + mlx4_slave_event(dev, i, eqe); + } + else /* IB port */ + /* port-up event will be sent to a slave when the + * slave's alias-guid is set. This is done in alias_GUID.c + */ + set_all_slave_state(dev, port, MLX4_DEV_EVENT_PORT_UP); } break; case MLX4_EVENT_TYPE_CQ_ERROR: mlx4_warn(dev, "CQ %s on CQN %06x\n", eqe->event.cq_err.syndrome == 1 ? "overrun" : "access violation", be32_to_cpu(eqe->event.cq_err.cqn) & 0xffffff); - mlx4_cq_event(dev, be32_to_cpu(eqe->event.cq_err.cqn), + if (mlx4_is_master(dev)) { + ret = mlx4_get_slave_from_resource_id(dev, + RES_CQ, + be32_to_cpu(eqe->event.cq_err.cqn) + & 0xffffff, &slave); + if (ret && ret != -ENOENT) { + mlx4_dbg(dev, "CQ event %02x(%02x) on " + "EQ %d at index %u: could " + "not get slave id (%d)\n", + eqe->type, eqe->subtype, + eq->eqn, eq->cons_index, ret); + break; + } + + if (!ret && slave != dev->caps.function) { + mlx4_slave_event(dev, slave, eqe); + break; + } + } + mlx4_cq_event(dev, + be32_to_cpu(eqe->event.cq_err.cqn) + & 0xffffff, eqe->type); break; case MLX4_EVENT_TYPE_EQ_OVERFLOW: mlx4_warn(dev, "EQ overrun on EQN %d\n", eq->eqn); break; + case MLX4_EVENT_TYPE_OP_REQUIRED: + atomic_inc(&priv->opreq_count); + /* FW commands can't be executed from interrupt context + working in deferred task */ + queue_work(mlx4_wq, &priv->opreq_task); + break; + + case MLX4_EVENT_TYPE_COMM_CHANNEL: + if (!mlx4_is_master(dev)) { + mlx4_warn(dev, "Received comm channel event " + "for non master device\n"); + break; + } + + memcpy(&priv->mfunc.master.comm_arm_bit_vector, + eqe->event.comm_channel_arm.bit_vec, + sizeof eqe->event.comm_channel_arm.bit_vec); + + if (!queue_work(priv->mfunc.master.comm_wq, + &priv->mfunc.master.comm_work)) + mlx4_warn(dev, "Failed to queue comm channel work\n"); + + if (!queue_work(priv->mfunc.master.comm_wq, + &priv->mfunc.master.arm_comm_work)) + mlx4_warn(dev, "Failed to queue arm comm channel work\n"); + break; + + case MLX4_EVENT_TYPE_FLR_EVENT: + flr_slave = be32_to_cpu(eqe->event.flr_event.slave_id); + if (!mlx4_is_master(dev)) { + mlx4_warn(dev, "Non-master function received" + "FLR event\n"); + break; + } + + mlx4_dbg(dev, "FLR event for slave: %d\n", flr_slave); + + if (flr_slave >= dev->num_slaves) { + mlx4_warn(dev, + "Got FLR for unknown function: %d\n", + flr_slave); + update_slave_state = 0; + } else + update_slave_state = 1; + + spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags); + if (update_slave_state) { + priv->mfunc.master.slave_state[flr_slave].active = false; + priv->mfunc.master.slave_state[flr_slave].last_cmd = MLX4_COMM_CMD_FLR; + priv->mfunc.master.slave_state[flr_slave].is_slave_going_down = 1; + } + spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags); + queue_work(priv->mfunc.master.comm_wq, + &priv->mfunc.master.slave_flr_event_work); + break; + + case MLX4_EVENT_TYPE_FATAL_WARNING: + if (eqe->subtype == MLX4_FATAL_WARNING_SUBTYPE_WARMING) { + if (mlx4_is_master(dev)) + for (i = 0; i < dev->num_slaves; i++) { + mlx4_dbg(dev, "%s: Sending " + "MLX4_FATAL_WARNING_SUBTYPE_WARMING" + " to slave: %d\n", __func__, i); + if (i == dev->caps.function) + continue; + mlx4_slave_event(dev, i, eqe); + } + mlx4_err(dev, "Temperature Threshold was reached! " + "Threshold: %d celsius degrees; " + "Current Temperature: %d\n", + be16_to_cpu(eqe->event.warming.warning_threshold), + be16_to_cpu(eqe->event.warming.current_temperature)); + } else + mlx4_warn(dev, "Unhandled event FATAL WARNING (%02x), " + "subtype %02x on EQ %d at index %u. owner=%x, " + "nent=0x%x, slave=%x, ownership=%s\n", + eqe->type, eqe->subtype, eq->eqn, + eq->cons_index, eqe->owner, eq->nent, + eqe->slave_id, + !!(eqe->owner & 0x80) ^ + !!(eq->cons_index & eq->nent) ? "HW" : "SW"); + + break; + + case MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT: + mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_MGMT_CHANGE, + (unsigned long) eqe); + break; + + case MLX4_EVENT_TYPE_RECOVERABLE_ERROR_EVENT: + switch (eqe->subtype) { + case MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_BAD_CABLE: + mlx4_warn(dev, "Bad cable detected on port %u\n", + eqe->event.bad_cable.port); + break; + case MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_UNSUPPORTED_CABLE: + mlx4_warn(dev, "Unsupported cable detected\n"); + break; + default: + mlx4_dbg(dev, "Unhandled recoverable error event " + "detected: %02x(%02x) on EQ %d at index %u. " + "owner=%x, nent=0x%x, ownership=%s\n", + eqe->type, eqe->subtype, eq->eqn, + eq->cons_index, eqe->owner, eq->nent, + !!(eqe->owner & 0x80) ^ + !!(eq->cons_index & eq->nent) ? "HW" : "SW"); + break; + } + break; + case MLX4_EVENT_TYPE_EEC_CATAS_ERROR: case MLX4_EVENT_TYPE_ECC_DETECT: default: - mlx4_warn(dev, "Unhandled event %02x(%02x) on EQ %d at index %u\n", - eqe->type, eqe->subtype, eq->eqn, eq->cons_index); + mlx4_warn(dev, "Unhandled event %02x(%02x) on EQ %d at " + "index %u. owner=%x, nent=0x%x, slave=%x, " + "ownership=%s\n", + eqe->type, eqe->subtype, eq->eqn, + eq->cons_index, eqe->owner, eq->nent, + eqe->slave_id, + !!(eqe->owner & 0x80) ^ + !!(eq->cons_index & eq->nent) ? "HW" : "SW"); break; }; ++eq->cons_index; eqes_found = 1; ++set_ci; /* * The HCA will think the queue has overflowed if we * don't tell it we've been processing events. We * create our EQs with MLX4_NUM_SPARE_EQE extra * entries, so we must update our consumer index at * least that often. */ if (unlikely(set_ci >= MLX4_NUM_SPARE_EQE)) { eq_set_ci(eq, 0); set_ci = 0; } } eq_set_ci(eq, 1); return eqes_found; } static irqreturn_t mlx4_interrupt(int irq, void *dev_ptr) { struct mlx4_dev *dev = dev_ptr; struct mlx4_priv *priv = mlx4_priv(dev); int work = 0; int i; writel(priv->eq_table.clr_mask, priv->eq_table.clr_int); for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) work |= mlx4_eq_int(dev, &priv->eq_table.eq[i]); return IRQ_RETVAL(work); } static irqreturn_t mlx4_msi_x_interrupt(int irq, void *eq_ptr) { struct mlx4_eq *eq = eq_ptr; struct mlx4_dev *dev = eq->dev; mlx4_eq_int(dev, eq); /* MSI-X vectors always belong to us */ return IRQ_HANDLED; } +int mlx4_MAP_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_event_eq_info *event_eq = + priv->mfunc.master.slave_state[slave].event_eq; + u32 in_modifier = vhcr->in_modifier; + u32 eqn = in_modifier & 0x3FF; + u64 in_param = vhcr->in_param; + int err = 0; + int i; + + if (slave == dev->caps.function) + err = mlx4_cmd(dev, in_param, (in_modifier & 0x80000000) | eqn, + 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + if (!err) + for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i) + if (in_param & (1LL << i)) + event_eq[i].eqn = in_modifier >> 31 ? -1 : eqn; + + return err; +} + static int mlx4_MAP_EQ(struct mlx4_dev *dev, u64 event_mask, int unmap, int eq_num) { return mlx4_cmd(dev, event_mask, (unmap << 31) | eq_num, - 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B); + 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); } static int mlx4_SW2HW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, int eq_num) { - return mlx4_cmd(dev, mailbox->dma, eq_num, 0, MLX4_CMD_SW2HW_EQ, - MLX4_CMD_TIME_CLASS_A); + return mlx4_cmd(dev, mailbox->dma, eq_num, 0, + MLX4_CMD_SW2HW_EQ, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); } static int mlx4_HW2SW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, int eq_num) { - return mlx4_cmd_box(dev, 0, mailbox->dma, eq_num, 0, MLX4_CMD_HW2SW_EQ, - MLX4_CMD_TIME_CLASS_A); + return mlx4_cmd_box(dev, 0, mailbox->dma, eq_num, + 0, MLX4_CMD_HW2SW_EQ, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); } static int mlx4_num_eq_uar(struct mlx4_dev *dev) { /* * Each UAR holds 4 EQ doorbells. To figure out how many UARs * we need to map, take the difference of highest index and * the lowest index we'll use and add 1. */ - return (dev->caps.num_comp_vectors + 1 + dev->caps.reserved_eqs) / 4 - - dev->caps.reserved_eqs / 4 + 1; + return (dev->caps.num_comp_vectors + 1 + dev->caps.reserved_eqs + + dev->caps.comp_pool)/4 - dev->caps.reserved_eqs/4 + 1; } static void __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev, struct mlx4_eq *eq) { struct mlx4_priv *priv = mlx4_priv(dev); int index; index = eq->eqn / 4 - dev->caps.reserved_eqs / 4; if (!priv->eq_table.uar_map[index]) { priv->eq_table.uar_map[index] = ioremap(pci_resource_start(dev->pdev, 2) + ((eq->eqn / 4) << PAGE_SHIFT), PAGE_SIZE); if (!priv->eq_table.uar_map[index]) { mlx4_err(dev, "Couldn't map EQ doorbell for EQN 0x%06x\n", eq->eqn); return NULL; } } return priv->eq_table.uar_map[index] + 0x800 + 8 * (eq->eqn % 4); } +static void mlx4_unmap_uar(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int i; + + for (i = 0; i < mlx4_num_eq_uar(dev); ++i) + if (priv->eq_table.uar_map[i]) { + iounmap(priv->eq_table.uar_map[i]); + priv->eq_table.uar_map[i] = NULL; + } +} + static int mlx4_create_eq(struct mlx4_dev *dev, int nent, u8 intr, struct mlx4_eq *eq) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cmd_mailbox *mailbox; struct mlx4_eq_context *eq_context; int npages; u64 *dma_list = NULL; dma_addr_t t; u64 mtt_addr; int err = -ENOMEM; int i; eq->dev = dev; eq->nent = roundup_pow_of_two(max(nent, 2)); - npages = PAGE_ALIGN(eq->nent * MLX4_EQ_ENTRY_SIZE) / PAGE_SIZE; + /* CX3 is capable of extending the CQE\EQE from 32 to 64 bytes */ + npages = PAGE_ALIGN(eq->nent * (MLX4_EQ_ENTRY_SIZE << dev->caps.eqe_factor)) / PAGE_SIZE; eq->page_list = kmalloc(npages * sizeof *eq->page_list, GFP_KERNEL); if (!eq->page_list) goto err_out; for (i = 0; i < npages; ++i) eq->page_list[i].buf = NULL; dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); if (!dma_list) goto err_out_free; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) goto err_out_free; eq_context = mailbox->buf; for (i = 0; i < npages; ++i) { eq->page_list[i].buf = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, &t, GFP_KERNEL); if (!eq->page_list[i].buf) goto err_out_free_pages; dma_list[i] = t; eq->page_list[i].map = t; memset(eq->page_list[i].buf, 0, PAGE_SIZE); } eq->eqn = mlx4_bitmap_alloc(&priv->eq_table.bitmap); if (eq->eqn == -1) goto err_out_free_pages; eq->doorbell = mlx4_get_eq_uar(dev, eq); if (!eq->doorbell) { err = -ENOMEM; goto err_out_free_eq; } err = mlx4_mtt_init(dev, npages, PAGE_SHIFT, &eq->mtt); if (err) goto err_out_free_eq; err = mlx4_write_mtt(dev, &eq->mtt, 0, npages, dma_list); if (err) goto err_out_free_mtt; memset(eq_context, 0, sizeof *eq_context); eq_context->flags = cpu_to_be32(MLX4_EQ_STATUS_OK | MLX4_EQ_STATE_ARMED); eq_context->log_eq_size = ilog2(eq->nent); eq_context->intr = intr; eq_context->log_page_size = PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT; mtt_addr = mlx4_mtt_addr(dev, &eq->mtt); eq_context->mtt_base_addr_h = mtt_addr >> 32; eq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); err = mlx4_SW2HW_EQ(dev, mailbox, eq->eqn); if (err) { mlx4_warn(dev, "SW2HW_EQ failed (%d)\n", err); goto err_out_free_mtt; } kfree(dma_list); mlx4_free_cmd_mailbox(dev, mailbox); eq->cons_index = 0; return err; err_out_free_mtt: mlx4_mtt_cleanup(dev, &eq->mtt); err_out_free_eq: - mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn); + mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn, MLX4_USE_RR); err_out_free_pages: for (i = 0; i < npages; ++i) if (eq->page_list[i].buf) dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, eq->page_list[i].buf, eq->page_list[i].map); mlx4_free_cmd_mailbox(dev, mailbox); err_out_free: kfree(eq->page_list); kfree(dma_list); err_out: return err; } static void mlx4_free_eq(struct mlx4_dev *dev, struct mlx4_eq *eq) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cmd_mailbox *mailbox; int err; - int npages = PAGE_ALIGN(MLX4_EQ_ENTRY_SIZE * eq->nent) / PAGE_SIZE; int i; + /* CX3 is capable of extending the CQE\EQE from 32 to 64 bytes */ + int npages = PAGE_ALIGN((MLX4_EQ_ENTRY_SIZE << dev->caps.eqe_factor) * eq->nent) / PAGE_SIZE; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return; err = mlx4_HW2SW_EQ(dev, mailbox, eq->eqn); if (err) mlx4_warn(dev, "HW2SW_EQ failed (%d)\n", err); if (0) { mlx4_dbg(dev, "Dumping EQ context %02x:\n", eq->eqn); for (i = 0; i < sizeof (struct mlx4_eq_context) / 4; ++i) { if (i % 4 == 0) - printk("[%02x] ", i * 4); - printk(" %08x", be32_to_cpup(mailbox->buf + i * 4)); + pr_cont("[%02x] ", i * 4); + pr_cont(" %08x", be32_to_cpup(mailbox->buf + i * 4)); if ((i + 1) % 4 == 0) - printk("\n"); + pr_cont("\n"); } } mlx4_mtt_cleanup(dev, &eq->mtt); for (i = 0; i < npages; ++i) - pci_free_consistent(dev->pdev, PAGE_SIZE, + dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, eq->page_list[i].buf, eq->page_list[i].map); kfree(eq->page_list); - mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn); + mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn, MLX4_USE_RR); mlx4_free_cmd_mailbox(dev, mailbox); } static void mlx4_free_irqs(struct mlx4_dev *dev) { struct mlx4_eq_table *eq_table = &mlx4_priv(dev)->eq_table; - int i; + struct mlx4_priv *priv = mlx4_priv(dev); + int i, vec; if (eq_table->have_irq) free_irq(dev->pdev->irq, dev); + for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) if (eq_table->eq[i].have_irq) { free_irq(eq_table->eq[i].irq, eq_table->eq + i); eq_table->eq[i].have_irq = 0; } + for (i = 0; i < dev->caps.comp_pool; i++) { + /* + * Freeing the assigned irq's + * all bits should be 0, but we need to validate + */ + if (priv->msix_ctl.pool_bm & 1ULL << i) { + /* NO need protecting*/ + vec = dev->caps.num_comp_vectors + 1 + i; + free_irq(priv->eq_table.eq[vec].irq, + &priv->eq_table.eq[vec]); + } + } + + kfree(eq_table->irq_names); } static int mlx4_map_clr_int(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); priv->clr_base = ioremap(pci_resource_start(dev->pdev, priv->fw.clr_int_bar) + priv->fw.clr_int_base, MLX4_CLR_INT_SIZE); if (!priv->clr_base) { mlx4_err(dev, "Couldn't map interrupt clear register, aborting.\n"); return -ENOMEM; } return 0; } static void mlx4_unmap_clr_int(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); iounmap(priv->clr_base); } int mlx4_alloc_eq_table(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); priv->eq_table.eq = kcalloc(dev->caps.num_eqs - dev->caps.reserved_eqs, sizeof *priv->eq_table.eq, GFP_KERNEL); if (!priv->eq_table.eq) return -ENOMEM; return 0; } void mlx4_free_eq_table(struct mlx4_dev *dev) { kfree(mlx4_priv(dev)->eq_table.eq); } int mlx4_init_eq_table(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int err; int i; - priv->eq_table.uar_map = kcalloc(sizeof *priv->eq_table.uar_map, - mlx4_num_eq_uar(dev), GFP_KERNEL); + priv->eq_table.uar_map = kcalloc(mlx4_num_eq_uar(dev), + sizeof *priv->eq_table.uar_map, + GFP_KERNEL); if (!priv->eq_table.uar_map) { err = -ENOMEM; goto err_out_free; } err = mlx4_bitmap_init(&priv->eq_table.bitmap, dev->caps.num_eqs, dev->caps.num_eqs - 1, dev->caps.reserved_eqs, 0); if (err) goto err_out_free; for (i = 0; i < mlx4_num_eq_uar(dev); ++i) priv->eq_table.uar_map[i] = NULL; - err = mlx4_map_clr_int(dev); - if (err) - goto err_out_bitmap; + if (!mlx4_is_slave(dev)) { + err = mlx4_map_clr_int(dev); + if (err) + goto err_out_bitmap; - priv->eq_table.clr_mask = - swab32(1 << (priv->eq_table.inta_pin & 31)); - priv->eq_table.clr_int = priv->clr_base + - (priv->eq_table.inta_pin < 32 ? 4 : 0); + priv->eq_table.clr_mask = + swab32(1 << (priv->eq_table.inta_pin & 31)); + priv->eq_table.clr_int = priv->clr_base + + (priv->eq_table.inta_pin < 32 ? 4 : 0); + } - priv->eq_table.irq_names = kmalloc(16 * dev->caps.num_comp_vectors, GFP_KERNEL); + priv->eq_table.irq_names = + kmalloc(MLX4_IRQNAME_SIZE * (dev->caps.num_comp_vectors + 1 + + dev->caps.comp_pool), + GFP_KERNEL); if (!priv->eq_table.irq_names) { err = -ENOMEM; - goto err_out_bitmap; + goto err_out_clr_int; } for (i = 0; i < dev->caps.num_comp_vectors; ++i) { - err = mlx4_create_eq(dev, dev->caps.num_cqs + MLX4_NUM_SPARE_EQE, + err = mlx4_create_eq(dev, dev->caps.num_cqs - + dev->caps.reserved_cqs + + MLX4_NUM_SPARE_EQE, (dev->flags & MLX4_FLAG_MSI_X) ? i : 0, &priv->eq_table.eq[i]); if (err) { --i; goto err_out_unmap; } } err = mlx4_create_eq(dev, MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE, (dev->flags & MLX4_FLAG_MSI_X) ? dev->caps.num_comp_vectors : 0, &priv->eq_table.eq[dev->caps.num_comp_vectors]); if (err) goto err_out_comp; + /*if additional completion vectors poolsize is 0 this loop will not run*/ + for (i = dev->caps.num_comp_vectors + 1; + i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i) { + + err = mlx4_create_eq(dev, dev->caps.num_cqs - + dev->caps.reserved_cqs + + MLX4_NUM_SPARE_EQE, + (dev->flags & MLX4_FLAG_MSI_X) ? i : 0, + &priv->eq_table.eq[i]); + if (err) { + --i; + goto err_out_unmap; + } + } + + if (dev->flags & MLX4_FLAG_MSI_X) { - static const char async_eq_name[] = DRV_NAME "(async)"; const char *eq_name; for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) { if (i < dev->caps.num_comp_vectors) { - snprintf(priv->eq_table.irq_names + i * 16, 16, - "eth-mlx4-%d", i); - eq_name = priv->eq_table.irq_names + i * 16; - } else - eq_name = async_eq_name; + snprintf(priv->eq_table.irq_names + + i * MLX4_IRQNAME_SIZE, + MLX4_IRQNAME_SIZE, + "mlx4-comp-%d@pci:%s", i, + pci_name(dev->pdev)); + } else { + snprintf(priv->eq_table.irq_names + + i * MLX4_IRQNAME_SIZE, + MLX4_IRQNAME_SIZE, + "mlx4-async@pci:%s", + pci_name(dev->pdev)); + } + eq_name = priv->eq_table.irq_names + + i * MLX4_IRQNAME_SIZE; err = request_irq(priv->eq_table.eq[i].irq, mlx4_msi_x_interrupt, 0, eq_name, priv->eq_table.eq + i); if (err) goto err_out_async; priv->eq_table.eq[i].have_irq = 1; } } else { + snprintf(priv->eq_table.irq_names, + MLX4_IRQNAME_SIZE, + DRV_NAME "@pci:%s", + pci_name(dev->pdev)); err = request_irq(dev->pdev->irq, mlx4_interrupt, - IRQF_SHARED, DRV_NAME, dev); + IRQF_SHARED, priv->eq_table.irq_names, dev); if (err) goto err_out_async; priv->eq_table.have_irq = 1; } - err = mlx4_MAP_EQ(dev, MLX4_ASYNC_EVENT_MASK, 0, + err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); if (err) mlx4_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n", priv->eq_table.eq[dev->caps.num_comp_vectors].eqn, err); for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) eq_set_ci(&priv->eq_table.eq[i], 1); return 0; err_out_async: mlx4_free_eq(dev, &priv->eq_table.eq[dev->caps.num_comp_vectors]); err_out_comp: i = dev->caps.num_comp_vectors - 1; err_out_unmap: while (i >= 0) { mlx4_free_eq(dev, &priv->eq_table.eq[i]); --i; } - mlx4_unmap_clr_int(dev); mlx4_free_irqs(dev); +err_out_clr_int: + if (!mlx4_is_slave(dev)) + mlx4_unmap_clr_int(dev); + err_out_bitmap: + mlx4_unmap_uar(dev); mlx4_bitmap_cleanup(&priv->eq_table.bitmap); err_out_free: kfree(priv->eq_table.uar_map); return err; } void mlx4_cleanup_eq_table(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int i; - mlx4_MAP_EQ(dev, MLX4_ASYNC_EVENT_MASK, 1, + mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 1, priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); mlx4_free_irqs(dev); - for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) + for (i = 0; i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i) mlx4_free_eq(dev, &priv->eq_table.eq[i]); - mlx4_unmap_clr_int(dev); + if (!mlx4_is_slave(dev)) + mlx4_unmap_clr_int(dev); - for (i = 0; i < mlx4_num_eq_uar(dev); ++i) - if (priv->eq_table.uar_map[i]) - iounmap(priv->eq_table.uar_map[i]); - + mlx4_unmap_uar(dev); mlx4_bitmap_cleanup(&priv->eq_table.bitmap); kfree(priv->eq_table.uar_map); } /* A test that verifies that we can accept interrupts on all * the irq vectors of the device. * Interrupts are checked using the NOP command. */ int mlx4_test_interrupts(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int i; int err; err = mlx4_NOP(dev); /* When not in MSI_X, there is only one irq to check */ - if (!(dev->flags & MLX4_FLAG_MSI_X)) + if (!(dev->flags & MLX4_FLAG_MSI_X) || mlx4_is_slave(dev)) return err; /* A loop over all completion vectors, for each vector we will check * whether it works by mapping command completions to that vector * and performing a NOP command */ for(i = 0; !err && (i < dev->caps.num_comp_vectors); ++i) { /* Temporary use polling for command completions */ mlx4_cmd_use_polling(dev); - /* Map the new eq to handle all asynchronous events */ - err = mlx4_MAP_EQ(dev, MLX4_ASYNC_EVENT_MASK, 0, + /* Map the new eq to handle all asyncronous events */ + err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, priv->eq_table.eq[i].eqn); if (err) { mlx4_warn(dev, "Failed mapping eq for interrupt test\n"); mlx4_cmd_use_events(dev); break; } /* Go back to using events */ mlx4_cmd_use_events(dev); err = mlx4_NOP(dev); } /* Return to default */ - mlx4_MAP_EQ(dev, MLX4_ASYNC_EVENT_MASK, 0, + mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); return err; } EXPORT_SYMBOL(mlx4_test_interrupts); + +int mlx4_assign_eq(struct mlx4_dev *dev, char* name, int * vector) +{ + + struct mlx4_priv *priv = mlx4_priv(dev); + int vec = 0, err = 0, i; + + mutex_lock(&priv->msix_ctl.pool_lock); + for (i = 0; !vec && i < dev->caps.comp_pool; i++) { + if (~priv->msix_ctl.pool_bm & 1ULL << i) { + priv->msix_ctl.pool_bm |= 1ULL << i; + vec = dev->caps.num_comp_vectors + 1 + i; + snprintf(priv->eq_table.irq_names + + vec * MLX4_IRQNAME_SIZE, + MLX4_IRQNAME_SIZE, "%s", name); + err = request_irq(priv->eq_table.eq[vec].irq, + mlx4_msi_x_interrupt, 0, + &priv->eq_table.irq_names[vec<<5], + priv->eq_table.eq + vec); + if (err) { + /*zero out bit by fliping it*/ + priv->msix_ctl.pool_bm ^= 1 << i; + vec = 0; + continue; + /*we dont want to break here*/ + } + eq_set_ci(&priv->eq_table.eq[vec], 1); + } + } + mutex_unlock(&priv->msix_ctl.pool_lock); + + if (vec) { + *vector = vec; + } else { + *vector = 0; + err = (i == dev->caps.comp_pool) ? -ENOSPC : err; + } + return err; +} +EXPORT_SYMBOL(mlx4_assign_eq); + +void mlx4_release_eq(struct mlx4_dev *dev, int vec) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + /*bm index*/ + int i = vec - dev->caps.num_comp_vectors - 1; + + if (likely(i >= 0)) { + /*sanity check , making sure were not trying to free irq's + Belonging to a legacy EQ*/ + mutex_lock(&priv->msix_ctl.pool_lock); + if (priv->msix_ctl.pool_bm & 1ULL << i) { + free_irq(priv->eq_table.eq[vec].irq, + &priv->eq_table.eq[vec]); + priv->msix_ctl.pool_bm &= ~(1ULL << i); + } + mutex_unlock(&priv->msix_ctl.pool_lock); + } + +} +EXPORT_SYMBOL(mlx4_release_eq); + Index: stable/9/sys/ofed/drivers/net/mlx4/fw.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/fw.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/fw.c (revision 273864) @@ -1,1030 +1,1954 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include #include +#include +#include #include "fw.h" #include "icm.h" enum { MLX4_COMMAND_INTERFACE_MIN_REV = 2, MLX4_COMMAND_INTERFACE_MAX_REV = 3, MLX4_COMMAND_INTERFACE_NEW_PORT_CMDS = 3, }; extern void __buggy_use_of_MLX4_GET(void); extern void __buggy_use_of_MLX4_PUT(void); -static int enable_qos; +static bool enable_qos; module_param(enable_qos, bool, 0444); MODULE_PARM_DESC(enable_qos, "Enable Quality of Service support in the HCA (default: off)"); -static int mlx4_pre_t11_mode = 0; -module_param_named(enable_pre_t11_mode, mlx4_pre_t11_mode, int, 0644); -MODULE_PARM_DESC(enable_pre_t11_mode, "For FCoXX, enable pre-t11 mode if non-zero (default: 0)"); - #define MLX4_GET(dest, source, offset) \ do { \ void *__p = (char *) (source) + (offset); \ switch (sizeof (dest)) { \ case 1: (dest) = *(u8 *) __p; break; \ case 2: (dest) = be16_to_cpup(__p); break; \ case 4: (dest) = be32_to_cpup(__p); break; \ case 8: (dest) = be64_to_cpup(__p); break; \ default: __buggy_use_of_MLX4_GET(); \ } \ } while (0) #define MLX4_PUT(dest, source, offset) \ do { \ void *__d = ((char *) (dest) + (offset)); \ switch (sizeof(source)) { \ case 1: *(u8 *) __d = (source); break; \ case 2: *(__be16 *) __d = cpu_to_be16(source); break; \ case 4: *(__be32 *) __d = cpu_to_be32(source); break; \ case 8: *(__be64 *) __d = cpu_to_be64(source); break; \ default: __buggy_use_of_MLX4_PUT(); \ } \ } while (0) static void dump_dev_cap_flags(struct mlx4_dev *dev, u64 flags) { static const char *fname[] = { [ 0] = "RC transport", [ 1] = "UC transport", [ 2] = "UD transport", [ 3] = "XRC transport", [ 4] = "reliable multicast", [ 5] = "FCoIB support", [ 6] = "SRQ support", [ 7] = "IPoIB checksum offload", [ 8] = "P_Key violation counter", [ 9] = "Q_Key violation counter", [10] = "VMM", [12] = "DPDP", + [15] = "Big LSO headers", [16] = "MW support", [17] = "APM support", [18] = "Atomic ops support", [19] = "Raw multicast support", [20] = "Address vector port checking support", [21] = "UD multicast support", [24] = "Demand paging support", [25] = "Router support", [30] = "IBoE support", - [48] = "Basic counters support", - [49] = "Extended counters support", + [32] = "Unicast loopback support", + [34] = "FCS header control", + [38] = "Wake On LAN support", + [40] = "UDP RSS support", + [41] = "Unicast VEP steering support", + [42] = "Multicast VEP steering support", + [44] = "Cross-channel (sync_qp) operations support", + [48] = "Counters support", + [59] = "Port management change event support", + [60] = "eSwitch support", + [61] = "64 byte EQE support", + [62] = "64 byte CQE support", }; int i; mlx4_dbg(dev, "DEV_CAP flags:\n"); for (i = 0; i < ARRAY_SIZE(fname); ++i) if (fname[i] && (flags & (1LL << i))) mlx4_dbg(dev, " %s\n", fname[i]); } +static void dump_dev_cap_flags2(struct mlx4_dev *dev, u64 flags) +{ + static const char * const fname[] = { + [0] = "RSS support", + [1] = "RSS Toeplitz Hash Function support", + [2] = "RSS XOR Hash Function support", + [3] = "Device manage flow steering support", + [4] = "FSM (MAC unti-spoofing) support", + [5] = "VST (control vlan insertion/stripping) support", + [6] = "Dynamic QP updates support", + [7] = "Loopback source checks support", + [8] = "Device managed flow steering IPoIB support", + [9] = "ETS configuration support", + [10] = "ETH backplane autoneg report", + [11] = "Ethernet Flow control statistics support", + [12] = "Recoverable error events support", + [13] = "Time stamping support", + [14] = "Report driver version to FW support" + }; + int i; + + for (i = 0; i < ARRAY_SIZE(fname); ++i) + if (fname[i] && (flags & (1LL << i))) + mlx4_dbg(dev, " %s\n", fname[i]); +} + int mlx4_MOD_STAT_CFG(struct mlx4_dev *dev, struct mlx4_mod_stat_cfg *cfg) { struct mlx4_cmd_mailbox *mailbox; u32 *inbox; int err = 0; #define MOD_STAT_CFG_IN_SIZE 0x100 #define MOD_STAT_CFG_PG_SZ_M_OFFSET 0x002 #define MOD_STAT_CFG_PG_SZ_OFFSET 0x003 mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); inbox = mailbox->buf; memset(inbox, 0, MOD_STAT_CFG_IN_SIZE); MLX4_PUT(inbox, cfg->log_pg_sz, MOD_STAT_CFG_PG_SZ_OFFSET); MLX4_PUT(inbox, cfg->log_pg_sz_m, MOD_STAT_CFG_PG_SZ_M_OFFSET); err = mlx4_cmd(dev, mailbox->dma, 0, 0, MLX4_CMD_MOD_STAT_CFG, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); mlx4_free_cmd_mailbox(dev, mailbox); return err; } +int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + u8 field, port; + u32 size; + int err = 0; + +#define QUERY_FUNC_CAP_FLAGS_OFFSET 0x0 +#define QUERY_FUNC_CAP_NUM_PORTS_OFFSET 0x1 +#define QUERY_FUNC_CAP_PF_BHVR_OFFSET 0x4 +#define QUERY_FUNC_CAP_FMR_OFFSET 0x8 +#define QUERY_FUNC_CAP_QP_QUOTA_OFFSET_DEP 0x10 +#define QUERY_FUNC_CAP_CQ_QUOTA_OFFSET_DEP 0x14 +#define QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET_DEP 0x18 +#define QUERY_FUNC_CAP_MPT_QUOTA_OFFSET_DEP 0x20 +#define QUERY_FUNC_CAP_MTT_QUOTA_OFFSET_DEP 0x24 +#define QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP 0x28 +#define QUERY_FUNC_CAP_MAX_EQ_OFFSET 0x2c +#define QUERY_FUNC_CAP_RESERVED_EQ_OFFSET 0x30 + +#define QUERY_FUNC_CAP_QP_QUOTA_OFFSET 0x50 +#define QUERY_FUNC_CAP_CQ_QUOTA_OFFSET 0x54 +#define QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET 0x58 +#define QUERY_FUNC_CAP_MPT_QUOTA_OFFSET 0x60 +#define QUERY_FUNC_CAP_MTT_QUOTA_OFFSET 0x64 +#define QUERY_FUNC_CAP_MCG_QUOTA_OFFSET 0x68 + +#define QUERY_FUNC_CAP_FMR_FLAG 0x80 +#define QUERY_FUNC_CAP_FLAG_RDMA 0x40 +#define QUERY_FUNC_CAP_FLAG_ETH 0x80 +#define QUERY_FUNC_CAP_FLAG_QUOTAS 0x10 + +/* when opcode modifier = 1 */ +#define QUERY_FUNC_CAP_PHYS_PORT_OFFSET 0x3 +#define QUERY_FUNC_CAP_FLAGS0_OFFSET 0x8 +#define QUERY_FUNC_CAP_FLAGS1_OFFSET 0xc +#define QUERY_FUNC_CAP_COUNTER_INDEX_OFFSET 0xd + +#define QUERY_FUNC_CAP_QP0_TUNNEL 0x10 +#define QUERY_FUNC_CAP_QP0_PROXY 0x14 +#define QUERY_FUNC_CAP_QP1_TUNNEL 0x18 +#define QUERY_FUNC_CAP_QP1_PROXY 0x1c + +#define QUERY_FUNC_CAP_ETH_PROPS_FORCE_MAC 0x40 +#define QUERY_FUNC_CAP_ETH_PROPS_FORCE_VLAN 0x80 +#define QUERY_FUNC_CAP_PROPS_DEF_COUNTER 0x20 + +#define QUERY_FUNC_CAP_RDMA_PROPS_FORCE_PHY_WQE_GID 0x80 + + if (vhcr->op_modifier == 1) { + port = vhcr->in_modifier; /* phys-port = logical-port */ + MLX4_PUT(outbox->buf, port, QUERY_FUNC_CAP_PHYS_PORT_OFFSET); + + field = 0; + /* ensure that phy_wqe_gid bit is not set */ + MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS0_OFFSET); + + /* ensure force vlan and force mac bits are not set + * and that default counter bit is set + */ + field = QUERY_FUNC_CAP_PROPS_DEF_COUNTER; /* def counter */ + MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS1_OFFSET); + + /* There is always default counter legal or sink counter */ + field = mlx4_get_default_counter_index(dev, slave, vhcr->in_modifier); + MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_COUNTER_INDEX_OFFSET); + + /* size is now the QP number */ + size = dev->phys_caps.base_tunnel_sqpn + 8 * slave + port - 1; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP0_TUNNEL); + + size += 2; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP1_TUNNEL); + + size = dev->phys_caps.base_proxy_sqpn + 8 * slave + port - 1; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP0_PROXY); + + size += 2; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP1_PROXY); + + } else if (vhcr->op_modifier == 0) { + /* enable rdma and ethernet interfaces, and new quota locations */ + field = (QUERY_FUNC_CAP_FLAG_ETH | QUERY_FUNC_CAP_FLAG_RDMA | + QUERY_FUNC_CAP_FLAG_QUOTAS); + MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS_OFFSET); + + field = dev->caps.num_ports; + MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_NUM_PORTS_OFFSET); + + size = dev->caps.function_caps; /* set PF behaviours */ + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_PF_BHVR_OFFSET); + + field = 0; /* protected FMR support not available as yet */ + MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FMR_OFFSET); + + size = priv->mfunc.master.res_tracker.res_alloc[RES_QP].quota[slave]; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP_QUOTA_OFFSET); + size = dev->caps.num_qps; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP_QUOTA_OFFSET_DEP); + + size = priv->mfunc.master.res_tracker.res_alloc[RES_SRQ].quota[slave]; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET); + size = dev->caps.num_srqs; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET_DEP); + + size = priv->mfunc.master.res_tracker.res_alloc[RES_CQ].quota[slave]; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET); + size = dev->caps.num_cqs; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET_DEP); + + size = dev->caps.num_eqs; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MAX_EQ_OFFSET); + + size = dev->caps.reserved_eqs; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET); + + size = priv->mfunc.master.res_tracker.res_alloc[RES_MPT].quota[slave]; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET); + size = dev->caps.num_mpts; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET_DEP); + + size = priv->mfunc.master.res_tracker.res_alloc[RES_MTT].quota[slave]; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET); + size = dev->caps.num_mtts; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET_DEP); + + size = dev->caps.num_mgms + dev->caps.num_amgms; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET); + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP); + + } else + err = -EINVAL; + + return err; +} + +int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u32 gen_or_port, + struct mlx4_func_cap *func_cap) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 *outbox; + u8 field, op_modifier; + u32 size; + int err = 0, quotas = 0; + + op_modifier = !!gen_or_port; /* 0 = general, 1 = logical port */ + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + err = mlx4_cmd_box(dev, 0, mailbox->dma, gen_or_port, op_modifier, + MLX4_CMD_QUERY_FUNC_CAP, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + goto out; + + outbox = mailbox->buf; + + if (!op_modifier) { + MLX4_GET(field, outbox, QUERY_FUNC_CAP_FLAGS_OFFSET); + if (!(field & (QUERY_FUNC_CAP_FLAG_ETH | QUERY_FUNC_CAP_FLAG_RDMA))) { + mlx4_err(dev, "The host supports neither eth nor rdma interfaces\n"); + err = -EPROTONOSUPPORT; + goto out; + } + func_cap->flags = field; + quotas = !!(func_cap->flags & QUERY_FUNC_CAP_FLAG_QUOTAS); + + MLX4_GET(field, outbox, QUERY_FUNC_CAP_NUM_PORTS_OFFSET); + func_cap->num_ports = field; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_PF_BHVR_OFFSET); + func_cap->pf_context_behaviour = size; + + if (quotas) { + MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP_QUOTA_OFFSET); + func_cap->qp_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET); + func_cap->srq_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET); + func_cap->cq_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET); + func_cap->mpt_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET); + func_cap->mtt_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET); + func_cap->mcg_quota = size & 0xFFFFFF; + + } else { + MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP_QUOTA_OFFSET_DEP); + func_cap->qp_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET_DEP); + func_cap->srq_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET_DEP); + func_cap->cq_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET_DEP); + func_cap->mpt_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET_DEP); + func_cap->mtt_quota = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP); + func_cap->mcg_quota = size & 0xFFFFFF; + } + MLX4_GET(size, outbox, QUERY_FUNC_CAP_MAX_EQ_OFFSET); + func_cap->max_eq = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET); + func_cap->reserved_eq = size & 0xFFFFFF; + + goto out; + } + + /* logical port query */ + if (gen_or_port > dev->caps.num_ports) { + err = -EINVAL; + goto out; + } + + if (dev->caps.port_type[gen_or_port] == MLX4_PORT_TYPE_ETH) { + MLX4_GET(field, outbox, QUERY_FUNC_CAP_FLAGS1_OFFSET); + if (field & QUERY_FUNC_CAP_ETH_PROPS_FORCE_VLAN) { + mlx4_err(dev, "VLAN is enforced on this port\n"); + err = -EPROTONOSUPPORT; + goto out; + } + + if (field & QUERY_FUNC_CAP_ETH_PROPS_FORCE_MAC) { + mlx4_err(dev, "Force mac is enabled on this port\n"); + err = -EPROTONOSUPPORT; + goto out; + } + } else if (dev->caps.port_type[gen_or_port] == MLX4_PORT_TYPE_IB) { + MLX4_GET(field, outbox, QUERY_FUNC_CAP_FLAGS0_OFFSET); + if (field & QUERY_FUNC_CAP_RDMA_PROPS_FORCE_PHY_WQE_GID) { + mlx4_err(dev, "phy_wqe_gid is " + "enforced on this ib port\n"); + err = -EPROTONOSUPPORT; + goto out; + } + } + + MLX4_GET(field, outbox, QUERY_FUNC_CAP_PHYS_PORT_OFFSET); + func_cap->physical_port = field; + if (func_cap->physical_port != gen_or_port) { + err = -ENOSYS; + goto out; + } + + MLX4_GET(field, outbox, QUERY_FUNC_CAP_FLAGS1_OFFSET); + if (field & QUERY_FUNC_CAP_PROPS_DEF_COUNTER) { + MLX4_GET(field, outbox, QUERY_FUNC_CAP_COUNTER_INDEX_OFFSET); + func_cap->def_counter_index = field; + } else { + func_cap->def_counter_index = MLX4_SINK_COUNTER_INDEX; + } + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP0_TUNNEL); + func_cap->qp0_tunnel_qpn = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP0_PROXY); + func_cap->qp0_proxy_qpn = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP1_TUNNEL); + func_cap->qp1_tunnel_qpn = size & 0xFFFFFF; + + MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP1_PROXY); + func_cap->qp1_proxy_qpn = size & 0xFFFFFF; + + /* All other resources are allocated by the master, but we still report + * 'num' and 'reserved' capabilities as follows: + * - num remains the maximum resource index + * - 'num - reserved' is the total available objects of a resource, but + * resource indices may be less than 'reserved' + * TODO: set per-resource quotas */ + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + + return err; +} + int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) { struct mlx4_cmd_mailbox *mailbox; u32 *outbox; u8 field; - u32 field32; + u32 field32, flags, ext_flags; u16 size; u16 stat_rate; int err; int i; - u32 in_modifier; - u64 out_param; - u32 tmp1, tmp2; #define QUERY_DEV_CAP_OUT_SIZE 0x100 #define QUERY_DEV_CAP_MAX_SRQ_SZ_OFFSET 0x10 #define QUERY_DEV_CAP_MAX_QP_SZ_OFFSET 0x11 #define QUERY_DEV_CAP_RSVD_QP_OFFSET 0x12 #define QUERY_DEV_CAP_MAX_QP_OFFSET 0x13 #define QUERY_DEV_CAP_RSVD_SRQ_OFFSET 0x14 #define QUERY_DEV_CAP_MAX_SRQ_OFFSET 0x15 #define QUERY_DEV_CAP_RSVD_EEC_OFFSET 0x16 #define QUERY_DEV_CAP_MAX_EEC_OFFSET 0x17 #define QUERY_DEV_CAP_MAX_CQ_SZ_OFFSET 0x19 #define QUERY_DEV_CAP_RSVD_CQ_OFFSET 0x1a #define QUERY_DEV_CAP_MAX_CQ_OFFSET 0x1b #define QUERY_DEV_CAP_MAX_MPT_OFFSET 0x1d #define QUERY_DEV_CAP_RSVD_EQ_OFFSET 0x1e #define QUERY_DEV_CAP_MAX_EQ_OFFSET 0x1f #define QUERY_DEV_CAP_RSVD_MTT_OFFSET 0x20 #define QUERY_DEV_CAP_MAX_MRW_SZ_OFFSET 0x21 #define QUERY_DEV_CAP_RSVD_MRW_OFFSET 0x22 #define QUERY_DEV_CAP_MAX_MTT_SEG_OFFSET 0x23 #define QUERY_DEV_CAP_MAX_AV_OFFSET 0x27 #define QUERY_DEV_CAP_MAX_REQ_QP_OFFSET 0x29 #define QUERY_DEV_CAP_MAX_RES_QP_OFFSET 0x2b #define QUERY_DEV_CAP_MAX_GSO_OFFSET 0x2d +#define QUERY_DEV_CAP_RSS_OFFSET 0x2e #define QUERY_DEV_CAP_MAX_RDMA_OFFSET 0x2f -#define QUERY_DEV_CAP_STAT_CFG_INL_OFFSET 0x31 #define QUERY_DEV_CAP_RSZ_SRQ_OFFSET 0x33 #define QUERY_DEV_CAP_ACK_DELAY_OFFSET 0x35 #define QUERY_DEV_CAP_MTU_WIDTH_OFFSET 0x36 #define QUERY_DEV_CAP_VL_PORT_OFFSET 0x37 #define QUERY_DEV_CAP_MAX_MSG_SZ_OFFSET 0x38 #define QUERY_DEV_CAP_MAX_GID_OFFSET 0x3b #define QUERY_DEV_CAP_RATE_SUPPORT_OFFSET 0x3c +#define QUERY_DEV_CAP_CQ_TS_SUPPORT_OFFSET 0x3e #define QUERY_DEV_CAP_MAX_PKEY_OFFSET 0x3f #define QUERY_DEV_CAP_EXT_FLAGS_OFFSET 0x40 -#define QUERY_DEV_CAP_UDP_RSS_OFFSET 0x42 -#define QUERY_DEV_CAP_ETH_UC_LOOPBACK_OFFSET 0x43 +#define QUERY_DEV_CAP_SYNC_QP_OFFSET 0x42 #define QUERY_DEV_CAP_FLAGS_OFFSET 0x44 #define QUERY_DEV_CAP_RSVD_UAR_OFFSET 0x48 #define QUERY_DEV_CAP_UAR_SZ_OFFSET 0x49 #define QUERY_DEV_CAP_PAGE_SZ_OFFSET 0x4b #define QUERY_DEV_CAP_BF_OFFSET 0x4c #define QUERY_DEV_CAP_LOG_BF_REG_SZ_OFFSET 0x4d #define QUERY_DEV_CAP_LOG_MAX_BF_REGS_PER_PAGE_OFFSET 0x4e #define QUERY_DEV_CAP_LOG_MAX_BF_PAGES_OFFSET 0x4f #define QUERY_DEV_CAP_MAX_SG_SQ_OFFSET 0x51 #define QUERY_DEV_CAP_MAX_DESC_SZ_SQ_OFFSET 0x52 #define QUERY_DEV_CAP_MAX_SG_RQ_OFFSET 0x55 #define QUERY_DEV_CAP_MAX_DESC_SZ_RQ_OFFSET 0x56 #define QUERY_DEV_CAP_MAX_QP_MCG_OFFSET 0x61 #define QUERY_DEV_CAP_RSVD_MCG_OFFSET 0x62 #define QUERY_DEV_CAP_MAX_MCG_OFFSET 0x63 #define QUERY_DEV_CAP_RSVD_PD_OFFSET 0x64 #define QUERY_DEV_CAP_MAX_PD_OFFSET 0x65 #define QUERY_DEV_CAP_RSVD_XRC_OFFSET 0x66 #define QUERY_DEV_CAP_MAX_XRC_OFFSET 0x67 +#define QUERY_DEV_CAP_MAX_BASIC_COUNTERS_OFFSET 0x68 +#define QUERY_DEV_CAP_MAX_EXTENDED_COUNTERS_OFFSET 0x6c +#define QUERY_DEV_CAP_PORT_FLOWSTATS_COUNTERS_OFFSET 0x70 +#define QUERY_DEV_CAP_FLOW_STEERING_RANGE_EN_OFFSET 0x76 +#define QUERY_DEV_CAP_EXT_2_FLAGS_OFFSET 0x70 +#define QUERY_DEV_CAP_FLOW_STEERING_IPOIB_OFFSET 0x74 +#define QUERY_DEV_CAP_FLOW_STEERING_MAX_QP_OFFSET 0x77 #define QUERY_DEV_CAP_RDMARC_ENTRY_SZ_OFFSET 0x80 #define QUERY_DEV_CAP_QPC_ENTRY_SZ_OFFSET 0x82 #define QUERY_DEV_CAP_AUX_ENTRY_SZ_OFFSET 0x84 #define QUERY_DEV_CAP_ALTC_ENTRY_SZ_OFFSET 0x86 #define QUERY_DEV_CAP_EQC_ENTRY_SZ_OFFSET 0x88 #define QUERY_DEV_CAP_CQC_ENTRY_SZ_OFFSET 0x8a #define QUERY_DEV_CAP_SRQ_ENTRY_SZ_OFFSET 0x8c #define QUERY_DEV_CAP_C_MPT_ENTRY_SZ_OFFSET 0x8e #define QUERY_DEV_CAP_MTT_ENTRY_SZ_OFFSET 0x90 #define QUERY_DEV_CAP_D_MPT_ENTRY_SZ_OFFSET 0x92 #define QUERY_DEV_CAP_BMME_FLAGS_OFFSET 0x94 #define QUERY_DEV_CAP_RSVD_LKEY_OFFSET 0x98 +#define QUERY_DEV_CAP_ETS_CFG_OFFSET 0x9c #define QUERY_DEV_CAP_MAX_ICM_SZ_OFFSET 0xa0 -#define QUERY_DEV_CAP_MAX_BASIC_CNT_OFFSET 0x68 -#define QUERY_DEV_CAP_MAX_EXT_CNT_OFFSET 0x6c + dev_cap->flags2 = 0; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); outbox = mailbox->buf; err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_QUERY_DEV_CAP, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); if (err) goto out; MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_QP_OFFSET); dev_cap->reserved_qps = 1 << (field & 0xf); MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_QP_OFFSET); dev_cap->max_qps = 1 << (field & 0x1f); MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_SRQ_OFFSET); dev_cap->reserved_srqs = 1 << (field >> 4); MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_SRQ_OFFSET); dev_cap->max_srqs = 1 << (field & 0x1f); MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_CQ_SZ_OFFSET); dev_cap->max_cq_sz = 1 << field; MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_CQ_OFFSET); dev_cap->reserved_cqs = 1 << (field & 0xf); MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_CQ_OFFSET); dev_cap->max_cqs = 1 << (field & 0x1f); MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MPT_OFFSET); dev_cap->max_mpts = 1 << (field & 0x3f); MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_EQ_OFFSET); - dev_cap->reserved_eqs = 1 << (field & 0xf); + dev_cap->reserved_eqs = field & 0xf; MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_EQ_OFFSET); dev_cap->max_eqs = 1 << (field & 0xf); MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_MTT_OFFSET); dev_cap->reserved_mtts = 1 << (field >> 4); MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MRW_SZ_OFFSET); dev_cap->max_mrw_sz = 1 << field; MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_MRW_OFFSET); dev_cap->reserved_mrws = 1 << (field & 0xf); MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MTT_SEG_OFFSET); dev_cap->max_mtt_seg = 1 << (field & 0x3f); MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_REQ_QP_OFFSET); dev_cap->max_requester_per_qp = 1 << (field & 0x3f); MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_RES_QP_OFFSET); dev_cap->max_responder_per_qp = 1 << (field & 0x3f); MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_GSO_OFFSET); field &= 0x1f; if (!field) dev_cap->max_gso_sz = 0; else dev_cap->max_gso_sz = 1 << field; + MLX4_GET(field, outbox, QUERY_DEV_CAP_RSS_OFFSET); + if (field & 0x20) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_RSS_XOR; + if (field & 0x10) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_RSS_TOP; + field &= 0xf; + if (field) { + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_RSS; + dev_cap->max_rss_tbl_sz = 1 << field; + } else + dev_cap->max_rss_tbl_sz = 0; MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_RDMA_OFFSET); dev_cap->max_rdma_global = 1 << (field & 0x3f); MLX4_GET(field, outbox, QUERY_DEV_CAP_ACK_DELAY_OFFSET); dev_cap->local_ca_ack_delay = field & 0x1f; MLX4_GET(field, outbox, QUERY_DEV_CAP_VL_PORT_OFFSET); dev_cap->num_ports = field & 0xf; MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MSG_SZ_OFFSET); dev_cap->max_msg_sz = 1 << (field & 0x1f); + MLX4_GET(field, outbox, QUERY_DEV_CAP_PORT_FLOWSTATS_COUNTERS_OFFSET); + if (field & 0x10) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN; + MLX4_GET(field, outbox, QUERY_DEV_CAP_FLOW_STEERING_RANGE_EN_OFFSET); + if (field & 0x80) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_FS_EN; + MLX4_GET(field, outbox, QUERY_DEV_CAP_FLOW_STEERING_IPOIB_OFFSET); + if (field & 0x80) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_DMFS_IPOIB; + dev_cap->fs_log_max_ucast_qp_range_size = field & 0x1f; + MLX4_GET(field, outbox, QUERY_DEV_CAP_FLOW_STEERING_MAX_QP_OFFSET); + dev_cap->fs_max_num_qp_per_entry = field; MLX4_GET(stat_rate, outbox, QUERY_DEV_CAP_RATE_SUPPORT_OFFSET); dev_cap->stat_rate_support = stat_rate; - MLX4_GET(field, outbox, QUERY_DEV_CAP_UDP_RSS_OFFSET); - dev_cap->udp_rss = field & 0x1; - MLX4_GET(field, outbox, QUERY_DEV_CAP_ETH_UC_LOOPBACK_OFFSET); - dev_cap->loopback_support = field & 0x1; - dev_cap->wol = field & 0x40; - MLX4_GET(tmp1, outbox, QUERY_DEV_CAP_EXT_FLAGS_OFFSET); - MLX4_GET(tmp2, outbox, QUERY_DEV_CAP_FLAGS_OFFSET); - dev_cap->flags = tmp2 | (u64)tmp1 << 32; + MLX4_GET(field, outbox, QUERY_DEV_CAP_CQ_TS_SUPPORT_OFFSET); + if (field & 0x80) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_TS; + MLX4_GET(ext_flags, outbox, QUERY_DEV_CAP_EXT_FLAGS_OFFSET); + MLX4_GET(flags, outbox, QUERY_DEV_CAP_FLAGS_OFFSET); + dev_cap->flags = flags | (u64)ext_flags << 32; + MLX4_GET(field, outbox, QUERY_DEV_CAP_SYNC_QP_OFFSET); + dev_cap->sync_qp = field & 0x10; MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_UAR_OFFSET); dev_cap->reserved_uars = field >> 4; MLX4_GET(field, outbox, QUERY_DEV_CAP_UAR_SZ_OFFSET); dev_cap->uar_size = 1 << ((field & 0x3f) + 20); MLX4_GET(field, outbox, QUERY_DEV_CAP_PAGE_SZ_OFFSET); dev_cap->min_page_sz = 1 << field; MLX4_GET(field, outbox, QUERY_DEV_CAP_BF_OFFSET); if (field & 0x80) { MLX4_GET(field, outbox, QUERY_DEV_CAP_LOG_BF_REG_SZ_OFFSET); dev_cap->bf_reg_size = 1 << (field & 0x1f); MLX4_GET(field, outbox, QUERY_DEV_CAP_LOG_MAX_BF_REGS_PER_PAGE_OFFSET); - if ((1 << (field & 0x3f)) > (PAGE_SIZE / dev_cap->bf_reg_size)) { - mlx4_dbg(dev, "log blue flame is invalid (%d), forcing 3\n", field & 0x1f); + if ((1 << (field & 0x3f)) > (PAGE_SIZE / dev_cap->bf_reg_size)) field = 3; - } dev_cap->bf_regs_per_page = 1 << (field & 0x3f); mlx4_dbg(dev, "BlueFlame available (reg size %d, regs/page %d)\n", dev_cap->bf_reg_size, dev_cap->bf_regs_per_page); } else { dev_cap->bf_reg_size = 0; mlx4_dbg(dev, "BlueFlame not available\n"); } MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_SG_SQ_OFFSET); dev_cap->max_sq_sg = field; MLX4_GET(size, outbox, QUERY_DEV_CAP_MAX_DESC_SZ_SQ_OFFSET); dev_cap->max_sq_desc_sz = size; MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_QP_MCG_OFFSET); dev_cap->max_qp_per_mcg = 1 << field; MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_MCG_OFFSET); dev_cap->reserved_mgms = field & 0xf; MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MCG_OFFSET); dev_cap->max_mcgs = 1 << field; MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_PD_OFFSET); dev_cap->reserved_pds = field >> 4; MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_PD_OFFSET); dev_cap->max_pds = 1 << (field & 0x3f); - MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_XRC_OFFSET); dev_cap->reserved_xrcds = field >> 4; MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_XRC_OFFSET); dev_cap->max_xrcds = 1 << (field & 0x1f); MLX4_GET(size, outbox, QUERY_DEV_CAP_RDMARC_ENTRY_SZ_OFFSET); dev_cap->rdmarc_entry_sz = size; MLX4_GET(size, outbox, QUERY_DEV_CAP_QPC_ENTRY_SZ_OFFSET); dev_cap->qpc_entry_sz = size; MLX4_GET(size, outbox, QUERY_DEV_CAP_AUX_ENTRY_SZ_OFFSET); dev_cap->aux_entry_sz = size; MLX4_GET(size, outbox, QUERY_DEV_CAP_ALTC_ENTRY_SZ_OFFSET); dev_cap->altc_entry_sz = size; MLX4_GET(size, outbox, QUERY_DEV_CAP_EQC_ENTRY_SZ_OFFSET); dev_cap->eqc_entry_sz = size; MLX4_GET(size, outbox, QUERY_DEV_CAP_CQC_ENTRY_SZ_OFFSET); dev_cap->cqc_entry_sz = size; MLX4_GET(size, outbox, QUERY_DEV_CAP_SRQ_ENTRY_SZ_OFFSET); dev_cap->srq_entry_sz = size; MLX4_GET(size, outbox, QUERY_DEV_CAP_C_MPT_ENTRY_SZ_OFFSET); dev_cap->cmpt_entry_sz = size; MLX4_GET(size, outbox, QUERY_DEV_CAP_MTT_ENTRY_SZ_OFFSET); dev_cap->mtt_entry_sz = size; MLX4_GET(size, outbox, QUERY_DEV_CAP_D_MPT_ENTRY_SZ_OFFSET); dev_cap->dmpt_entry_sz = size; MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_SRQ_SZ_OFFSET); dev_cap->max_srq_sz = 1 << field; MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_QP_SZ_OFFSET); dev_cap->max_qp_sz = 1 << field; - MLX4_GET(field, outbox, QUERY_DEV_CAP_STAT_CFG_INL_OFFSET); - dev_cap->inline_cfg = field & 1; MLX4_GET(field, outbox, QUERY_DEV_CAP_RSZ_SRQ_OFFSET); dev_cap->resize_srq = field & 1; MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_SG_RQ_OFFSET); dev_cap->max_rq_sg = field; MLX4_GET(size, outbox, QUERY_DEV_CAP_MAX_DESC_SZ_RQ_OFFSET); dev_cap->max_rq_desc_sz = size; MLX4_GET(dev_cap->bmme_flags, outbox, QUERY_DEV_CAP_BMME_FLAGS_OFFSET); MLX4_GET(dev_cap->reserved_lkey, outbox, QUERY_DEV_CAP_RSVD_LKEY_OFFSET); + MLX4_GET(field32, outbox, QUERY_DEV_CAP_ETS_CFG_OFFSET); + if (field32 & (1 << 0)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_ETH_BACKPL_AN_REP; + if (field32 & (1 << 7)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT; + if (field32 & (1 << 8)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_DRIVER_VERSION_TO_FW; + if (field32 & (1 << 13)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_ETS_CFG; + MLX4_GET(dev_cap->max_icm_sz, outbox, QUERY_DEV_CAP_MAX_ICM_SZ_OFFSET); - MLX4_GET(dev_cap->max_basic_counters, outbox, - QUERY_DEV_CAP_MAX_BASIC_CNT_OFFSET); - MLX4_GET(dev_cap->max_ext_counters, outbox, - QUERY_DEV_CAP_MAX_EXT_CNT_OFFSET); + if (dev_cap->flags & MLX4_DEV_CAP_FLAG_COUNTERS) + MLX4_GET(dev_cap->max_basic_counters, outbox, + QUERY_DEV_CAP_MAX_BASIC_COUNTERS_OFFSET); + /* FW reports 256 however real value is 255 */ + dev_cap->max_basic_counters = min_t(u32, dev_cap->max_basic_counters, 255); + if (dev_cap->flags & MLX4_DEV_CAP_FLAG_COUNTERS_EXT) + MLX4_GET(dev_cap->max_extended_counters, outbox, + QUERY_DEV_CAP_MAX_EXTENDED_COUNTERS_OFFSET); + MLX4_GET(field32, outbox, QUERY_DEV_CAP_EXT_2_FLAGS_OFFSET); + if (field32 & (1 << 16)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_UPDATE_QP; + if (field32 & (1 << 19)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_LB_SRC_CHK; + if (field32 & (1 << 20)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_FSM; + if (field32 & (1 << 26)) + dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_VLAN_CONTROL; + if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { for (i = 1; i <= dev_cap->num_ports; ++i) { MLX4_GET(field, outbox, QUERY_DEV_CAP_VL_PORT_OFFSET); dev_cap->max_vl[i] = field >> 4; MLX4_GET(field, outbox, QUERY_DEV_CAP_MTU_WIDTH_OFFSET); dev_cap->ib_mtu[i] = field >> 4; dev_cap->max_port_width[i] = field & 0xf; MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_GID_OFFSET); dev_cap->max_gids[i] = 1 << (field & 0xf); MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_PKEY_OFFSET); dev_cap->max_pkeys[i] = 1 << (field & 0xf); } } else { #define QUERY_PORT_SUPPORTED_TYPE_OFFSET 0x00 #define QUERY_PORT_MTU_OFFSET 0x01 #define QUERY_PORT_ETH_MTU_OFFSET 0x02 #define QUERY_PORT_WIDTH_OFFSET 0x06 #define QUERY_PORT_MAX_GID_PKEY_OFFSET 0x07 #define QUERY_PORT_MAX_MACVLAN_OFFSET 0x0a #define QUERY_PORT_MAX_VL_OFFSET 0x0b #define QUERY_PORT_MAC_OFFSET 0x10 #define QUERY_PORT_TRANS_VENDOR_OFFSET 0x18 #define QUERY_PORT_WAVELENGTH_OFFSET 0x1c #define QUERY_PORT_TRANS_CODE_OFFSET 0x20 -#define STAT_CFG_PORT_MODE (1 << 28) -#define STAT_CFG_PORT_OFFSET 0x8 -#define STAT_CFG_PORT_MASK (1 << 20) -#define STAT_CFG_MOD_INLINE 0x3 - for (i = 1; i <= dev_cap->num_ports; ++i) { err = mlx4_cmd_box(dev, 0, mailbox->dma, i, 0, MLX4_CMD_QUERY_PORT, - MLX4_CMD_TIME_CLASS_B); + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); if (err) goto out; MLX4_GET(field, outbox, QUERY_PORT_SUPPORTED_TYPE_OFFSET); dev_cap->supported_port_types[i] = field & 3; + dev_cap->suggested_type[i] = (field >> 3) & 1; + dev_cap->default_sense[i] = (field >> 4) & 1; MLX4_GET(field, outbox, QUERY_PORT_MTU_OFFSET); dev_cap->ib_mtu[i] = field & 0xf; MLX4_GET(field, outbox, QUERY_PORT_WIDTH_OFFSET); dev_cap->max_port_width[i] = field & 0xf; MLX4_GET(field, outbox, QUERY_PORT_MAX_GID_PKEY_OFFSET); dev_cap->max_gids[i] = 1 << (field >> 4); dev_cap->max_pkeys[i] = 1 << (field & 0xf); MLX4_GET(field, outbox, QUERY_PORT_MAX_VL_OFFSET); dev_cap->max_vl[i] = field & 0xf; MLX4_GET(field, outbox, QUERY_PORT_MAX_MACVLAN_OFFSET); dev_cap->log_max_macs[i] = field & 0xf; dev_cap->log_max_vlans[i] = field >> 4; MLX4_GET(dev_cap->eth_mtu[i], outbox, QUERY_PORT_ETH_MTU_OFFSET); MLX4_GET(dev_cap->def_mac[i], outbox, QUERY_PORT_MAC_OFFSET); MLX4_GET(field32, outbox, QUERY_PORT_TRANS_VENDOR_OFFSET); dev_cap->trans_type[i] = field32 >> 24; dev_cap->vendor_oui[i] = field32 & 0xffffff; MLX4_GET(dev_cap->wavelength[i], outbox, QUERY_PORT_WAVELENGTH_OFFSET); MLX4_GET(dev_cap->trans_code[i], outbox, QUERY_PORT_TRANS_CODE_OFFSET); - - /* Query stat cfg for port enablement */ - if (dev_cap->inline_cfg) { - in_modifier = STAT_CFG_PORT_MODE | i << 8 | - STAT_CFG_PORT_OFFSET; - err = mlx4_cmd_imm(dev, 0, &out_param, - in_modifier, - STAT_CFG_MOD_INLINE, - MLX4_CMD_MOD_STAT_CFG, - MLX4_CMD_TIME_CLASS_B); - if (!err) - if (!(out_param & STAT_CFG_PORT_MASK)) - dev_cap->supported_port_types[i] = 0; - } } } mlx4_dbg(dev, "Base MM extensions: flags %08x, rsvd L_Key %08x\n", dev_cap->bmme_flags, dev_cap->reserved_lkey); /* * Each UAR has 4 EQ doorbells; so if a UAR is reserved, then * we can't use any EQs whose doorbell falls on that page, * even if the EQ itself isn't reserved. */ dev_cap->reserved_eqs = max(dev_cap->reserved_uars * 4, dev_cap->reserved_eqs); mlx4_dbg(dev, "Max ICM size %lld MB\n", (unsigned long long) dev_cap->max_icm_sz >> 20); mlx4_dbg(dev, "Max QPs: %d, reserved QPs: %d, entry size: %d\n", dev_cap->max_qps, dev_cap->reserved_qps, dev_cap->qpc_entry_sz); mlx4_dbg(dev, "Max SRQs: %d, reserved SRQs: %d, entry size: %d\n", dev_cap->max_srqs, dev_cap->reserved_srqs, dev_cap->srq_entry_sz); mlx4_dbg(dev, "Max CQs: %d, reserved CQs: %d, entry size: %d\n", dev_cap->max_cqs, dev_cap->reserved_cqs, dev_cap->cqc_entry_sz); mlx4_dbg(dev, "Max EQs: %d, reserved EQs: %d, entry size: %d\n", dev_cap->max_eqs, dev_cap->reserved_eqs, dev_cap->eqc_entry_sz); mlx4_dbg(dev, "reserved MPTs: %d, reserved MTTs: %d\n", dev_cap->reserved_mrws, dev_cap->reserved_mtts); mlx4_dbg(dev, "Max PDs: %d, reserved PDs: %d, reserved UARs: %d\n", dev_cap->max_pds, dev_cap->reserved_pds, dev_cap->reserved_uars); mlx4_dbg(dev, "Max QP/MCG: %d, reserved MGMs: %d\n", dev_cap->max_pds, dev_cap->reserved_mgms); mlx4_dbg(dev, "Max CQEs: %d, max WQEs: %d, max SRQ WQEs: %d\n", dev_cap->max_cq_sz, dev_cap->max_qp_sz, dev_cap->max_srq_sz); mlx4_dbg(dev, "Local CA ACK delay: %d, max MTU: %d, port width cap: %d\n", dev_cap->local_ca_ack_delay, 128 << dev_cap->ib_mtu[1], dev_cap->max_port_width[1]); mlx4_dbg(dev, "Max SQ desc size: %d, max SQ S/G: %d\n", dev_cap->max_sq_desc_sz, dev_cap->max_sq_sg); mlx4_dbg(dev, "Max RQ desc size: %d, max RQ S/G: %d\n", dev_cap->max_rq_desc_sz, dev_cap->max_rq_sg); mlx4_dbg(dev, "Max GSO size: %d\n", dev_cap->max_gso_sz); + mlx4_dbg(dev, "Max basic counters: %d\n", dev_cap->max_basic_counters); + mlx4_dbg(dev, "Max extended counters: %d\n", dev_cap->max_extended_counters); + mlx4_dbg(dev, "Max RSS Table size: %d\n", dev_cap->max_rss_tbl_sz); dump_dev_cap_flags(dev, dev_cap->flags); + dump_dev_cap_flags2(dev, dev_cap->flags2); out: mlx4_free_cmd_mailbox(dev, mailbox); return err; } +int mlx4_QUERY_DEV_CAP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + u64 flags; + int err = 0; + u8 field; + + err = mlx4_cmd_box(dev, 0, outbox->dma, 0, 0, MLX4_CMD_QUERY_DEV_CAP, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (err) + return err; + + /* add port mng change event capability unconditionally to slaves */ + MLX4_GET(flags, outbox->buf, QUERY_DEV_CAP_EXT_FLAGS_OFFSET); + flags |= MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV; + MLX4_PUT(outbox->buf, flags, QUERY_DEV_CAP_EXT_FLAGS_OFFSET); + + /* For guests, report Blueflame disabled */ + MLX4_GET(field, outbox->buf, QUERY_DEV_CAP_BF_OFFSET); + field &= 0x7f; + MLX4_PUT(outbox->buf, field, QUERY_DEV_CAP_BF_OFFSET); + + /* turn off device-managed steering capability if not enabled */ + if (dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED) { + MLX4_GET(field, outbox->buf, + QUERY_DEV_CAP_FLOW_STEERING_RANGE_EN_OFFSET); + field &= 0x7f; + MLX4_PUT(outbox->buf, field, + QUERY_DEV_CAP_FLOW_STEERING_RANGE_EN_OFFSET); + } + return 0; +} + +int mlx4_QUERY_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + u64 def_mac; + u8 port_type; + u16 short_field; + int err; + int admin_link_state; + +#define MLX4_VF_PORT_NO_LINK_SENSE_MASK 0xE0 +#define MLX4_PORT_LINK_UP_MASK 0x80 +#define QUERY_PORT_CUR_MAX_PKEY_OFFSET 0x0c +#define QUERY_PORT_CUR_MAX_GID_OFFSET 0x0e + + err = mlx4_cmd_box(dev, 0, outbox->dma, vhcr->in_modifier, 0, + MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + + if (!err && dev->caps.function != slave) { + /* set slave default_mac address to be zero MAC */ + def_mac = priv->mfunc.master.vf_oper[slave].vport[vhcr->in_modifier].state.mac; + MLX4_PUT(outbox->buf, def_mac, QUERY_PORT_MAC_OFFSET); + + /* get port type - currently only eth is enabled */ + MLX4_GET(port_type, outbox->buf, + QUERY_PORT_SUPPORTED_TYPE_OFFSET); + + /* No link sensing allowed */ + port_type &= MLX4_VF_PORT_NO_LINK_SENSE_MASK; + /* set port type to currently operating port type */ + port_type |= (dev->caps.port_type[vhcr->in_modifier] & 0x3); + + admin_link_state = priv->mfunc.master.vf_oper[slave].vport[vhcr->in_modifier].state.link_state; + if (IFLA_VF_LINK_STATE_ENABLE == admin_link_state) + port_type |= MLX4_PORT_LINK_UP_MASK; + else if (IFLA_VF_LINK_STATE_DISABLE == admin_link_state) + port_type &= ~MLX4_PORT_LINK_UP_MASK; + + MLX4_PUT(outbox->buf, port_type, + QUERY_PORT_SUPPORTED_TYPE_OFFSET); + + if (dev->caps.port_type[vhcr->in_modifier] == MLX4_PORT_TYPE_ETH) + short_field = mlx4_get_slave_num_gids(dev, slave); + else + short_field = 1; /* slave max gids */ + MLX4_PUT(outbox->buf, short_field, + QUERY_PORT_CUR_MAX_GID_OFFSET); + + short_field = dev->caps.pkey_table_len[vhcr->in_modifier]; + MLX4_PUT(outbox->buf, short_field, + QUERY_PORT_CUR_MAX_PKEY_OFFSET); + } + + return err; +} + +int mlx4_get_slave_pkey_gid_tbl_len(struct mlx4_dev *dev, u8 port, + int *gid_tbl_len, int *pkey_tbl_len) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 *outbox; + u16 field; + int err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + err = mlx4_cmd_box(dev, 0, mailbox->dma, port, 0, + MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); + if (err) + goto out; + + outbox = mailbox->buf; + + MLX4_GET(field, outbox, QUERY_PORT_CUR_MAX_GID_OFFSET); + *gid_tbl_len = field; + + MLX4_GET(field, outbox, QUERY_PORT_CUR_MAX_PKEY_OFFSET); + *pkey_tbl_len = field; + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_get_slave_pkey_gid_tbl_len); + int mlx4_map_cmd(struct mlx4_dev *dev, u16 op, struct mlx4_icm *icm, u64 virt) { struct mlx4_cmd_mailbox *mailbox; struct mlx4_icm_iter iter; __be64 *pages; int lg; int nent = 0; int i; int err = 0; int ts = 0, tc = 0; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); memset(mailbox->buf, 0, MLX4_MAILBOX_SIZE); pages = mailbox->buf; for (mlx4_icm_first(icm, &iter); !mlx4_icm_last(&iter); mlx4_icm_next(&iter)) { /* * We have to pass pages that are aligned to their * size, so find the least significant 1 in the * address or size and use that as our log2 size. */ lg = ffs(mlx4_icm_addr(&iter) | mlx4_icm_size(&iter)) - 1; if (lg < MLX4_ICM_PAGE_SHIFT) { mlx4_warn(dev, "Got FW area not aligned to %d (%llx/%lx).\n", MLX4_ICM_PAGE_SIZE, (unsigned long long) mlx4_icm_addr(&iter), mlx4_icm_size(&iter)); err = -EINVAL; goto out; } for (i = 0; i < mlx4_icm_size(&iter) >> lg; ++i) { if (virt != -1) { pages[nent * 2] = cpu_to_be64(virt); virt += 1 << lg; } pages[nent * 2 + 1] = cpu_to_be64((mlx4_icm_addr(&iter) + (i << lg)) | (lg - MLX4_ICM_PAGE_SHIFT)); ts += 1 << (lg - 10); ++tc; if (++nent == MLX4_MAILBOX_SIZE / 16) { err = mlx4_cmd(dev, mailbox->dma, nent, 0, op, - MLX4_CMD_TIME_CLASS_B); + MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); if (err) goto out; nent = 0; } } } if (nent) - err = mlx4_cmd(dev, mailbox->dma, nent, 0, op, MLX4_CMD_TIME_CLASS_B); + err = mlx4_cmd(dev, mailbox->dma, nent, 0, op, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); if (err) goto out; switch (op) { case MLX4_CMD_MAP_FA: mlx4_dbg(dev, "Mapped %d chunks/%d KB for FW.\n", tc, ts); break; case MLX4_CMD_MAP_ICM_AUX: mlx4_dbg(dev, "Mapped %d chunks/%d KB for ICM aux.\n", tc, ts); break; case MLX4_CMD_MAP_ICM: mlx4_dbg(dev, "Mapped %d chunks/%d KB at %llx for ICM.\n", tc, ts, (unsigned long long) virt - (ts << 10)); break; } out: mlx4_free_cmd_mailbox(dev, mailbox); return err; } int mlx4_MAP_FA(struct mlx4_dev *dev, struct mlx4_icm *icm) { return mlx4_map_cmd(dev, MLX4_CMD_MAP_FA, icm, -1); } int mlx4_UNMAP_FA(struct mlx4_dev *dev) { - return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_UNMAP_FA, MLX4_CMD_TIME_CLASS_B); + return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_UNMAP_FA, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); } int mlx4_RUN_FW(struct mlx4_dev *dev) { - return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_RUN_FW, MLX4_CMD_TIME_CLASS_A); + return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_RUN_FW, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); } int mlx4_QUERY_FW(struct mlx4_dev *dev) { struct mlx4_fw *fw = &mlx4_priv(dev)->fw; struct mlx4_cmd *cmd = &mlx4_priv(dev)->cmd; struct mlx4_cmd_mailbox *mailbox; u32 *outbox; int err = 0; u64 fw_ver; u16 cmd_if_rev; u8 lg; #define QUERY_FW_OUT_SIZE 0x100 #define QUERY_FW_VER_OFFSET 0x00 -#define MC_PROMISC_VER 0x2000702bcull +#define QUERY_FW_PPF_ID 0x09 #define QUERY_FW_CMD_IF_REV_OFFSET 0x0a #define QUERY_FW_MAX_CMD_OFFSET 0x0f #define QUERY_FW_ERR_START_OFFSET 0x30 #define QUERY_FW_ERR_SIZE_OFFSET 0x38 #define QUERY_FW_ERR_BAR_OFFSET 0x3c #define QUERY_FW_SIZE_OFFSET 0x00 #define QUERY_FW_CLR_INT_BASE_OFFSET 0x20 #define QUERY_FW_CLR_INT_BAR_OFFSET 0x28 +#define QUERY_FW_COMM_BASE_OFFSET 0x40 +#define QUERY_FW_COMM_BAR_OFFSET 0x48 + +#define QUERY_FW_CLOCK_OFFSET 0x50 +#define QUERY_FW_CLOCK_BAR 0x58 + mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); outbox = mailbox->buf; err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_QUERY_FW, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); if (err) goto out; MLX4_GET(fw_ver, outbox, QUERY_FW_VER_OFFSET); /* * FW subminor version is at more significant bits than minor * version, so swap here. */ dev->caps.fw_ver = (fw_ver & 0xffff00000000ull) | ((fw_ver & 0xffff0000ull) >> 16) | ((fw_ver & 0x0000ffffull) << 16); - if (dev->caps.fw_ver < MC_PROMISC_VER) - dev->caps.mc_promisc_mode = 2; - else - dev->caps.mc_promisc_mode = 1; + MLX4_GET(lg, outbox, QUERY_FW_PPF_ID); + dev->caps.function = lg; + + if (mlx4_is_slave(dev)) + goto out; + + MLX4_GET(cmd_if_rev, outbox, QUERY_FW_CMD_IF_REV_OFFSET); if (cmd_if_rev < MLX4_COMMAND_INTERFACE_MIN_REV || cmd_if_rev > MLX4_COMMAND_INTERFACE_MAX_REV) { mlx4_err(dev, "Installed FW has unsupported " "command interface revision %d.\n", cmd_if_rev); mlx4_err(dev, "(Installed FW version is %d.%d.%03d)\n", (int) (dev->caps.fw_ver >> 32), (int) (dev->caps.fw_ver >> 16) & 0xffff, (int) dev->caps.fw_ver & 0xffff); mlx4_err(dev, "This driver version supports only revisions %d to %d.\n", MLX4_COMMAND_INTERFACE_MIN_REV, MLX4_COMMAND_INTERFACE_MAX_REV); err = -ENODEV; goto out; } if (cmd_if_rev < MLX4_COMMAND_INTERFACE_NEW_PORT_CMDS) dev->flags |= MLX4_FLAG_OLD_PORT_CMDS; MLX4_GET(lg, outbox, QUERY_FW_MAX_CMD_OFFSET); cmd->max_cmds = 1 << lg; mlx4_dbg(dev, "FW version %d.%d.%03d (cmd intf rev %d), max commands %d\n", (int) (dev->caps.fw_ver >> 32), (int) (dev->caps.fw_ver >> 16) & 0xffff, (int) dev->caps.fw_ver & 0xffff, cmd_if_rev, cmd->max_cmds); MLX4_GET(fw->catas_offset, outbox, QUERY_FW_ERR_START_OFFSET); MLX4_GET(fw->catas_size, outbox, QUERY_FW_ERR_SIZE_OFFSET); MLX4_GET(fw->catas_bar, outbox, QUERY_FW_ERR_BAR_OFFSET); fw->catas_bar = (fw->catas_bar >> 6) * 2; mlx4_dbg(dev, "Catastrophic error buffer at 0x%llx, size 0x%x, BAR %d\n", (unsigned long long) fw->catas_offset, fw->catas_size, fw->catas_bar); MLX4_GET(fw->fw_pages, outbox, QUERY_FW_SIZE_OFFSET); MLX4_GET(fw->clr_int_base, outbox, QUERY_FW_CLR_INT_BASE_OFFSET); MLX4_GET(fw->clr_int_bar, outbox, QUERY_FW_CLR_INT_BAR_OFFSET); fw->clr_int_bar = (fw->clr_int_bar >> 6) * 2; + MLX4_GET(fw->comm_base, outbox, QUERY_FW_COMM_BASE_OFFSET); + MLX4_GET(fw->comm_bar, outbox, QUERY_FW_COMM_BAR_OFFSET); + fw->comm_bar = (fw->comm_bar >> 6) * 2; + mlx4_dbg(dev, "Communication vector bar:%d offset:0x%llx\n", + fw->comm_bar, (unsigned long long)fw->comm_base); mlx4_dbg(dev, "FW size %d KB\n", fw->fw_pages >> 2); + MLX4_GET(fw->clock_offset, outbox, QUERY_FW_CLOCK_OFFSET); + MLX4_GET(fw->clock_bar, outbox, QUERY_FW_CLOCK_BAR); + fw->clock_bar = (fw->clock_bar >> 6) * 2; + mlx4_dbg(dev, "Internal clock bar:%d offset:0x%llx\n", + fw->comm_bar, (unsigned long long)fw->comm_base); + /* * Round up number of system pages needed in case * MLX4_ICM_PAGE_SIZE < PAGE_SIZE. */ fw->fw_pages = ALIGN(fw->fw_pages, PAGE_SIZE / MLX4_ICM_PAGE_SIZE) >> (PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT); mlx4_dbg(dev, "Clear int @ %llx, BAR %d\n", (unsigned long long) fw->clr_int_base, fw->clr_int_bar); out: mlx4_free_cmd_mailbox(dev, mailbox); return err; } -static void get_board_id(void *vsd, char *board_id) +int mlx4_QUERY_FW_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) { + u8 *outbuf; + int err; + + outbuf = outbox->buf; + err = mlx4_cmd_box(dev, 0, outbox->dma, 0, 0, MLX4_CMD_QUERY_FW, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (err) + return err; + + /* for slaves, set pci PPF ID to invalid and zero out everything + * else except FW version */ + outbuf[0] = outbuf[1] = 0; + memset(&outbuf[8], 0, QUERY_FW_OUT_SIZE - 8); + outbuf[QUERY_FW_PPF_ID] = MLX4_INVALID_SLAVE_ID; + + return 0; +} + +static void get_board_id(void *vsd, char *board_id, char *vsdstr) +{ int i; #define VSD_OFFSET_SIG1 0x00 #define VSD_OFFSET_SIG2 0xde #define VSD_OFFSET_MLX_BOARD_ID 0xd0 #define VSD_OFFSET_TS_BOARD_ID 0x20 +#define VSD_LEN 0xd0 #define VSD_SIGNATURE_TOPSPIN 0x5ad + memset(vsdstr, 0, MLX4_VSD_LEN); + + for (i = 0; i < VSD_LEN / 4; i++) + ((u32 *)vsdstr)[i] = + swab32(*(u32 *)(vsd + i * 4)); + memset(board_id, 0, MLX4_BOARD_ID_LEN); if (be16_to_cpup(vsd + VSD_OFFSET_SIG1) == VSD_SIGNATURE_TOPSPIN && be16_to_cpup(vsd + VSD_OFFSET_SIG2) == VSD_SIGNATURE_TOPSPIN) { strlcpy(board_id, vsd + VSD_OFFSET_TS_BOARD_ID, MLX4_BOARD_ID_LEN); } else { /* * The board ID is a string but the firmware byte * swaps each 4-byte word before passing it back to * us. Therefore we need to swab it before printing. */ for (i = 0; i < 4; ++i) ((u32 *) board_id)[i] = swab32(*(u32 *) (vsd + VSD_OFFSET_MLX_BOARD_ID + i * 4)); } } int mlx4_QUERY_ADAPTER(struct mlx4_dev *dev, struct mlx4_adapter *adapter) { struct mlx4_cmd_mailbox *mailbox; u32 *outbox; int err; #define QUERY_ADAPTER_OUT_SIZE 0x100 #define QUERY_ADAPTER_INTA_PIN_OFFSET 0x10 #define QUERY_ADAPTER_VSD_OFFSET 0x20 +#define QUERY_ADAPTER_VSD_VENDOR_ID_OFFSET 0x1e mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); outbox = mailbox->buf; err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_QUERY_ADAPTER, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); if (err) goto out; MLX4_GET(adapter->inta_pin, outbox, QUERY_ADAPTER_INTA_PIN_OFFSET); + adapter->vsd_vendor_id = be16_to_cpup((u16 *)outbox + + QUERY_ADAPTER_VSD_VENDOR_ID_OFFSET / 2); + get_board_id(outbox + QUERY_ADAPTER_VSD_OFFSET / 4, - adapter->board_id); + adapter->board_id, adapter->vsd); out: mlx4_free_cmd_mailbox(dev, mailbox); return err; } int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param) { struct mlx4_cmd_mailbox *mailbox; __be32 *inbox; + u32 mw_enable; int err; #define INIT_HCA_IN_SIZE 0x200 +#define INIT_HCA_DRV_NAME_FOR_FW_MAX_SIZE 64 #define INIT_HCA_VERSION_OFFSET 0x000 #define INIT_HCA_VERSION 2 #define INIT_HCA_CACHELINE_SZ_OFFSET 0x0e -#define INIT_HCA_X86_64_BYTE_CACHELINE_SZ 0x40 #define INIT_HCA_FLAGS_OFFSET 0x014 +#define INIT_HCA_RECOVERABLE_ERROR_EVENT_OFFSET 0x018 #define INIT_HCA_QPC_OFFSET 0x020 #define INIT_HCA_QPC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x10) #define INIT_HCA_LOG_QP_OFFSET (INIT_HCA_QPC_OFFSET + 0x17) #define INIT_HCA_SRQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x28) #define INIT_HCA_LOG_SRQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x2f) #define INIT_HCA_CQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x30) #define INIT_HCA_LOG_CQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x37) +#define INIT_HCA_EQE_CQE_OFFSETS (INIT_HCA_QPC_OFFSET + 0x38) #define INIT_HCA_ALTC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x40) #define INIT_HCA_AUXC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x50) #define INIT_HCA_EQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x60) #define INIT_HCA_LOG_EQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x67) #define INIT_HCA_RDMARC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x70) #define INIT_HCA_LOG_RD_OFFSET (INIT_HCA_QPC_OFFSET + 0x77) #define INIT_HCA_MCAST_OFFSET 0x0c0 #define INIT_HCA_MC_BASE_OFFSET (INIT_HCA_MCAST_OFFSET + 0x00) #define INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x12) #define INIT_HCA_LOG_MC_HASH_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x16) +#define INIT_HCA_UC_STEERING_OFFSET (INIT_HCA_MCAST_OFFSET + 0x18) #define INIT_HCA_LOG_MC_TABLE_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x1b) +#define INIT_HCA_DEVICE_MANAGED_FLOW_STEERING_EN 0x6 +#define INIT_HCA_DRIVER_VERSION_OFFSET 0x140 +#define INIT_HCA_FS_PARAM_OFFSET 0x1d0 +#define INIT_HCA_FS_BASE_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x00) +#define INIT_HCA_FS_LOG_ENTRY_SZ_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x12) +#define INIT_HCA_FS_LOG_TABLE_SZ_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x1b) +#define INIT_HCA_FS_ETH_BITS_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x21) +#define INIT_HCA_FS_ETH_NUM_ADDRS_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x22) +#define INIT_HCA_FS_IB_BITS_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x25) +#define INIT_HCA_FS_IB_NUM_ADDRS_OFFSET (INIT_HCA_FS_PARAM_OFFSET + 0x26) #define INIT_HCA_TPT_OFFSET 0x0f0 #define INIT_HCA_DMPT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x00) +#define INIT_HCA_TPT_MW_OFFSET (INIT_HCA_TPT_OFFSET + 0x08) +#define INIT_HCA_TPT_MW_ENABLE (1 << 31) #define INIT_HCA_LOG_MPT_SZ_OFFSET (INIT_HCA_TPT_OFFSET + 0x0b) #define INIT_HCA_MTT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x10) #define INIT_HCA_CMPT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x18) #define INIT_HCA_UAR_OFFSET 0x120 #define INIT_HCA_LOG_UAR_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x0a) #define INIT_HCA_UAR_PAGE_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x0b) mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); inbox = mailbox->buf; memset(inbox, 0, INIT_HCA_IN_SIZE); *((u8 *) mailbox->buf + INIT_HCA_VERSION_OFFSET) = INIT_HCA_VERSION; -#if defined(__x86_64__) || defined(__PPC64__) - *((u8 *) mailbox->buf + INIT_HCA_CACHELINE_SZ_OFFSET) = INIT_HCA_X86_64_BYTE_CACHELINE_SZ; -#endif + *((u8 *) mailbox->buf + INIT_HCA_CACHELINE_SZ_OFFSET) = + ((ilog2(cache_line_size()) - 4) << 5) | (1 << 4); + #if defined(__LITTLE_ENDIAN) *(inbox + INIT_HCA_FLAGS_OFFSET / 4) &= ~cpu_to_be32(1 << 1); #elif defined(__BIG_ENDIAN) *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 1); #else #error Host endianness not defined #endif /* Check port for UD address vector: */ *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1); /* Enable IPoIB checksumming if we can: */ if (dev->caps.flags & MLX4_DEV_CAP_FLAG_IPOIB_CSUM) *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 3); /* Enable QoS support if module parameter set */ if (enable_qos) *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 2); - /* counters mode */ - *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= - cpu_to_be32(dev->caps.counters_mode << 4); + /* Enable fast drop performance optimization */ + if (dev->caps.fast_drop) + *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 7); + /* enable counters */ + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS) + *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 4); + + /* CX3 is capable of extending CQEs\EQEs from 32 to 64 bytes */ + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_64B_EQE) { + *(inbox + INIT_HCA_EQE_CQE_OFFSETS / 4) |= cpu_to_be32(1 << 29); + dev->caps.eqe_size = 64; + dev->caps.eqe_factor = 1; + } else { + dev->caps.eqe_size = 32; + dev->caps.eqe_factor = 0; + } + + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_64B_CQE) { + *(inbox + INIT_HCA_EQE_CQE_OFFSETS / 4) |= cpu_to_be32(1 << 30); + dev->caps.cqe_size = 64; + dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_64B_CQE; + } else { + dev->caps.cqe_size = 32; + } + + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT) + *(inbox + INIT_HCA_RECOVERABLE_ERROR_EVENT_OFFSET / 4) |= cpu_to_be32(1 << 31); + + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_DRIVER_VERSION_TO_FW) { + strncpy((u8 *)mailbox->buf + INIT_HCA_DRIVER_VERSION_OFFSET, + DRV_NAME_FOR_FW, + INIT_HCA_DRV_NAME_FOR_FW_MAX_SIZE - 1); + mlx4_dbg(dev, "Reporting Driver Version to FW: %s\n", + (u8 *)mailbox->buf + INIT_HCA_DRIVER_VERSION_OFFSET); + } + /* QPC/EEC/CQC/EQC/RDMARC attributes */ MLX4_PUT(inbox, param->qpc_base, INIT_HCA_QPC_BASE_OFFSET); MLX4_PUT(inbox, param->log_num_qps, INIT_HCA_LOG_QP_OFFSET); MLX4_PUT(inbox, param->srqc_base, INIT_HCA_SRQC_BASE_OFFSET); MLX4_PUT(inbox, param->log_num_srqs, INIT_HCA_LOG_SRQ_OFFSET); MLX4_PUT(inbox, param->cqc_base, INIT_HCA_CQC_BASE_OFFSET); MLX4_PUT(inbox, param->log_num_cqs, INIT_HCA_LOG_CQ_OFFSET); MLX4_PUT(inbox, param->altc_base, INIT_HCA_ALTC_BASE_OFFSET); MLX4_PUT(inbox, param->auxc_base, INIT_HCA_AUXC_BASE_OFFSET); MLX4_PUT(inbox, param->eqc_base, INIT_HCA_EQC_BASE_OFFSET); MLX4_PUT(inbox, param->log_num_eqs, INIT_HCA_LOG_EQ_OFFSET); MLX4_PUT(inbox, param->rdmarc_base, INIT_HCA_RDMARC_BASE_OFFSET); MLX4_PUT(inbox, param->log_rd_per_qp, INIT_HCA_LOG_RD_OFFSET); - /* multicast attributes */ + /* steering attributes */ + if (dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) { + *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= + cpu_to_be32(1 << + INIT_HCA_DEVICE_MANAGED_FLOW_STEERING_EN); - MLX4_PUT(inbox, param->mc_base, INIT_HCA_MC_BASE_OFFSET); - MLX4_PUT(inbox, param->log_mc_entry_sz, INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET); - MLX4_PUT(inbox, param->log_mc_hash_sz, INIT_HCA_LOG_MC_HASH_SZ_OFFSET); - MLX4_PUT(inbox, param->log_mc_table_sz, INIT_HCA_LOG_MC_TABLE_SZ_OFFSET); + MLX4_PUT(inbox, param->mc_base, INIT_HCA_FS_BASE_OFFSET); + MLX4_PUT(inbox, param->log_mc_entry_sz, + INIT_HCA_FS_LOG_ENTRY_SZ_OFFSET); + MLX4_PUT(inbox, param->log_mc_table_sz, + INIT_HCA_FS_LOG_TABLE_SZ_OFFSET); + /* Enable Ethernet flow steering + * with udp unicast and tcp unicast + */ + MLX4_PUT(inbox, (u8) (MLX4_FS_UDP_UC_EN | MLX4_FS_TCP_UC_EN), + INIT_HCA_FS_ETH_BITS_OFFSET); + MLX4_PUT(inbox, (u16) MLX4_FS_NUM_OF_L2_ADDR, + INIT_HCA_FS_ETH_NUM_ADDRS_OFFSET); + /* Enable IPoIB flow steering + * with udp unicast and tcp unicast + */ + MLX4_PUT(inbox, (u8) (MLX4_FS_UDP_UC_EN | MLX4_FS_TCP_UC_EN), + INIT_HCA_FS_IB_BITS_OFFSET); + MLX4_PUT(inbox, (u16) MLX4_FS_NUM_OF_L2_ADDR, + INIT_HCA_FS_IB_NUM_ADDRS_OFFSET); + } else { + MLX4_PUT(inbox, param->mc_base, INIT_HCA_MC_BASE_OFFSET); + MLX4_PUT(inbox, param->log_mc_entry_sz, + INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET); + MLX4_PUT(inbox, param->log_mc_hash_sz, + INIT_HCA_LOG_MC_HASH_SZ_OFFSET); + MLX4_PUT(inbox, param->log_mc_table_sz, + INIT_HCA_LOG_MC_TABLE_SZ_OFFSET); + if (dev->caps.steering_mode == MLX4_STEERING_MODE_B0) + MLX4_PUT(inbox, (u8) (1 << 3), + INIT_HCA_UC_STEERING_OFFSET); + } /* TPT attributes */ MLX4_PUT(inbox, param->dmpt_base, INIT_HCA_DMPT_BASE_OFFSET); + mw_enable = param->mw_enable ? INIT_HCA_TPT_MW_ENABLE : 0; + MLX4_PUT(inbox, mw_enable, INIT_HCA_TPT_MW_OFFSET); MLX4_PUT(inbox, param->log_mpt_sz, INIT_HCA_LOG_MPT_SZ_OFFSET); MLX4_PUT(inbox, param->mtt_base, INIT_HCA_MTT_BASE_OFFSET); MLX4_PUT(inbox, param->cmpt_base, INIT_HCA_CMPT_BASE_OFFSET); /* UAR attributes */ - MLX4_PUT(inbox, (u8) (PAGE_SHIFT - 12), INIT_HCA_UAR_PAGE_SZ_OFFSET); + MLX4_PUT(inbox, param->uar_page_sz, INIT_HCA_UAR_PAGE_SZ_OFFSET); MLX4_PUT(inbox, param->log_uar_sz, INIT_HCA_LOG_UAR_SZ_OFFSET); - if (!mlx4_pre_t11_mode && dev->caps.flags & (u32) MLX4_DEV_CAP_FLAG_FC_T11) - *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 10); + err = mlx4_cmd(dev, mailbox->dma, 0, 0, MLX4_CMD_INIT_HCA, 10000, + MLX4_CMD_NATIVE); - err = mlx4_cmd(dev, mailbox->dma, 0, 0, MLX4_CMD_INIT_HCA, 10000); - if (err) mlx4_err(dev, "INIT_HCA returns %d\n", err); mlx4_free_cmd_mailbox(dev, mailbox); return err; } +int mlx4_QUERY_HCA(struct mlx4_dev *dev, + struct mlx4_init_hca_param *param) +{ + struct mlx4_cmd_mailbox *mailbox; + __be32 *outbox; + u32 dword_field; + u32 mw_enable; + int err; + u8 byte_field; + +#define QUERY_HCA_GLOBAL_CAPS_OFFSET 0x04 +#define QUERY_HCA_CORE_CLOCK_OFFSET 0x0c + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + outbox = mailbox->buf; + + err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, + MLX4_CMD_QUERY_HCA, + MLX4_CMD_TIME_CLASS_B, + !mlx4_is_slave(dev)); + if (err) + goto out; + + MLX4_GET(param->global_caps, outbox, QUERY_HCA_GLOBAL_CAPS_OFFSET); + MLX4_GET(param->hca_core_clock, outbox, QUERY_HCA_CORE_CLOCK_OFFSET); + + /* QPC/EEC/CQC/EQC/RDMARC attributes */ + + MLX4_GET(param->qpc_base, outbox, INIT_HCA_QPC_BASE_OFFSET); + MLX4_GET(param->log_num_qps, outbox, INIT_HCA_LOG_QP_OFFSET); + MLX4_GET(param->srqc_base, outbox, INIT_HCA_SRQC_BASE_OFFSET); + MLX4_GET(param->log_num_srqs, outbox, INIT_HCA_LOG_SRQ_OFFSET); + MLX4_GET(param->cqc_base, outbox, INIT_HCA_CQC_BASE_OFFSET); + MLX4_GET(param->log_num_cqs, outbox, INIT_HCA_LOG_CQ_OFFSET); + MLX4_GET(param->altc_base, outbox, INIT_HCA_ALTC_BASE_OFFSET); + MLX4_GET(param->auxc_base, outbox, INIT_HCA_AUXC_BASE_OFFSET); + MLX4_GET(param->eqc_base, outbox, INIT_HCA_EQC_BASE_OFFSET); + MLX4_GET(param->log_num_eqs, outbox, INIT_HCA_LOG_EQ_OFFSET); + MLX4_GET(param->rdmarc_base, outbox, INIT_HCA_RDMARC_BASE_OFFSET); + MLX4_GET(param->log_rd_per_qp, outbox, INIT_HCA_LOG_RD_OFFSET); + + MLX4_GET(dword_field, outbox, INIT_HCA_FLAGS_OFFSET); + if (dword_field & (1 << INIT_HCA_DEVICE_MANAGED_FLOW_STEERING_EN)) { + param->steering_mode = MLX4_STEERING_MODE_DEVICE_MANAGED; + } else { + MLX4_GET(byte_field, outbox, INIT_HCA_UC_STEERING_OFFSET); + if (byte_field & 0x8) + param->steering_mode = MLX4_STEERING_MODE_B0; + else + param->steering_mode = MLX4_STEERING_MODE_A0; + } + /* steering attributes */ + if (param->steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) { + MLX4_GET(param->mc_base, outbox, INIT_HCA_FS_BASE_OFFSET); + MLX4_GET(param->log_mc_entry_sz, outbox, + INIT_HCA_FS_LOG_ENTRY_SZ_OFFSET); + MLX4_GET(param->log_mc_table_sz, outbox, + INIT_HCA_FS_LOG_TABLE_SZ_OFFSET); + } else { + MLX4_GET(param->mc_base, outbox, INIT_HCA_MC_BASE_OFFSET); + MLX4_GET(param->log_mc_entry_sz, outbox, + INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET); + MLX4_GET(param->log_mc_hash_sz, outbox, + INIT_HCA_LOG_MC_HASH_SZ_OFFSET); + MLX4_GET(param->log_mc_table_sz, outbox, + INIT_HCA_LOG_MC_TABLE_SZ_OFFSET); + } + + /* CX3 is capable of extending CQEs\EQEs from 32 to 64 bytes */ + MLX4_GET(byte_field, outbox, INIT_HCA_EQE_CQE_OFFSETS); + if (byte_field & 0x20) /* 64-bytes eqe enabled */ + param->dev_cap_enabled |= MLX4_DEV_CAP_64B_EQE_ENABLED; + if (byte_field & 0x40) /* 64-bytes cqe enabled */ + param->dev_cap_enabled |= MLX4_DEV_CAP_64B_CQE_ENABLED; + + /* TPT attributes */ + + MLX4_GET(param->dmpt_base, outbox, INIT_HCA_DMPT_BASE_OFFSET); + MLX4_GET(mw_enable, outbox, INIT_HCA_TPT_MW_OFFSET); + param->mw_enable = (mw_enable & INIT_HCA_TPT_MW_ENABLE) == + INIT_HCA_TPT_MW_ENABLE; + MLX4_GET(param->log_mpt_sz, outbox, INIT_HCA_LOG_MPT_SZ_OFFSET); + MLX4_GET(param->mtt_base, outbox, INIT_HCA_MTT_BASE_OFFSET); + MLX4_GET(param->cmpt_base, outbox, INIT_HCA_CMPT_BASE_OFFSET); + + /* UAR attributes */ + + MLX4_GET(param->uar_page_sz, outbox, INIT_HCA_UAR_PAGE_SZ_OFFSET); + MLX4_GET(param->log_uar_sz, outbox, INIT_HCA_LOG_UAR_SZ_OFFSET); + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + + return err; +} + +/* for IB-type ports only in SRIOV mode. Checks that both proxy QP0 + * and real QP0 are active, so that the paravirtualized QP0 is ready + * to operate */ +static int check_qp0_state(struct mlx4_dev *dev, int function, int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + /* irrelevant if not infiniband */ + if (priv->mfunc.master.qp0_state[port].proxy_qp0_active && + priv->mfunc.master.qp0_state[port].qp0_active) + return 1; + return 0; +} + +int mlx4_INIT_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int port = vhcr->in_modifier; + int err; + + if (priv->mfunc.master.slave_state[slave].init_port_mask & (1 << port)) + return 0; + + if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB) { + /* Enable port only if it was previously disabled */ + if (!priv->mfunc.master.init_port_ref[port]) { + err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_INIT_PORT, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (err) + return err; + } + priv->mfunc.master.slave_state[slave].init_port_mask |= (1 << port); + } else { + if (slave == mlx4_master_func_num(dev)) { + if (check_qp0_state(dev, slave, port) && + !priv->mfunc.master.qp0_state[port].port_active) { + err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_INIT_PORT, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (err) + return err; + priv->mfunc.master.qp0_state[port].port_active = 1; + priv->mfunc.master.slave_state[slave].init_port_mask |= (1 << port); + } + } else + priv->mfunc.master.slave_state[slave].init_port_mask |= (1 << port); + } + ++priv->mfunc.master.init_port_ref[port]; + return 0; +} + int mlx4_INIT_PORT(struct mlx4_dev *dev, int port) { struct mlx4_cmd_mailbox *mailbox; u32 *inbox; int err; u32 flags; u16 field; if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { #define INIT_PORT_IN_SIZE 256 #define INIT_PORT_FLAGS_OFFSET 0x00 #define INIT_PORT_FLAG_SIG (1 << 18) #define INIT_PORT_FLAG_NG (1 << 17) #define INIT_PORT_FLAG_G0 (1 << 16) #define INIT_PORT_VL_SHIFT 4 #define INIT_PORT_PORT_WIDTH_SHIFT 8 #define INIT_PORT_MTU_OFFSET 0x04 #define INIT_PORT_MAX_GID_OFFSET 0x06 #define INIT_PORT_MAX_PKEY_OFFSET 0x0a #define INIT_PORT_GUID0_OFFSET 0x10 #define INIT_PORT_NODE_GUID_OFFSET 0x18 #define INIT_PORT_SI_GUID_OFFSET 0x20 mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); inbox = mailbox->buf; memset(inbox, 0, INIT_PORT_IN_SIZE); flags = 0; flags |= (dev->caps.vl_cap[port] & 0xf) << INIT_PORT_VL_SHIFT; flags |= (dev->caps.port_width_cap[port] & 0xf) << INIT_PORT_PORT_WIDTH_SHIFT; MLX4_PUT(inbox, flags, INIT_PORT_FLAGS_OFFSET); field = 128 << dev->caps.ib_mtu_cap[port]; MLX4_PUT(inbox, field, INIT_PORT_MTU_OFFSET); field = dev->caps.gid_table_len[port]; MLX4_PUT(inbox, field, INIT_PORT_MAX_GID_OFFSET); field = dev->caps.pkey_table_len[port]; MLX4_PUT(inbox, field, INIT_PORT_MAX_PKEY_OFFSET); err = mlx4_cmd(dev, mailbox->dma, port, 0, MLX4_CMD_INIT_PORT, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); mlx4_free_cmd_mailbox(dev, mailbox); } else err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_INIT_PORT, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); return err; } EXPORT_SYMBOL_GPL(mlx4_INIT_PORT); +int mlx4_CLOSE_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int port = vhcr->in_modifier; + int err; + + if (!(priv->mfunc.master.slave_state[slave].init_port_mask & + (1 << port))) + return 0; + + if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB) { + if (priv->mfunc.master.init_port_ref[port] == 1) { + err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, + 1000, MLX4_CMD_NATIVE); + if (err) + return err; + } + priv->mfunc.master.slave_state[slave].init_port_mask &= ~(1 << port); + } else { + /* infiniband port */ + if (slave == mlx4_master_func_num(dev)) { + if (!priv->mfunc.master.qp0_state[port].qp0_active && + priv->mfunc.master.qp0_state[port].port_active) { + err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, + 1000, MLX4_CMD_NATIVE); + if (err) + return err; + priv->mfunc.master.slave_state[slave].init_port_mask &= ~(1 << port); + priv->mfunc.master.qp0_state[port].port_active = 0; + } + } else + priv->mfunc.master.slave_state[slave].init_port_mask &= ~(1 << port); + } + --priv->mfunc.master.init_port_ref[port]; + return 0; +} + int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port) { - return mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, 1000); + return mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, 1000, + MLX4_CMD_WRAPPED); } EXPORT_SYMBOL_GPL(mlx4_CLOSE_PORT); int mlx4_CLOSE_HCA(struct mlx4_dev *dev, int panic) { - return mlx4_cmd(dev, 0, 0, panic, MLX4_CMD_CLOSE_HCA, 1000); + return mlx4_cmd(dev, 0, 0, panic, MLX4_CMD_CLOSE_HCA, 1000, + MLX4_CMD_NATIVE); } int mlx4_SET_ICM_SIZE(struct mlx4_dev *dev, u64 icm_size, u64 *aux_pages) { int ret = mlx4_cmd_imm(dev, icm_size, aux_pages, 0, 0, MLX4_CMD_SET_ICM_SIZE, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); if (ret) return ret; /* * Round up number of system pages needed in case * MLX4_ICM_PAGE_SIZE < PAGE_SIZE. */ *aux_pages = ALIGN(*aux_pages, PAGE_SIZE / MLX4_ICM_PAGE_SIZE) >> (PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT); return 0; } int mlx4_NOP(struct mlx4_dev *dev) { /* Input modifier of 0x1f means "finish as soon as possible." */ - return mlx4_cmd(dev, 0, 0x1f, 0, MLX4_CMD_NOP, 100); + return mlx4_cmd(dev, 0, 0x1f, 0, MLX4_CMD_NOP, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); } +int mlx4_query_diag_counters(struct mlx4_dev *dev, int array_length, + u8 op_modifier, u32 in_offset[], + u32 counter_out[]) +{ + struct mlx4_cmd_mailbox *mailbox; + u32 *outbox; + int ret; + int i; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + outbox = mailbox->buf; + + ret = mlx4_cmd_box(dev, 0, mailbox->dma, 0, op_modifier, + MLX4_CMD_DIAG_RPRT, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (ret) + goto out; + + for (i = 0; i < array_length; i++) { + if (in_offset[i] > MLX4_MAILBOX_SIZE) { + ret = -EINVAL; + goto out; + } + + MLX4_GET(counter_out[i], outbox, in_offset[i]); + } + +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return ret; +} +EXPORT_SYMBOL_GPL(mlx4_query_diag_counters); + +int mlx4_MOD_STAT_CFG_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + return -EPERM; +} + #define MLX4_WOL_SETUP_MODE (5 << 28) int mlx4_wol_read(struct mlx4_dev *dev, u64 *config, int port) { u32 in_mod = MLX4_WOL_SETUP_MODE | port << 8; return mlx4_cmd_imm(dev, 0, config, in_mod, 0x3, - MLX4_CMD_MOD_STAT_CFG, MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_MOD_STAT_CFG, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); } EXPORT_SYMBOL_GPL(mlx4_wol_read); int mlx4_wol_write(struct mlx4_dev *dev, u64 config, int port) { u32 in_mod = MLX4_WOL_SETUP_MODE | port << 8; return mlx4_cmd(dev, config, in_mod, 0x1, MLX4_CMD_MOD_STAT_CFG, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); } EXPORT_SYMBOL_GPL(mlx4_wol_write); -int mlx4_query_diag_counters(struct mlx4_dev *dev, int array_length, - u8 op_modifier, u32 in_offset[], u32 counter_out[]) +enum { + ADD_TO_MCG = 0x26, +}; + + +void mlx4_opreq_action(struct work_struct *work) { + struct mlx4_priv *priv = container_of(work, struct mlx4_priv, opreq_task); + struct mlx4_dev *dev = &priv->dev; + int num_tasks = atomic_read(&priv->opreq_count); struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mgm *mgm; u32 *outbox; - int ret; + u32 modifier; + u16 token; + u16 type_m; + u16 type; + int err; + u32 num_qps; + struct mlx4_qp qp; int i; + u8 rem_mcg; + u8 prot; +#define GET_OP_REQ_MODIFIER_OFFSET 0x08 +#define GET_OP_REQ_TOKEN_OFFSET 0x14 +#define GET_OP_REQ_TYPE_OFFSET 0x1a +#define GET_OP_REQ_DATA_OFFSET 0x20 + mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); + if (IS_ERR(mailbox)) { + mlx4_err(dev, "Failed to allocate mailbox for GET_OP_REQ\n"); + return; + } outbox = mailbox->buf; - ret = mlx4_cmd_box(dev, 0, mailbox->dma, 0, op_modifier, - MLX4_CMD_DIAG_RPRT, MLX4_CMD_TIME_CLASS_A); - if (ret) - goto out; + while (num_tasks) { + err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, + MLX4_CMD_GET_OP_REQ, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) { + mlx4_err(dev, "Failed to retreive required operation: %d\n", err); + return; + } + MLX4_GET(modifier, outbox, GET_OP_REQ_MODIFIER_OFFSET); + MLX4_GET(token, outbox, GET_OP_REQ_TOKEN_OFFSET); + MLX4_GET(type, outbox, GET_OP_REQ_TYPE_OFFSET); + type_m = type >> 12; + type &= 0xfff; - for (i=0; i < array_length; i++) { - if (in_offset[i] > MLX4_MAILBOX_SIZE) { - ret = -EINVAL; + switch (type) { + case ADD_TO_MCG: + if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) { + mlx4_warn(dev, "ADD MCG operation is not supported in " + "DEVICE_MANAGED steerign mode\n"); + err = EPERM; + break; + } + mgm = (struct mlx4_mgm *) ((u8 *) (outbox) + GET_OP_REQ_DATA_OFFSET); + num_qps = be32_to_cpu(mgm->members_count) & MGM_QPN_MASK; + rem_mcg = ((u8 *) (&mgm->members_count))[0] & 1; + prot = ((u8 *) (&mgm->members_count))[0] >> 6; + + for (i = 0; i < num_qps; i++) { + qp.qpn = be32_to_cpu(mgm->qp[i]); + if (rem_mcg) + err = mlx4_multicast_detach(dev, &qp, mgm->gid, prot, 0); + else + err = mlx4_multicast_attach(dev, &qp, mgm->gid, mgm->gid[5] ,0, prot, NULL); + if (err) + break; + } + break; + default: + mlx4_warn(dev, "Bad type for required operation\n"); + err = EINVAL; + break; + } + err = mlx4_cmd(dev, 0, ((u32) err | cpu_to_be32(token) << 16), 1, + MLX4_CMD_GET_OP_REQ, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) { + mlx4_err(dev, "Failed to acknowledge required request: %d\n", err); goto out; } - - MLX4_GET(counter_out[i], outbox, in_offset[i]); + memset(outbox, 0, 0xffc); + num_tasks = atomic_dec_return(&priv->opreq_count); } out: mlx4_free_cmd_mailbox(dev, mailbox); - return ret; } -EXPORT_SYMBOL_GPL(mlx4_query_diag_counters); - -void mlx4_get_fc_t11_settings(struct mlx4_dev *dev, int *enable_pre_t11, int *t11_supported) -{ - *enable_pre_t11 = !!mlx4_pre_t11_mode; - *t11_supported = !!(dev->caps.flags & MLX4_DEV_CAP_FLAG_FC_T11); -} -EXPORT_SYMBOL_GPL(mlx4_get_fc_t11_settings); Index: stable/9/sys/ofed/drivers/net/mlx4/fw.h =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/fw.h (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/fw.h (revision 273864) @@ -1,187 +1,230 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2006, 2007 Cisco Systems. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef MLX4_FW_H #define MLX4_FW_H #include "mlx4.h" #include "icm.h" struct mlx4_mod_stat_cfg { u8 log_pg_sz; u8 log_pg_sz_m; }; struct mlx4_dev_cap { int max_srq_sz; int max_qp_sz; int reserved_qps; int max_qps; int reserved_srqs; int max_srqs; int max_cq_sz; int reserved_cqs; int max_cqs; int max_mpts; int reserved_eqs; int max_eqs; int reserved_mtts; int max_mrw_sz; int reserved_mrws; int max_mtt_seg; int max_requester_per_qp; int max_responder_per_qp; int max_rdma_global; int local_ca_ack_delay; int num_ports; u32 max_msg_sz; int ib_mtu[MLX4_MAX_PORTS + 1]; int max_port_width[MLX4_MAX_PORTS + 1]; int max_vl[MLX4_MAX_PORTS + 1]; int max_gids[MLX4_MAX_PORTS + 1]; int max_pkeys[MLX4_MAX_PORTS + 1]; u64 def_mac[MLX4_MAX_PORTS + 1]; u16 eth_mtu[MLX4_MAX_PORTS + 1]; int trans_type[MLX4_MAX_PORTS + 1]; int vendor_oui[MLX4_MAX_PORTS + 1]; u16 wavelength[MLX4_MAX_PORTS + 1]; u64 trans_code[MLX4_MAX_PORTS + 1]; u16 stat_rate_support; - int udp_rss; - int loopback_support; - int wol; + int fs_log_max_ucast_qp_range_size; + int fs_max_num_qp_per_entry; u64 flags; + u64 flags2; int reserved_uars; int uar_size; int min_page_sz; int bf_reg_size; int bf_regs_per_page; int max_sq_sg; int max_sq_desc_sz; int max_rq_sg; int max_rq_desc_sz; int max_qp_per_mcg; int reserved_mgms; int max_mcgs; int reserved_pds; int max_pds; int reserved_xrcds; int max_xrcds; int qpc_entry_sz; int rdmarc_entry_sz; int altc_entry_sz; int aux_entry_sz; int srq_entry_sz; int cqc_entry_sz; int eqc_entry_sz; int dmpt_entry_sz; int cmpt_entry_sz; int mtt_entry_sz; - int inline_cfg; int resize_srq; u32 bmme_flags; u32 reserved_lkey; u64 max_icm_sz; int max_gso_sz; + int max_rss_tbl_sz; u8 supported_port_types[MLX4_MAX_PORTS + 1]; + u8 suggested_type[MLX4_MAX_PORTS + 1]; + u8 default_sense[MLX4_MAX_PORTS + 1]; u8 log_max_macs[MLX4_MAX_PORTS + 1]; u8 log_max_vlans[MLX4_MAX_PORTS + 1]; u32 max_basic_counters; - u32 max_ext_counters; + u32 sync_qp; + u8 timestamp_support; + u32 max_extended_counters; }; +struct mlx4_func_cap { + u8 num_ports; + u8 flags; + u32 pf_context_behaviour; + int qp_quota; + int cq_quota; + int srq_quota; + int mpt_quota; + int mtt_quota; + int max_eq; + int reserved_eq; + int mcg_quota; + u32 qp0_tunnel_qpn; + u32 qp0_proxy_qpn; + u32 qp1_tunnel_qpn; + u32 qp1_proxy_qpn; + u8 physical_port; + u8 port_flags; + u8 def_counter_index; +}; + struct mlx4_adapter { + u16 vsd_vendor_id; char board_id[MLX4_BOARD_ID_LEN]; + char vsd[MLX4_VSD_LEN]; u8 inta_pin; }; struct mlx4_init_hca_param { u64 qpc_base; u64 rdmarc_base; u64 auxc_base; u64 altc_base; u64 srqc_base; u64 cqc_base; u64 eqc_base; u64 mc_base; u64 dmpt_base; u64 cmpt_base; u64 mtt_base; + u64 global_caps; u16 log_mc_entry_sz; u16 log_mc_hash_sz; + u16 hca_core_clock; u8 log_num_qps; u8 log_num_srqs; u8 log_num_cqs; u8 log_num_eqs; u8 log_rd_per_qp; u8 log_mc_table_sz; u8 log_mpt_sz; u8 log_uar_sz; + u8 uar_page_sz; /* log pg sz in 4k chunks */ + u8 mw_enable; /* Enable memory windows */ + u8 fs_hash_enable_bits; + u8 steering_mode; /* for QUERY_HCA */ + u64 dev_cap_enabled; }; struct mlx4_init_ib_param { int port_width; int vl_cap; int mtu_cap; u16 gid_cap; u16 pkey_cap; int set_guid0; u64 guid0; int set_node_guid; u64 node_guid; int set_si_guid; u64 si_guid; }; struct mlx4_set_ib_param { int set_si_guid; int reset_qkey_viol; u64 si_guid; u32 cap_mask; }; int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap); +int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u32 gen_or_port, + struct mlx4_func_cap *func_cap); +int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); int mlx4_MAP_FA(struct mlx4_dev *dev, struct mlx4_icm *icm); int mlx4_UNMAP_FA(struct mlx4_dev *dev); int mlx4_RUN_FW(struct mlx4_dev *dev); int mlx4_QUERY_FW(struct mlx4_dev *dev); int mlx4_QUERY_ADAPTER(struct mlx4_dev *dev, struct mlx4_adapter *adapter); int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param); +int mlx4_QUERY_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param); int mlx4_CLOSE_HCA(struct mlx4_dev *dev, int panic); int mlx4_map_cmd(struct mlx4_dev *dev, u16 op, struct mlx4_icm *icm, u64 virt); int mlx4_SET_ICM_SIZE(struct mlx4_dev *dev, u64 icm_size, u64 *aux_pages); int mlx4_NOP(struct mlx4_dev *dev); int mlx4_MOD_STAT_CFG(struct mlx4_dev *dev, struct mlx4_mod_stat_cfg *cfg); +void mlx4_opreq_action(struct work_struct *work); #endif /* MLX4_FW_H */ Index: stable/9/sys/ofed/drivers/net/mlx4/icm.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/icm.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/icm.c (revision 273864) @@ -1,455 +1,473 @@ /* - * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include #include #include #include +#include +#include #include #include "mlx4.h" #include "icm.h" #include "fw.h" /* * We allocate in as big chunks as we can, up to a maximum of 256 KB * per chunk. */ enum { MLX4_ICM_ALLOC_SIZE = 1 << 18, MLX4_TABLE_CHUNK_SIZE = 1 << 18 }; static void mlx4_free_icm_pages(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk) { int i; if (chunk->nsg > 0) pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages, PCI_DMA_BIDIRECTIONAL); for (i = 0; i < chunk->npages; ++i) __free_pages(sg_page(&chunk->mem[i]), get_order(chunk->mem[i].length)); } static void mlx4_free_icm_coherent(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk) { int i; for (i = 0; i < chunk->npages; ++i) dma_free_coherent(&dev->pdev->dev, chunk->mem[i].length, lowmem_page_address(sg_page(&chunk->mem[i])), sg_dma_address(&chunk->mem[i])); } void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent) { struct mlx4_icm_chunk *chunk, *tmp; if (!icm) return; list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) { if (coherent) mlx4_free_icm_coherent(dev, chunk); else mlx4_free_icm_pages(dev, chunk); kfree(chunk); } kfree(icm); } -static int mlx4_alloc_icm_pages(struct scatterlist *mem, int order, gfp_t gfp_mask) +static int mlx4_alloc_icm_pages(struct scatterlist *mem, int order, + gfp_t gfp_mask, int node) { struct page *page; - page = alloc_pages(gfp_mask, order); - if (!page) - return -ENOMEM; + page = alloc_pages_node(node, gfp_mask, order); + if (!page) { + page = alloc_pages(gfp_mask, order); + if (!page) + return -ENOMEM; + } sg_set_page(mem, page, PAGE_SIZE << order, 0); return 0; } static int mlx4_alloc_icm_coherent(struct device *dev, struct scatterlist *mem, int order, gfp_t gfp_mask) { void *buf = dma_alloc_coherent(dev, PAGE_SIZE << order, &sg_dma_address(mem), gfp_mask); if (!buf) return -ENOMEM; sg_set_buf(mem, buf, PAGE_SIZE << order); BUG_ON(mem->offset); sg_dma_len(mem) = PAGE_SIZE << order; return 0; } struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages, gfp_t gfp_mask, int coherent) { struct mlx4_icm *icm; struct mlx4_icm_chunk *chunk = NULL; int cur_order; int ret; /* We use sg_set_buf for coherent allocs, which assumes low memory */ BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM)); - icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); - if (!icm) - return NULL; + icm = kmalloc_node(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN), + dev->numa_node); + if (!icm) { + icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); + if (!icm) + return NULL; + } icm->refcount = 0; INIT_LIST_HEAD(&icm->chunk_list); cur_order = get_order(MLX4_ICM_ALLOC_SIZE); while (npages > 0) { if (!chunk) { - chunk = kmalloc(sizeof *chunk, - gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); - if (!chunk) - goto fail; + chunk = kmalloc_node(sizeof *chunk, + gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN), + dev->numa_node); + if (!chunk) { + chunk = kmalloc(sizeof *chunk, + gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); + if (!chunk) + goto fail; + } sg_init_table(chunk->mem, MLX4_ICM_CHUNK_LEN); chunk->npages = 0; chunk->nsg = 0; list_add_tail(&chunk->list, &icm->chunk_list); } while (1 << cur_order > npages) --cur_order; if (coherent) ret = mlx4_alloc_icm_coherent(&dev->pdev->dev, &chunk->mem[chunk->npages], cur_order, gfp_mask); else ret = mlx4_alloc_icm_pages(&chunk->mem[chunk->npages], - cur_order, gfp_mask); + cur_order, gfp_mask, + dev->numa_node); - if (!ret) { - ++chunk->npages; + if (ret) { + if (--cur_order < 0) + goto fail; + else + continue; + } - if (coherent) - ++chunk->nsg; - else if (chunk->npages == MLX4_ICM_CHUNK_LEN) { - chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, - chunk->npages, - PCI_DMA_BIDIRECTIONAL); + ++chunk->npages; - if (chunk->nsg <= 0) - goto fail; - } + if (coherent) + ++chunk->nsg; + else if (chunk->npages == MLX4_ICM_CHUNK_LEN) { + chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, + chunk->npages, + PCI_DMA_BIDIRECTIONAL); - if (chunk->npages == MLX4_ICM_CHUNK_LEN) - chunk = NULL; - - npages -= 1 << cur_order; - } else { - --cur_order; - if (cur_order < 0) + if (chunk->nsg <= 0) goto fail; } + + if (chunk->npages == MLX4_ICM_CHUNK_LEN) + chunk = NULL; + + npages -= 1 << cur_order; } if (!coherent && chunk) { chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, chunk->npages, PCI_DMA_BIDIRECTIONAL); if (chunk->nsg <= 0) goto fail; } return icm; fail: mlx4_free_icm(dev, icm, coherent); return NULL; } static int mlx4_MAP_ICM(struct mlx4_dev *dev, struct mlx4_icm *icm, u64 virt) { return mlx4_map_cmd(dev, MLX4_CMD_MAP_ICM, icm, virt); } -int mlx4_UNMAP_ICM(struct mlx4_dev *dev, u64 virt, u32 page_count) +static int mlx4_UNMAP_ICM(struct mlx4_dev *dev, u64 virt, u32 page_count) { return mlx4_cmd(dev, virt, page_count, 0, MLX4_CMD_UNMAP_ICM, - MLX4_CMD_TIME_CLASS_B); + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); } -int mlx4_MAP_ICM_page(struct mlx4_dev *dev, u64 dma_addr, u64 virt) -{ - struct mlx4_cmd_mailbox *mailbox; - __be64 *inbox; - int err; - - mailbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - inbox = mailbox->buf; - - inbox[0] = cpu_to_be64(virt); - inbox[1] = cpu_to_be64(dma_addr); - - err = mlx4_cmd(dev, mailbox->dma, 1, 0, MLX4_CMD_MAP_ICM, - MLX4_CMD_TIME_CLASS_B); - - mlx4_free_cmd_mailbox(dev, mailbox); - - if (!err) - mlx4_dbg(dev, "Mapped page at %llx to %llx for ICM.\n", - (unsigned long long) dma_addr, (unsigned long long) virt); - - return err; -} - int mlx4_MAP_ICM_AUX(struct mlx4_dev *dev, struct mlx4_icm *icm) { return mlx4_map_cmd(dev, MLX4_CMD_MAP_ICM_AUX, icm, -1); } int mlx4_UNMAP_ICM_AUX(struct mlx4_dev *dev) { - return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_UNMAP_ICM_AUX, MLX4_CMD_TIME_CLASS_B); + return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_UNMAP_ICM_AUX, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); } -int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj) +int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj) { - int i = (obj & (table->num_obj - 1)) / (MLX4_TABLE_CHUNK_SIZE / table->obj_size); + u32 i = (obj & (table->num_obj - 1)) / + (MLX4_TABLE_CHUNK_SIZE / table->obj_size); int ret = 0; mutex_lock(&table->mutex); if (table->icm[i]) { ++table->icm[i]->refcount; goto out; } table->icm[i] = mlx4_alloc_icm(dev, MLX4_TABLE_CHUNK_SIZE >> PAGE_SHIFT, (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN, table->coherent); if (!table->icm[i]) { ret = -ENOMEM; goto out; } if (mlx4_MAP_ICM(dev, table->icm[i], table->virt + (u64) i * MLX4_TABLE_CHUNK_SIZE)) { mlx4_free_icm(dev, table->icm[i], table->coherent); table->icm[i] = NULL; ret = -ENOMEM; goto out; } ++table->icm[i]->refcount; out: mutex_unlock(&table->mutex); return ret; } -void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj) +void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj) { - int i; + u32 i; + u64 offset; i = (obj & (table->num_obj - 1)) / (MLX4_TABLE_CHUNK_SIZE / table->obj_size); mutex_lock(&table->mutex); if (--table->icm[i]->refcount == 0) { - mlx4_UNMAP_ICM(dev, table->virt + i * MLX4_TABLE_CHUNK_SIZE, - MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE); - mlx4_free_icm(dev, table->icm[i], table->coherent); - table->icm[i] = NULL; + offset = (u64) i * MLX4_TABLE_CHUNK_SIZE; + + if (!mlx4_UNMAP_ICM(dev, table->virt + offset, + MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE)) { + mlx4_free_icm(dev, table->icm[i], table->coherent); + table->icm[i] = NULL; + } else { + pr_warn("mlx4_core: mlx4_UNMAP_ICM failed.\n"); + } } mutex_unlock(&table->mutex); } -void *mlx4_table_find(struct mlx4_icm_table *table, int obj, dma_addr_t *dma_handle) +void *mlx4_table_find(struct mlx4_icm_table *table, u32 obj, + dma_addr_t *dma_handle) { - int idx, offset, dma_offset, i; + int offset, dma_offset, i; + u64 idx; struct mlx4_icm_chunk *chunk; struct mlx4_icm *icm; struct page *page = NULL; if (!table->lowmem) return NULL; mutex_lock(&table->mutex); - idx = (obj & (table->num_obj - 1)) * table->obj_size; + idx = (u64) (obj & (table->num_obj - 1)) * table->obj_size; icm = table->icm[idx / MLX4_TABLE_CHUNK_SIZE]; dma_offset = offset = idx % MLX4_TABLE_CHUNK_SIZE; if (!icm) goto out; list_for_each_entry(chunk, &icm->chunk_list, list) { for (i = 0; i < chunk->npages; ++i) { if (dma_handle && dma_offset >= 0) { if (sg_dma_len(&chunk->mem[i]) > dma_offset) *dma_handle = sg_dma_address(&chunk->mem[i]) + dma_offset; dma_offset -= sg_dma_len(&chunk->mem[i]); } /* * DMA mapping can merge pages but not split them, * so if we found the page, dma_handle has already * been assigned to. */ if (chunk->mem[i].length > offset) { page = sg_page(&chunk->mem[i]); goto out; } offset -= chunk->mem[i].length; } } out: mutex_unlock(&table->mutex); return page ? lowmem_page_address(page) + offset : NULL; } int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, - int start, int end) + u32 start, u32 end) { int inc = MLX4_TABLE_CHUNK_SIZE / table->obj_size; - int i, err; + int err; + u32 i; for (i = start; i <= end; i += inc) { err = mlx4_table_get(dev, table, i); if (err) goto fail; } return 0; fail: while (i > start) { i -= inc; mlx4_table_put(dev, table, i); } return err; } void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, - int start, int end) + u32 start, u32 end) { - int i; + u32 i; for (i = start; i <= end; i += MLX4_TABLE_CHUNK_SIZE / table->obj_size) mlx4_table_put(dev, table, i); } int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table, - u64 virt, int obj_size, int nobj, int reserved, + u64 virt, int obj_size, u64 nobj, int reserved, int use_lowmem, int use_coherent) { int obj_per_chunk; int num_icm; unsigned chunk_size; int i; + u64 size; obj_per_chunk = MLX4_TABLE_CHUNK_SIZE / obj_size; - num_icm = (nobj + obj_per_chunk - 1) / obj_per_chunk; + num_icm = div_u64((nobj + obj_per_chunk - 1), obj_per_chunk); table->icm = kcalloc(num_icm, sizeof *table->icm, GFP_KERNEL); if (!table->icm) return -ENOMEM; table->virt = virt; table->num_icm = num_icm; table->num_obj = nobj; table->obj_size = obj_size; table->lowmem = use_lowmem; table->coherent = use_coherent; mutex_init(&table->mutex); + size = (u64) nobj * obj_size; for (i = 0; i * MLX4_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) { chunk_size = MLX4_TABLE_CHUNK_SIZE; - if ((i + 1) * MLX4_TABLE_CHUNK_SIZE > nobj * obj_size) - chunk_size = PAGE_ALIGN(nobj * obj_size - i * MLX4_TABLE_CHUNK_SIZE); + if ((i + 1) * MLX4_TABLE_CHUNK_SIZE > size) + chunk_size = PAGE_ALIGN(size - + i * MLX4_TABLE_CHUNK_SIZE); table->icm[i] = mlx4_alloc_icm(dev, chunk_size >> PAGE_SHIFT, (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN, use_coherent); if (!table->icm[i]) goto err; if (mlx4_MAP_ICM(dev, table->icm[i], virt + i * MLX4_TABLE_CHUNK_SIZE)) { mlx4_free_icm(dev, table->icm[i], use_coherent); table->icm[i] = NULL; goto err; } /* * Add a reference to this ICM chunk so that it never * gets freed (since it contains reserved firmware objects). */ ++table->icm[i]->refcount; } return 0; err: for (i = 0; i < num_icm; ++i) if (table->icm[i]) { - mlx4_UNMAP_ICM(dev, virt + i * MLX4_TABLE_CHUNK_SIZE, - MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE); - mlx4_free_icm(dev, table->icm[i], use_coherent); + if (!mlx4_UNMAP_ICM(dev, + virt + i * MLX4_TABLE_CHUNK_SIZE, + MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE)) { + mlx4_free_icm(dev, table->icm[i], use_coherent); + } else { + pr_warn("mlx4_core: mlx4_UNMAP_ICM failed.\n"); + return -ENOMEM; + } } + kfree(table->icm); return -ENOMEM; } void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table) { - int i; + int i, err = 0; for (i = 0; i < table->num_icm; ++i) if (table->icm[i]) { - mlx4_UNMAP_ICM(dev, table->virt + i * MLX4_TABLE_CHUNK_SIZE, - MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE); - mlx4_free_icm(dev, table->icm[i], table->coherent); + err = mlx4_UNMAP_ICM(dev, + table->virt + i * MLX4_TABLE_CHUNK_SIZE, + MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE); + if (!err) { + mlx4_free_icm(dev, table->icm[i], + table->coherent); + } else { + pr_warn("mlx4_core: mlx4_UNMAP_ICM failed.\n"); + break; + } } - kfree(table->icm); + if (!err) + kfree(table->icm); } Index: stable/9/sys/ofed/drivers/net/mlx4/icm.h =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/icm.h (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/icm.h (revision 273864) @@ -1,130 +1,129 @@ /* - * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef MLX4_ICM_H #define MLX4_ICM_H #include #include #include +#include #define MLX4_ICM_CHUNK_LEN \ ((256 - sizeof (struct list_head) - 2 * sizeof (int)) / \ (sizeof (struct scatterlist))) enum { MLX4_ICM_PAGE_SHIFT = 12, MLX4_ICM_PAGE_SIZE = 1 << MLX4_ICM_PAGE_SHIFT, }; struct mlx4_icm_chunk { struct list_head list; int npages; int nsg; struct scatterlist mem[MLX4_ICM_CHUNK_LEN]; }; struct mlx4_icm { struct list_head chunk_list; int refcount; }; struct mlx4_icm_iter { struct mlx4_icm *icm; struct mlx4_icm_chunk *chunk; int page_idx; }; struct mlx4_dev; struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages, gfp_t gfp_mask, int coherent); void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent); +int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj); +void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj); +int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, + u32 start, u32 end); +void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, + u32 start, u32 end); int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table, - u64 virt, int obj_size, int nobj, int reserved, + u64 virt, int obj_size, u64 nobj, int reserved, int use_lowmem, int use_coherent); void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table); -int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj); -void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj); -void *mlx4_table_find(struct mlx4_icm_table *table, int obj, dma_addr_t *dma_handle); -int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, - int start, int end); -void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, - int start, int end); +void *mlx4_table_find(struct mlx4_icm_table *table, u32 obj, dma_addr_t *dma_handle); static inline void mlx4_icm_first(struct mlx4_icm *icm, struct mlx4_icm_iter *iter) { iter->icm = icm; iter->chunk = list_empty(&icm->chunk_list) ? NULL : list_entry(icm->chunk_list.next, struct mlx4_icm_chunk, list); iter->page_idx = 0; } static inline int mlx4_icm_last(struct mlx4_icm_iter *iter) { return !iter->chunk; } static inline void mlx4_icm_next(struct mlx4_icm_iter *iter) { if (++iter->page_idx >= iter->chunk->nsg) { if (iter->chunk->list.next == &iter->icm->chunk_list) { iter->chunk = NULL; return; } iter->chunk = list_entry(iter->chunk->list.next, struct mlx4_icm_chunk, list); iter->page_idx = 0; } } static inline dma_addr_t mlx4_icm_addr(struct mlx4_icm_iter *iter) { return sg_dma_address(&iter->chunk->mem[iter->page_idx]); } static inline unsigned long mlx4_icm_size(struct mlx4_icm_iter *iter) { return sg_dma_len(&iter->chunk->mem[iter->page_idx]); } -int mlx4_UNMAP_ICM(struct mlx4_dev *dev, u64 virt, u32 page_count); -int mlx4_MAP_ICM_page(struct mlx4_dev *dev, u64 dma_addr, u64 virt); int mlx4_MAP_ICM_AUX(struct mlx4_dev *dev, struct mlx4_icm *icm); int mlx4_UNMAP_ICM_AUX(struct mlx4_dev *dev); #endif /* MLX4_ICM_H */ Index: stable/9/sys/ofed/drivers/net/mlx4/intf.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/intf.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/intf.c (revision 273864) @@ -1,212 +1,188 @@ /* * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include +#include + #include "mlx4.h" struct mlx4_device_context { struct list_head list; struct mlx4_interface *intf; void *context; }; static LIST_HEAD(intf_list); static LIST_HEAD(dev_list); static DEFINE_MUTEX(intf_mutex); static void mlx4_add_device(struct mlx4_interface *intf, struct mlx4_priv *priv) { struct mlx4_device_context *dev_ctx; dev_ctx = kmalloc(sizeof *dev_ctx, GFP_KERNEL); if (!dev_ctx) return; dev_ctx->intf = intf; dev_ctx->context = intf->add(&priv->dev); if (dev_ctx->context) { spin_lock_irq(&priv->ctx_lock); list_add_tail(&dev_ctx->list, &priv->ctx_list); spin_unlock_irq(&priv->ctx_lock); } else kfree(dev_ctx); } static void mlx4_remove_device(struct mlx4_interface *intf, struct mlx4_priv *priv) { struct mlx4_device_context *dev_ctx; list_for_each_entry(dev_ctx, &priv->ctx_list, list) if (dev_ctx->intf == intf) { spin_lock_irq(&priv->ctx_lock); list_del(&dev_ctx->list); spin_unlock_irq(&priv->ctx_lock); intf->remove(&priv->dev, dev_ctx->context); kfree(dev_ctx); return; } } int mlx4_register_interface(struct mlx4_interface *intf) { struct mlx4_priv *priv; if (!intf->add || !intf->remove) return -EINVAL; mutex_lock(&intf_mutex); list_add_tail(&intf->list, &intf_list); list_for_each_entry(priv, &dev_list, dev_list) mlx4_add_device(intf, priv); mutex_unlock(&intf_mutex); return 0; } EXPORT_SYMBOL_GPL(mlx4_register_interface); void mlx4_unregister_interface(struct mlx4_interface *intf) { struct mlx4_priv *priv; mutex_lock(&intf_mutex); list_for_each_entry(priv, &dev_list, dev_list) mlx4_remove_device(intf, priv); list_del(&intf->list); mutex_unlock(&intf_mutex); } EXPORT_SYMBOL_GPL(mlx4_unregister_interface); -struct mlx4_dev *mlx4_query_interface(void *int_dev, int *port) +void mlx4_dispatch_event(struct mlx4_dev *dev, enum mlx4_dev_event type, + unsigned long param) { - struct mlx4_priv *priv; - struct mlx4_device_context *dev_ctx; - enum mlx4_query_reply r; - unsigned long flags; - - mutex_lock(&intf_mutex); - - list_for_each_entry(priv, &dev_list, dev_list) { - spin_lock_irqsave(&priv->ctx_lock, flags); - list_for_each_entry(dev_ctx, &priv->ctx_list, list) { - if (!dev_ctx->intf->query) - continue; - r = dev_ctx->intf->query(dev_ctx->context, int_dev); - if (r != MLX4_QUERY_NOT_MINE) { - *port = r; - spin_unlock_irqrestore(&priv->ctx_lock, flags); - mutex_unlock(&intf_mutex); - return &priv->dev; - } - } - spin_unlock_irqrestore(&priv->ctx_lock, flags); - } - - mutex_unlock(&intf_mutex); - return NULL; -} -EXPORT_SYMBOL_GPL(mlx4_query_interface); - -void mlx4_dispatch_event(struct mlx4_dev *dev, enum mlx4_dev_event type, int port) -{ struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_device_context *dev_ctx; unsigned long flags; spin_lock_irqsave(&priv->ctx_lock, flags); list_for_each_entry(dev_ctx, &priv->ctx_list, list) if (dev_ctx->intf->event) - dev_ctx->intf->event(dev, dev_ctx->context, type, port); + dev_ctx->intf->event(dev, dev_ctx->context, type, param); spin_unlock_irqrestore(&priv->ctx_lock, flags); } int mlx4_register_device(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_interface *intf; mutex_lock(&intf_mutex); list_add_tail(&priv->dev_list, &dev_list); list_for_each_entry(intf, &intf_list, list) mlx4_add_device(intf, priv); mutex_unlock(&intf_mutex); - mlx4_start_catas_poll(dev); + if (!mlx4_is_slave(dev)) + mlx4_start_catas_poll(dev); return 0; } void mlx4_unregister_device(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_interface *intf; - mlx4_stop_catas_poll(dev); + if (!mlx4_is_slave(dev)) + mlx4_stop_catas_poll(dev); mutex_lock(&intf_mutex); list_for_each_entry(intf, &intf_list, list) mlx4_remove_device(intf, priv); - list_del(&priv->dev_list); + list_del_init(&priv->dev_list); mutex_unlock(&intf_mutex); } -void *mlx4_find_get_prot_dev(struct mlx4_dev *dev, enum mlx4_prot proto, int port) +void *mlx4_get_protocol_dev(struct mlx4_dev *dev, enum mlx4_protocol proto, int port) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_device_context *dev_ctx; unsigned long flags; void *result = NULL; spin_lock_irqsave(&priv->ctx_lock, flags); list_for_each_entry(dev_ctx, &priv->ctx_list, list) - if (dev_ctx->intf->protocol == proto && dev_ctx->intf->get_prot_dev) { - result = dev_ctx->intf->get_prot_dev(dev, dev_ctx->context, port); + if (dev_ctx->intf->protocol == proto && dev_ctx->intf->get_dev) { + result = dev_ctx->intf->get_dev(dev, dev_ctx->context, port); break; - } + } spin_unlock_irqrestore(&priv->ctx_lock, flags); return result; } - +EXPORT_SYMBOL_GPL(mlx4_get_protocol_dev); Index: stable/9/sys/ofed/drivers/net/mlx4/main.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/main.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/main.c (revision 273864) @@ -1,1702 +1,3802 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include +/* + * kmod.h must be included before module.h since it includes (indirectly) sys/module.h + * To use the FBSD macro sys/module.h should define MODULE_VERSION before linux/module does. +*/ #include -#include #include #include #include +#include #include +#include +#include +#include +#include #include #include #include "mlx4.h" #include "fw.h" #include "icm.h" +#include "mlx4_stats.h" MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("Mellanox ConnectX HCA low-level driver"); MODULE_LICENSE("Dual BSD/GPL"); -MODULE_VERSION(DRV_VERSION); struct workqueue_struct *mlx4_wq; #ifdef CONFIG_MLX4_DEBUG int mlx4_debug_level = 0; module_param_named(debug_level, mlx4_debug_level, int, 0644); MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0"); #endif /* CONFIG_MLX4_DEBUG */ -int mlx4_blck_lb=1; -module_param_named(block_loopback, mlx4_blck_lb, int, 0644); -MODULE_PARM_DESC(block_loopback, "Block multicast loopback packets if > 0"); - #ifdef CONFIG_PCI_MSI static int msi_x = 1; module_param(msi_x, int, 0444); -MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero"); +MODULE_PARM_DESC(msi_x, "0 - don't use MSI-X, 1 - use MSI-X, >1 - limit number of MSI-X irqs to msi_x (non-SRIOV only)"); #else /* CONFIG_PCI_MSI */ #define msi_x (0) #endif /* CONFIG_PCI_MSI */ +static int enable_sys_tune = 0; +module_param(enable_sys_tune, int, 0444); +MODULE_PARM_DESC(enable_sys_tune, "Tune the cpu's for better performance (default 0)"); + +int mlx4_blck_lb = 1; +module_param_named(block_loopback, mlx4_blck_lb, int, 0644); +MODULE_PARM_DESC(block_loopback, "Block multicast loopback packets if > 0 " + "(default: 1)"); +enum { + DEFAULT_DOMAIN = 0, + BDF_STR_SIZE = 8, /* bb:dd.f- */ + DBDF_STR_SIZE = 13 /* mmmm:bb:dd.f- */ +}; + +enum { + NUM_VFS, + PROBE_VF, + PORT_TYPE_ARRAY +}; + +enum { + VALID_DATA, + INVALID_DATA, + INVALID_STR +}; + +struct param_data { + int id; + struct mlx4_dbdf2val_lst dbdf2val; +}; + +static struct param_data num_vfs = { + .id = NUM_VFS, + .dbdf2val = { + .name = "num_vfs param", + .num_vals = 1, + .def_val = {0}, + .range = {0, MLX4_MAX_NUM_VF} + } +}; +module_param_string(num_vfs, num_vfs.dbdf2val.str, + sizeof(num_vfs.dbdf2val.str), 0444); +MODULE_PARM_DESC(num_vfs, + "Either single value (e.g. '5') to define uniform num_vfs value for all devices functions\n" + "\t\tor a string to map device function numbers to their num_vfs values (e.g. '0000:04:00.0-5,002b:1c:0b.a-15').\n" + "\t\tHexadecimal digits for the device function (e.g. 002b:1c:0b.a) and decimal for num_vfs value (e.g. 15)."); + +static struct param_data probe_vf = { + .id = PROBE_VF, + .dbdf2val = { + .name = "probe_vf param", + .num_vals = 1, + .def_val = {0}, + .range = {0, MLX4_MAX_NUM_VF} + } +}; +module_param_string(probe_vf, probe_vf.dbdf2val.str, + sizeof(probe_vf.dbdf2val.str), 0444); +MODULE_PARM_DESC(probe_vf, + "Either single value (e.g. '3') to define uniform number of VFs to probe by the pf driver for all devices functions\n" + "\t\tor a string to map device function numbers to their probe_vf values (e.g. '0000:04:00.0-3,002b:1c:0b.a-13').\n" + "\t\tHexadecimal digits for the device function (e.g. 002b:1c:0b.a) and decimal for probe_vf value (e.g. 13)."); + +int mlx4_log_num_mgm_entry_size = MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE; + +module_param_named(log_num_mgm_entry_size, + mlx4_log_num_mgm_entry_size, int, 0444); +MODULE_PARM_DESC(log_num_mgm_entry_size, "log mgm size, that defines the num" + " of qp per mcg, for example:" + " 10 gives 248.range: 7 <=" + " log_num_mgm_entry_size <= 12." + " To activate device managed" + " flow steering when available, set to -1"); + +static int high_rate_steer; +module_param(high_rate_steer, int, 0444); +MODULE_PARM_DESC(high_rate_steer, "Enable steering mode for higher packet rate" + " (default off)"); + +static int fast_drop; +module_param_named(fast_drop, fast_drop, int, 0444); +MODULE_PARM_DESC(fast_drop, + "Enable fast packet drop when no recieve WQEs are posted"); + +int mlx4_enable_64b_cqe_eqe = 1; +module_param_named(enable_64b_cqe_eqe, mlx4_enable_64b_cqe_eqe, int, 0644); +MODULE_PARM_DESC(enable_64b_cqe_eqe, + "Enable 64 byte CQEs/EQEs when the the FW supports this if non-zero (default: 1)"); + +#define HCA_GLOBAL_CAP_MASK 0 + +#define PF_CONTEXT_BEHAVIOUR_MASK MLX4_FUNC_CAP_64B_EQE_CQE + static char mlx4_version[] __devinitdata = DRV_NAME ": Mellanox ConnectX core driver v" DRV_VERSION " (" DRV_RELDATE ")\n"; -struct mutex drv_mutex; +static int log_num_mac = 7; +module_param_named(log_num_mac, log_num_mac, int, 0444); +MODULE_PARM_DESC(log_num_mac, "Log2 max number of MACs per ETH port (1-7)"); -static struct mlx4_profile default_profile = { - .num_qp = 1 << 18, - .num_srq = 1 << 16, - .rdmarc_per_qp = 1 << 4, - .num_cq = 1 << 16, - .num_mcg = 1 << 13, - .num_mpt = 1 << 19, - .num_mtt = 1 << 20, +static int log_num_vlan; +module_param_named(log_num_vlan, log_num_vlan, int, 0444); +MODULE_PARM_DESC(log_num_vlan, + "(Obsolete) Log2 max number of VLANs per ETH port (0-7)"); +/* Log2 max number of VLANs per ETH port (0-7) */ +#define MLX4_LOG_NUM_VLANS 7 + +int log_mtts_per_seg = ilog2(1); +module_param_named(log_mtts_per_seg, log_mtts_per_seg, int, 0444); +MODULE_PARM_DESC(log_mtts_per_seg, "Log2 number of MTT entries per segment " + "(0-7) (default: 0)"); + +static struct param_data port_type_array = { + .id = PORT_TYPE_ARRAY, + .dbdf2val = { + .name = "port_type_array param", + .num_vals = 2, + .def_val = {MLX4_PORT_TYPE_ETH, MLX4_PORT_TYPE_ETH}, + .range = {MLX4_PORT_TYPE_IB, MLX4_PORT_TYPE_NA} + } }; +module_param_string(port_type_array, port_type_array.dbdf2val.str, + sizeof(port_type_array.dbdf2val.str), 0444); +MODULE_PARM_DESC(port_type_array, + "Either pair of values (e.g. '1,2') to define uniform port1/port2 types configuration for all devices functions\n" + "\t\tor a string to map device function numbers to their pair of port types values (e.g. '0000:04:00.0-1;2,002b:1c:0b.a-1;1').\n" + "\t\tValid port types: 1-ib, 2-eth, 3-auto, 4-N/A\n" + "\t\tIn case that only one port is available use the N/A port type for port2 (e.g '1,4')."); -static int log_num_mac = 2; -module_param_named(log_num_mac, log_num_mac, int, 0444); -MODULE_PARM_DESC(log_num_mac, "Log2 max number of MACs per ETH port (1-7)"); -static int use_prio; -module_param_named(use_prio, use_prio, bool, 0444); -MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports " - "(0/1, default 0)"); +struct mlx4_port_config { + struct list_head list; + enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1]; + struct pci_dev *pdev; +}; -static struct mlx4_profile mod_param_profile = { 0 }; +#define MLX4_LOG_NUM_MTT 20 +/* We limit to 30 as of a bit map issue which uses int and not uint. + see mlx4_buddy_init -> bitmap_zero which gets int. +*/ +#define MLX4_MAX_LOG_NUM_MTT 30 +static struct mlx4_profile mod_param_profile = { + .num_qp = 19, + .num_srq = 16, + .rdmarc_per_qp = 4, + .num_cq = 16, + .num_mcg = 13, + .num_mpt = 19, + .num_mtt_segs = 0, /* max(20, 2*MTTs for host memory)) */ +}; module_param_named(log_num_qp, mod_param_profile.num_qp, int, 0444); -MODULE_PARM_DESC(log_num_qp, "log maximum number of QPs per HCA"); +MODULE_PARM_DESC(log_num_qp, "log maximum number of QPs per HCA (default: 19)"); module_param_named(log_num_srq, mod_param_profile.num_srq, int, 0444); -MODULE_PARM_DESC(log_num_srq, "log maximum number of SRQs per HCA"); +MODULE_PARM_DESC(log_num_srq, "log maximum number of SRQs per HCA " + "(default: 16)"); -module_param_named(log_rdmarc_per_qp, mod_param_profile.rdmarc_per_qp, int, 0444); -MODULE_PARM_DESC(log_rdmarc_per_qp, "log number of RDMARC buffers per QP"); +module_param_named(log_rdmarc_per_qp, mod_param_profile.rdmarc_per_qp, int, + 0444); +MODULE_PARM_DESC(log_rdmarc_per_qp, "log number of RDMARC buffers per QP " + "(default: 4)"); module_param_named(log_num_cq, mod_param_profile.num_cq, int, 0444); -MODULE_PARM_DESC(log_num_cq, "log maximum number of CQs per HCA"); +MODULE_PARM_DESC(log_num_cq, "log maximum number of CQs per HCA (default: 16)"); module_param_named(log_num_mcg, mod_param_profile.num_mcg, int, 0444); -MODULE_PARM_DESC(log_num_mcg, "log maximum number of multicast groups per HCA"); +MODULE_PARM_DESC(log_num_mcg, "log maximum number of multicast groups per HCA " + "(default: 13)"); module_param_named(log_num_mpt, mod_param_profile.num_mpt, int, 0444); MODULE_PARM_DESC(log_num_mpt, - "log maximum number of memory protection table entries per HCA"); + "log maximum number of memory protection table entries per " + "HCA (default: 19)"); -module_param_named(log_num_mtt, mod_param_profile.num_mtt, int, 0444); +module_param_named(log_num_mtt, mod_param_profile.num_mtt_segs, int, 0444); MODULE_PARM_DESC(log_num_mtt, - "log maximum number of memory translation table segments per HCA"); + "log maximum number of memory translation table segments per " + "HCA (default: max(20, 2*MTTs for register all of the host memory limited to 30))"); -static int log_mtts_per_seg = 0; -module_param_named(log_mtts_per_seg, log_mtts_per_seg, int, 0444); -MODULE_PARM_DESC(log_mtts_per_seg, "Log2 number of MTT entries per segment (1-7)"); +enum { + MLX4_IF_STATE_BASIC, + MLX4_IF_STATE_EXTENDED +}; -static void process_mod_param_profile(void) +static inline u64 dbdf_to_u64(int domain, int bus, int dev, int fn) { - default_profile.num_qp = (mod_param_profile.num_qp ? - 1 << mod_param_profile.num_qp : - default_profile.num_qp); - default_profile.num_srq = (mod_param_profile.num_srq ? - 1 << mod_param_profile.num_srq : - default_profile.num_srq); - default_profile.rdmarc_per_qp = (mod_param_profile.rdmarc_per_qp ? - 1 << mod_param_profile.rdmarc_per_qp : - default_profile.rdmarc_per_qp); - default_profile.num_cq = (mod_param_profile.num_cq ? - 1 << mod_param_profile.num_cq : - default_profile.num_cq); - default_profile.num_mcg = (mod_param_profile.num_mcg ? - 1 << mod_param_profile.num_mcg : - default_profile.num_mcg); - default_profile.num_mpt = (mod_param_profile.num_mpt ? - 1 << mod_param_profile.num_mpt : - default_profile.num_mpt); - default_profile.num_mtt = (mod_param_profile.num_mtt ? - 1 << mod_param_profile.num_mtt : - default_profile.num_mtt); + return (domain << 20) | (bus << 12) | (dev << 4) | fn; } -struct mlx4_port_config +static inline void pr_bdf_err(const char *dbdf, const char *pname) { - struct list_head list; - enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1]; - struct pci_dev *pdev; -}; -static LIST_HEAD(config_list); + pr_warn("mlx4_core: '%s' is not valid bdf in '%s'\n", dbdf, pname); +} -static void mlx4_config_cleanup(void) +static inline void pr_val_err(const char *dbdf, const char *pname, + const char *val) { - struct mlx4_port_config *config, *tmp; + pr_warn("mlx4_core: value '%s' of bdf '%s' in '%s' is not valid\n" + , val, dbdf, pname); +} - list_for_each_entry_safe(config, tmp, &config_list, list) { - list_del(&config->list); - kfree(config); +static inline void pr_out_of_range_bdf(const char *dbdf, int val, + struct mlx4_dbdf2val_lst *dbdf2val) +{ + pr_warn("mlx4_core: value %d in bdf '%s' of '%s' is out of its valid range (%d,%d)\n" + , val, dbdf, dbdf2val->name , dbdf2val->range.min, + dbdf2val->range.max); +} + +static inline void pr_out_of_range(struct mlx4_dbdf2val_lst *dbdf2val) +{ + pr_warn("mlx4_core: value of '%s' is out of its valid range (%d,%d)\n" + , dbdf2val->name , dbdf2val->range.min, dbdf2val->range.max); +} + +static inline int is_in_range(int val, struct mlx4_range *r) +{ + return (val >= r->min && val <= r->max); +} + +static int update_defaults(struct param_data *pdata) +{ + long int val[MLX4_MAX_BDF_VALS]; + int ret; + char *t, *p = pdata->dbdf2val.str; + char sval[32]; + int val_len; + + if (!strlen(p) || strchr(p, ':') || strchr(p, '.') || strchr(p, ';')) + return INVALID_STR; + + switch (pdata->id) { + case PORT_TYPE_ARRAY: + t = strchr(p, ','); + if (!t || t == p || (t - p) > sizeof(sval)) + return INVALID_STR; + + val_len = t - p; + strncpy(sval, p, val_len); + sval[val_len] = 0; + + ret = kstrtol(sval, 0, &val[0]); + if (ret == -EINVAL) + return INVALID_STR; + if (ret || !is_in_range(val[0], &pdata->dbdf2val.range)) { + pr_out_of_range(&pdata->dbdf2val); + return INVALID_DATA; + } + + ret = kstrtol(t + 1, 0, &val[1]); + if (ret == -EINVAL) + return INVALID_STR; + if (ret || !is_in_range(val[1], &pdata->dbdf2val.range)) { + pr_out_of_range(&pdata->dbdf2val); + return INVALID_DATA; + } + + pdata->dbdf2val.tbl[0].val[0] = val[0]; + pdata->dbdf2val.tbl[0].val[1] = val[1]; + break; + + case NUM_VFS: + case PROBE_VF: + ret = kstrtol(p, 0, &val[0]); + if (ret == -EINVAL) + return INVALID_STR; + if (ret || !is_in_range(val[0], &pdata->dbdf2val.range)) { + pr_out_of_range(&pdata->dbdf2val); + return INVALID_DATA; + } + pdata->dbdf2val.tbl[0].val[0] = val[0]; + break; } + pdata->dbdf2val.tbl[1].dbdf = MLX4_ENDOF_TBL; + + return VALID_DATA; } -void *mlx4_get_prot_dev(struct mlx4_dev *dev, enum mlx4_prot proto, int port) +int mlx4_fill_dbdf2val_tbl(struct mlx4_dbdf2val_lst *dbdf2val_lst) { - return mlx4_find_get_prot_dev(dev, proto, port); + int domain, bus, dev, fn; + u64 dbdf; + char *p, *t, *v; + char tmp[32]; + char sbdf[32]; + char sep = ','; + int j, k, str_size, i = 1; + int prfx_size; + + p = dbdf2val_lst->str; + + for (j = 0; j < dbdf2val_lst->num_vals; j++) + dbdf2val_lst->tbl[0].val[j] = dbdf2val_lst->def_val[j]; + dbdf2val_lst->tbl[1].dbdf = MLX4_ENDOF_TBL; + + str_size = strlen(dbdf2val_lst->str); + + if (str_size == 0) + return 0; + + while (strlen(p)) { + prfx_size = BDF_STR_SIZE; + sbdf[prfx_size] = 0; + strncpy(sbdf, p, prfx_size); + domain = DEFAULT_DOMAIN; + if (sscanf(sbdf, "%02x:%02x.%x-", &bus, &dev, &fn) != 3) { + prfx_size = DBDF_STR_SIZE; + sbdf[prfx_size] = 0; + strncpy(sbdf, p, prfx_size); + if (sscanf(sbdf, "%04x:%02x:%02x.%x-", &domain, &bus, + &dev, &fn) != 4) { + pr_bdf_err(sbdf, dbdf2val_lst->name); + goto err; + } + sprintf(tmp, "%04x:%02x:%02x.%x-", domain, bus, dev, + fn); + } else { + sprintf(tmp, "%02x:%02x.%x-", bus, dev, fn); + } + + if (strnicmp(sbdf, tmp, sizeof(tmp))) { + pr_bdf_err(sbdf, dbdf2val_lst->name); + goto err; + } + + dbdf = dbdf_to_u64(domain, bus, dev, fn); + + for (j = 1; j < i; j++) + if (dbdf2val_lst->tbl[j].dbdf == dbdf) { + pr_warn("mlx4_core: in '%s', %s appears multiple times\n" + , dbdf2val_lst->name, sbdf); + goto err; + } + + if (i >= MLX4_DEVS_TBL_SIZE) { + pr_warn("mlx4_core: Too many devices in '%s'\n" + , dbdf2val_lst->name); + goto err; + } + + p += prfx_size; + t = strchr(p, sep); + t = t ? t : p + strlen(p); + if (p >= t) { + pr_val_err(sbdf, dbdf2val_lst->name, ""); + goto err; + } + + for (k = 0; k < dbdf2val_lst->num_vals; k++) { + char sval[32]; + long int val; + int ret, val_len; + char vsep = ';'; + + v = (k == dbdf2val_lst->num_vals - 1) ? t : strchr(p, vsep); + if (!v || v > t || v == p || (v - p) > sizeof(sval)) { + pr_val_err(sbdf, dbdf2val_lst->name, p); + goto err; + } + val_len = v - p; + strncpy(sval, p, val_len); + sval[val_len] = 0; + + ret = kstrtol(sval, 0, &val); + if (ret) { + if (strchr(p, vsep)) + pr_warn("mlx4_core: too many vals in bdf '%s' of '%s'\n" + , sbdf, dbdf2val_lst->name); + else + pr_val_err(sbdf, dbdf2val_lst->name, + sval); + goto err; + } + if (!is_in_range(val, &dbdf2val_lst->range)) { + pr_out_of_range_bdf(sbdf, val, dbdf2val_lst); + goto err; + } + + dbdf2val_lst->tbl[i].val[k] = val; + p = v; + if (p[0] == vsep) + p++; + } + + dbdf2val_lst->tbl[i].dbdf = dbdf; + if (strlen(p)) { + if (p[0] != sep) { + pr_warn("mlx4_core: expect separator '%c' before '%s' in '%s'\n" + , sep, p, dbdf2val_lst->name); + goto err; + } + p++; + } + i++; + if (i < MLX4_DEVS_TBL_SIZE) + dbdf2val_lst->tbl[i].dbdf = MLX4_ENDOF_TBL; + } + + return 0; + +err: + dbdf2val_lst->tbl[1].dbdf = MLX4_ENDOF_TBL; + pr_warn("mlx4_core: The value of '%s' is incorrect. The value is discarded!\n" + , dbdf2val_lst->name); + + return -EINVAL; } -EXPORT_SYMBOL(mlx4_get_prot_dev); +EXPORT_SYMBOL(mlx4_fill_dbdf2val_tbl); -void mlx4_set_iboe_counter(struct mlx4_dev *dev, int index, u8 port) +int mlx4_get_val(struct mlx4_dbdf2val *tbl, struct pci_dev *pdev, int idx, + int *val) { - struct mlx4_priv *priv = mlx4_priv(dev); + u64 dbdf; + int i = 1; - priv->iboe_counter_index[port - 1] = index; + *val = tbl[0].val[idx]; + if (!pdev) + return -EINVAL; + + if (!pdev->bus) { + return -EINVAL; + } + + dbdf = dbdf_to_u64(pci_get_domain(pdev->dev.bsddev), pci_get_bus(pdev->dev.bsddev), + PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); + + while ((i < MLX4_DEVS_TBL_SIZE) && (tbl[i].dbdf != MLX4_ENDOF_TBL)) { + if (tbl[i].dbdf == dbdf) { + *val = tbl[i].val[idx]; + return 0; + } + i++; + } + + return 0; } -EXPORT_SYMBOL(mlx4_set_iboe_counter); +EXPORT_SYMBOL(mlx4_get_val); -int mlx4_get_iboe_counter(struct mlx4_dev *dev, u8 port) +static void process_mod_param_profile(struct mlx4_profile *profile) { - struct mlx4_priv *priv = mlx4_priv(dev); + vm_size_t hwphyssz; + hwphyssz = 0; + TUNABLE_ULONG_FETCH("hw.realmem", (u_long *) &hwphyssz); - return priv->iboe_counter_index[port - 1]; + profile->num_qp = 1 << mod_param_profile.num_qp; + profile->num_srq = 1 << mod_param_profile.num_srq; + profile->rdmarc_per_qp = 1 << mod_param_profile.rdmarc_per_qp; + profile->num_cq = 1 << mod_param_profile.num_cq; + profile->num_mcg = 1 << mod_param_profile.num_mcg; + profile->num_mpt = 1 << mod_param_profile.num_mpt; + /* + * We want to scale the number of MTTs with the size of the + * system memory, since it makes sense to register a lot of + * memory on a system with a lot of memory. As a heuristic, + * make sure we have enough MTTs to register twice the system + * memory (with PAGE_SIZE entries). + * + * This number has to be a power of two and fit into 32 bits + * due to device limitations. We cap this at 2^30 as of bit map + * limitation to work with int instead of uint (mlx4_buddy_init -> bitmap_zero) + * That limits us to 4TB of memory registration per HCA with + * 4KB pages, which is probably OK for the next few months. + */ + if (mod_param_profile.num_mtt_segs) + profile->num_mtt_segs = 1 << mod_param_profile.num_mtt_segs; + else { + profile->num_mtt_segs = + roundup_pow_of_two(max_t(unsigned, + 1 << (MLX4_LOG_NUM_MTT - log_mtts_per_seg), + min(1UL << + (MLX4_MAX_LOG_NUM_MTT - + log_mtts_per_seg), + (hwphyssz << 1) + >> log_mtts_per_seg))); + /* set the actual value, so it will be reflected to the user + using the sysfs */ + mod_param_profile.num_mtt_segs = ilog2(profile->num_mtt_segs); + } } -EXPORT_SYMBOL(mlx4_get_iboe_counter); int mlx4_check_port_params(struct mlx4_dev *dev, enum mlx4_port_type *port_type) { int i; for (i = 0; i < dev->caps.num_ports - 1; i++) { if (port_type[i] != port_type[i + 1]) { if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) { mlx4_err(dev, "Only same port types supported " "on this HCA, aborting.\n"); return -EINVAL; } } } for (i = 0; i < dev->caps.num_ports; i++) { if (!(port_type[i] & dev->caps.supported_type[i+1])) { mlx4_err(dev, "Requested port type for port %d is not " "supported on this HCA\n", i + 1); return -EINVAL; } } return 0; } static void mlx4_set_port_mask(struct mlx4_dev *dev) { int i; for (i = 1; i <= dev->caps.num_ports; ++i) dev->caps.port_mask[i] = dev->caps.port_type[i]; } -static u8 get_counters_mode(u64 flags) -{ - switch (flags >> 48 & 3) { - case 2: - case 3: - return MLX4_CUNTERS_EXT; - case 1: - return MLX4_CUNTERS_BASIC; - default: - return MLX4_CUNTERS_DISABLED; - } -} - static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) { int err; int i; err = mlx4_QUERY_DEV_CAP(dev, dev_cap); if (err) { mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); return err; } if (dev_cap->min_page_sz > PAGE_SIZE) { mlx4_err(dev, "HCA minimum page size of %d bigger than " "kernel PAGE_SIZE of %d, aborting.\n", dev_cap->min_page_sz, PAGE_SIZE); return -ENODEV; } if (dev_cap->num_ports > MLX4_MAX_PORTS) { mlx4_err(dev, "HCA has %d ports, but we only support %d, " "aborting.\n", dev_cap->num_ports, MLX4_MAX_PORTS); return -ENODEV; } if (dev_cap->uar_size > pci_resource_len(dev->pdev, 2)) { mlx4_err(dev, "HCA reported UAR size of 0x%x bigger than " "PCI resource 2 size of 0x%llx, aborting.\n", dev_cap->uar_size, (unsigned long long) pci_resource_len(dev->pdev, 2)); return -ENODEV; } dev->caps.num_ports = dev_cap->num_ports; + dev->phys_caps.num_phys_eqs = MLX4_MAX_EQ_NUM; for (i = 1; i <= dev->caps.num_ports; ++i) { dev->caps.vl_cap[i] = dev_cap->max_vl[i]; dev->caps.ib_mtu_cap[i] = dev_cap->ib_mtu[i]; + dev->phys_caps.gid_phys_table_len[i] = dev_cap->max_gids[i]; + dev->phys_caps.pkey_phys_table_len[i] = dev_cap->max_pkeys[i]; + /* set gid and pkey table operating lengths by default + * to non-sriov values */ dev->caps.gid_table_len[i] = dev_cap->max_gids[i]; dev->caps.pkey_table_len[i] = dev_cap->max_pkeys[i]; dev->caps.port_width_cap[i] = dev_cap->max_port_width[i]; dev->caps.eth_mtu_cap[i] = dev_cap->eth_mtu[i]; dev->caps.def_mac[i] = dev_cap->def_mac[i]; dev->caps.supported_type[i] = dev_cap->supported_port_types[i]; + dev->caps.suggested_type[i] = dev_cap->suggested_type[i]; + dev->caps.default_sense[i] = dev_cap->default_sense[i]; dev->caps.trans_type[i] = dev_cap->trans_type[i]; dev->caps.vendor_oui[i] = dev_cap->vendor_oui[i]; dev->caps.wavelength[i] = dev_cap->wavelength[i]; dev->caps.trans_code[i] = dev_cap->trans_code[i]; } + dev->caps.uar_page_size = PAGE_SIZE; dev->caps.num_uars = dev_cap->uar_size / PAGE_SIZE; dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay; dev->caps.bf_reg_size = dev_cap->bf_reg_size; dev->caps.bf_regs_per_page = dev_cap->bf_regs_per_page; dev->caps.max_sq_sg = dev_cap->max_sq_sg; dev->caps.max_rq_sg = dev_cap->max_rq_sg; dev->caps.max_wqes = dev_cap->max_qp_sz; dev->caps.max_qp_init_rdma = dev_cap->max_requester_per_qp; dev->caps.max_srq_wqes = dev_cap->max_srq_sz; dev->caps.max_srq_sge = dev_cap->max_rq_sg - 1; dev->caps.reserved_srqs = dev_cap->reserved_srqs; dev->caps.max_sq_desc_sz = dev_cap->max_sq_desc_sz; dev->caps.max_rq_desc_sz = dev_cap->max_rq_desc_sz; - dev->caps.num_qp_per_mgm = MLX4_QP_PER_MGM; /* * Subtract 1 from the limit because we need to allocate a - * spare CQE so the HCA HW can tell the difference between an - * empty CQ and a full CQ. + * spare CQE to enable resizing the CQ */ dev->caps.max_cqes = dev_cap->max_cq_sz - 1; dev->caps.reserved_cqs = dev_cap->reserved_cqs; dev->caps.reserved_eqs = dev_cap->reserved_eqs; - dev->caps.mtts_per_seg = 1 << log_mtts_per_seg; - dev->caps.reserved_mtts = DIV_ROUND_UP(dev_cap->reserved_mtts, - dev->caps.mtts_per_seg); + dev->caps.reserved_mtts = dev_cap->reserved_mtts; dev->caps.reserved_mrws = dev_cap->reserved_mrws; - dev->caps.reserved_uars = dev_cap->reserved_uars; + + /* The first 128 UARs are used for EQ doorbells */ + dev->caps.reserved_uars = max_t(int, 128, dev_cap->reserved_uars); dev->caps.reserved_pds = dev_cap->reserved_pds; - dev->caps.mtt_entry_sz = dev->caps.mtts_per_seg * dev_cap->mtt_entry_sz; + dev->caps.reserved_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ? + dev_cap->reserved_xrcds : 0; + dev->caps.max_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ? + dev_cap->max_xrcds : 0; + dev->caps.mtt_entry_sz = dev_cap->mtt_entry_sz; + dev->caps.max_msg_sz = dev_cap->max_msg_sz; dev->caps.page_size_cap = ~(u32) (dev_cap->min_page_sz - 1); dev->caps.flags = dev_cap->flags; + dev->caps.flags2 = dev_cap->flags2; dev->caps.bmme_flags = dev_cap->bmme_flags; dev->caps.reserved_lkey = dev_cap->reserved_lkey; dev->caps.stat_rate_support = dev_cap->stat_rate_support; - dev->caps.udp_rss = dev_cap->udp_rss; - dev->caps.loopback_support = dev_cap->loopback_support; - dev->caps.wol = dev_cap->wol; + dev->caps.cq_timestamp = dev_cap->timestamp_support; dev->caps.max_gso_sz = dev_cap->max_gso_sz; - dev->caps.reserved_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ? - dev_cap->reserved_xrcds : 0; - dev->caps.max_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ? - dev_cap->max_xrcds : 0; + dev->caps.max_rss_tbl_sz = dev_cap->max_rss_tbl_sz; + /* Sense port always allowed on supported devices for ConnectX-1 and -2 */ + if (mlx4_priv(dev)->pci_dev_data & MLX4_PCI_DEV_FORCE_SENSE_PORT) + dev->caps.flags |= MLX4_DEV_CAP_FLAG_SENSE_SUPPORT; + /* Don't do sense port on multifunction devices (for now at least) */ + if (mlx4_is_mfunc(dev)) + dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_SENSE_SUPPORT; + dev->caps.log_num_macs = log_num_mac; - dev->caps.log_num_prios = use_prio ? 3 : 0; + dev->caps.log_num_vlans = MLX4_LOG_NUM_VLANS; + dev->caps.fast_drop = fast_drop ? + !!(dev->caps.flags & MLX4_DEV_CAP_FLAG_FAST_DROP) : + 0; + for (i = 1; i <= dev->caps.num_ports; ++i) { dev->caps.port_type[i] = MLX4_PORT_TYPE_NONE; if (dev->caps.supported_type[i]) { - if (dev->caps.supported_type[i] != MLX4_PORT_TYPE_ETH) - dev->caps.port_type[i] = MLX4_PORT_TYPE_IB; - else + /* if only ETH is supported - assign ETH */ + if (dev->caps.supported_type[i] == MLX4_PORT_TYPE_ETH) dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH; + /* if only IB is supported, assign IB */ + else if (dev->caps.supported_type[i] == + MLX4_PORT_TYPE_IB) + dev->caps.port_type[i] = MLX4_PORT_TYPE_IB; + else { + /* + * if IB and ETH are supported, we set the port + * type according to user selection of port type; + * if there is no user selection, take the FW hint + */ + int pta; + mlx4_get_val(port_type_array.dbdf2val.tbl, + pci_physfn(dev->pdev), i - 1, + &pta); + if (pta == MLX4_PORT_TYPE_NONE) { + dev->caps.port_type[i] = dev->caps.suggested_type[i] ? + MLX4_PORT_TYPE_ETH : MLX4_PORT_TYPE_IB; + } else if (pta == MLX4_PORT_TYPE_NA) { + mlx4_err(dev, "Port %d is valid port. " + "It is not allowed to configure its type to N/A(%d)\n", + i, MLX4_PORT_TYPE_NA); + return -EINVAL; + } else { + dev->caps.port_type[i] = pta; + } + } } - dev->caps.possible_type[i] = dev->caps.port_type[i]; + /* + * Link sensing is allowed on the port if 3 conditions are true: + * 1. Both protocols are supported on the port. + * 2. Different types are supported on the port + * 3. FW declared that it supports link sensing + */ mlx4_priv(dev)->sense.sense_allowed[i] = - dev->caps.supported_type[i] == MLX4_PORT_TYPE_AUTO; + ((dev->caps.supported_type[i] == MLX4_PORT_TYPE_AUTO) && + (dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) && + (dev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT)); + /* Disablling auto sense for default Eth ports support */ + mlx4_priv(dev)->sense.sense_allowed[i] = 0; + + /* + * If "default_sense" bit is set, we move the port to "AUTO" mode + * and perform sense_port FW command to try and set the correct + * port type from beginning + */ + if (mlx4_priv(dev)->sense.sense_allowed[i] && dev->caps.default_sense[i]) { + enum mlx4_port_type sensed_port = MLX4_PORT_TYPE_NONE; + dev->caps.possible_type[i] = MLX4_PORT_TYPE_AUTO; + mlx4_SENSE_PORT(dev, i, &sensed_port); + if (sensed_port != MLX4_PORT_TYPE_NONE) + dev->caps.port_type[i] = sensed_port; + } else { + dev->caps.possible_type[i] = dev->caps.port_type[i]; + } + if (dev->caps.log_num_macs > dev_cap->log_max_macs[i]) { dev->caps.log_num_macs = dev_cap->log_max_macs[i]; mlx4_warn(dev, "Requested number of MACs is too much " "for port %d, reducing to %d.\n", i, 1 << dev->caps.log_num_macs); } - dev->caps.log_num_vlans = dev_cap->log_max_vlans[i]; + if (dev->caps.log_num_vlans > dev_cap->log_max_vlans[i]) { + dev->caps.log_num_vlans = dev_cap->log_max_vlans[i]; + mlx4_warn(dev, "Requested number of VLANs is too much " + "for port %d, reducing to %d.\n", + i, 1 << dev->caps.log_num_vlans); + } } - dev->caps.counters_mode = get_counters_mode(dev_cap->flags); - dev->caps.max_basic_counters = 1 << ilog2(dev_cap->max_basic_counters); - dev->caps.max_ext_counters = 1 << ilog2(dev_cap->max_ext_counters); + dev->caps.max_basic_counters = dev_cap->max_basic_counters; + dev->caps.max_extended_counters = dev_cap->max_extended_counters; + /* support extended counters if available */ + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS_EXT) + dev->caps.max_counters = dev->caps.max_extended_counters; + else + dev->caps.max_counters = dev->caps.max_basic_counters; dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] = dev_cap->reserved_qps; dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] = (1 << dev->caps.log_num_macs) * (1 << dev->caps.log_num_vlans) * - (1 << dev->caps.log_num_prios) * dev->caps.num_ports; + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH] = MLX4_NUM_FEXCH; dev->caps.reserved_qps = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] + - dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR]; + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] + + dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH]; + dev->caps.sync_qp = dev_cap->sync_qp; + if (dev->pdev->device == 0x1003) + dev->caps.cq_flags |= MLX4_DEV_CAP_CQ_FLAG_IO; + + dev->caps.sqp_demux = (mlx4_is_master(dev)) ? MLX4_MAX_NUM_SLAVES : 0; + + if (!mlx4_enable_64b_cqe_eqe && !mlx4_is_slave(dev)) { + if (dev_cap->flags & + (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) { + mlx4_warn(dev, "64B EQEs/CQEs supported by the device but not enabled\n"); + dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE; + dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE; + } + } + + if ((dev->caps.flags & + (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) && + mlx4_is_master(dev)) + dev->caps.function_caps |= MLX4_FUNC_CAP_64B_EQE_CQE; + + if (!mlx4_is_slave(dev)) { + for (i = 0; i < dev->caps.num_ports; ++i) + dev->caps.def_counter_index[i] = i << 1; + } + return 0; } - -static int mlx4_save_config(struct mlx4_dev *dev) +/*The function checks if there are live vf, return the num of them*/ +static int mlx4_how_many_lives_vf(struct mlx4_dev *dev) { - struct mlx4_port_config *config; + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *s_state; int i; + int ret = 0; - list_for_each_entry(config, &config_list, list) { - if (config->pdev == dev->pdev) { - for (i = 1; i <= dev->caps.num_ports; i++) - config->port_type[i] = dev->caps.possible_type[i]; - return 0; + for (i = 1/*the ppf is 0*/; i < dev->num_slaves; ++i) { + s_state = &priv->mfunc.master.slave_state[i]; + if (s_state->active && s_state->last_cmd != + MLX4_COMM_CMD_RESET) { + mlx4_warn(dev, "%s: slave: %d is still active\n", + __func__, i); + ret++; } } + return ret; +} - config = kmalloc(sizeof(struct mlx4_port_config), GFP_KERNEL); - if (!config) - return -ENOMEM; +int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey) +{ + u32 qk = MLX4_RESERVED_QKEY_BASE; - config->pdev = dev->pdev; - for (i = 1; i <= dev->caps.num_ports; i++) - config->port_type[i] = dev->caps.possible_type[i]; + if (qpn >= dev->phys_caps.base_tunnel_sqpn + 8 * MLX4_MFUNC_MAX || + qpn < dev->phys_caps.base_proxy_sqpn) + return -EINVAL; - list_add_tail(&config->list, &config_list); + if (qpn >= dev->phys_caps.base_tunnel_sqpn) + /* tunnel qp */ + qk += qpn - dev->phys_caps.base_tunnel_sqpn; + else + qk += qpn - dev->phys_caps.base_proxy_sqpn; + *qkey = qk; + return 0; +} +EXPORT_SYMBOL(mlx4_get_parav_qkey); +void mlx4_sync_pkey_table(struct mlx4_dev *dev, int slave, int port, int i, int val) +{ + struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev); + + if (!mlx4_is_master(dev)) + return; + + priv->virt2phys_pkey[slave][port - 1][i] = val; +} +EXPORT_SYMBOL(mlx4_sync_pkey_table); + +void mlx4_put_slave_node_guid(struct mlx4_dev *dev, int slave, __be64 guid) +{ + struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev); + + if (!mlx4_is_master(dev)) + return; + + priv->slave_node_guids[slave] = guid; +} +EXPORT_SYMBOL(mlx4_put_slave_node_guid); + +__be64 mlx4_get_slave_node_guid(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev); + + if (!mlx4_is_master(dev)) + return 0; + + return priv->slave_node_guids[slave]; +} +EXPORT_SYMBOL(mlx4_get_slave_node_guid); + +int mlx4_is_slave_active(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *s_slave; + + if (!mlx4_is_master(dev)) + return 0; + + s_slave = &priv->mfunc.master.slave_state[slave]; + return !!s_slave->active; +} +EXPORT_SYMBOL(mlx4_is_slave_active); + +static void slave_adjust_steering_mode(struct mlx4_dev *dev, + struct mlx4_dev_cap *dev_cap, + struct mlx4_init_hca_param *hca_param) +{ + dev->caps.steering_mode = hca_param->steering_mode; + if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) + dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry; + else + dev->caps.num_qp_per_mgm = + 4 * ((1 << hca_param->log_mc_entry_sz)/16 - 2); + + mlx4_dbg(dev, "Steering mode is: %s\n", + mlx4_steering_mode_str(dev->caps.steering_mode)); +} + +static int mlx4_slave_cap(struct mlx4_dev *dev) +{ + int err; + u32 page_size; + struct mlx4_dev_cap dev_cap; + struct mlx4_func_cap func_cap; + struct mlx4_init_hca_param hca_param; + int i; + + memset(&hca_param, 0, sizeof(hca_param)); + err = mlx4_QUERY_HCA(dev, &hca_param); + if (err) { + mlx4_err(dev, "QUERY_HCA command failed, aborting.\n"); + return err; + } + + /*fail if the hca has an unknown capability */ + if ((hca_param.global_caps | HCA_GLOBAL_CAP_MASK) != + HCA_GLOBAL_CAP_MASK) { + mlx4_err(dev, "Unknown hca global capabilities\n"); + return -ENOSYS; + } + + mlx4_log_num_mgm_entry_size = hca_param.log_mc_entry_sz; + + dev->caps.hca_core_clock = hca_param.hca_core_clock; + + memset(&dev_cap, 0, sizeof(dev_cap)); + dev->caps.max_qp_dest_rdma = 1 << hca_param.log_rd_per_qp; + err = mlx4_dev_cap(dev, &dev_cap); + if (err) { + mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); + return err; + } + + err = mlx4_QUERY_FW(dev); + if (err) + mlx4_err(dev, "QUERY_FW command failed: could not get FW version.\n"); + + if (!hca_param.mw_enable) { + dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_MEM_WINDOW; + dev->caps.bmme_flags &= ~MLX4_BMME_FLAG_TYPE_2_WIN; + } + + page_size = ~dev->caps.page_size_cap + 1; + mlx4_warn(dev, "HCA minimum page size:%d\n", page_size); + if (page_size > PAGE_SIZE) { + mlx4_err(dev, "HCA minimum page size of %d bigger than " + "kernel PAGE_SIZE of %d, aborting.\n", + page_size, PAGE_SIZE); + return -ENODEV; + } + + /* slave gets uar page size from QUERY_HCA fw command */ + dev->caps.uar_page_size = 1 << (hca_param.uar_page_sz + 12); + + /* TODO: relax this assumption */ + if (dev->caps.uar_page_size != PAGE_SIZE) { + mlx4_err(dev, "UAR size:%d != kernel PAGE_SIZE of %d\n", + dev->caps.uar_page_size, PAGE_SIZE); + return -ENODEV; + } + + memset(&func_cap, 0, sizeof(func_cap)); + err = mlx4_QUERY_FUNC_CAP(dev, 0, &func_cap); + if (err) { + mlx4_err(dev, "QUERY_FUNC_CAP general command failed, aborting (%d).\n", + err); + return err; + } + + if ((func_cap.pf_context_behaviour | PF_CONTEXT_BEHAVIOUR_MASK) != + PF_CONTEXT_BEHAVIOUR_MASK) { + mlx4_err(dev, "Unknown pf context behaviour\n"); + return -ENOSYS; + } + + dev->caps.num_ports = func_cap.num_ports; + dev->quotas.qp = func_cap.qp_quota; + dev->quotas.srq = func_cap.srq_quota; + dev->quotas.cq = func_cap.cq_quota; + dev->quotas.mpt = func_cap.mpt_quota; + dev->quotas.mtt = func_cap.mtt_quota; + dev->caps.num_qps = 1 << hca_param.log_num_qps; + dev->caps.num_srqs = 1 << hca_param.log_num_srqs; + dev->caps.num_cqs = 1 << hca_param.log_num_cqs; + dev->caps.num_mpts = 1 << hca_param.log_mpt_sz; + dev->caps.num_eqs = func_cap.max_eq; + dev->caps.reserved_eqs = func_cap.reserved_eq; + dev->caps.num_pds = MLX4_NUM_PDS; + dev->caps.num_mgms = 0; + dev->caps.num_amgms = 0; + + if (dev->caps.num_ports > MLX4_MAX_PORTS) { + mlx4_err(dev, "HCA has %d ports, but we only support %d, " + "aborting.\n", dev->caps.num_ports, MLX4_MAX_PORTS); + return -ENODEV; + } + + dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + + if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy || + !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy) { + err = -ENOMEM; + goto err_mem; + } + + for (i = 1; i <= dev->caps.num_ports; ++i) { + err = mlx4_QUERY_FUNC_CAP(dev, (u32) i, &func_cap); + if (err) { + mlx4_err(dev, "QUERY_FUNC_CAP port command failed for" + " port %d, aborting (%d).\n", i, err); + goto err_mem; + } + dev->caps.qp0_tunnel[i - 1] = func_cap.qp0_tunnel_qpn; + dev->caps.qp0_proxy[i - 1] = func_cap.qp0_proxy_qpn; + dev->caps.qp1_tunnel[i - 1] = func_cap.qp1_tunnel_qpn; + dev->caps.qp1_proxy[i - 1] = func_cap.qp1_proxy_qpn; + dev->caps.def_counter_index[i - 1] = func_cap.def_counter_index; + + dev->caps.port_mask[i] = dev->caps.port_type[i]; + err = mlx4_get_slave_pkey_gid_tbl_len(dev, i, + &dev->caps.gid_table_len[i], + &dev->caps.pkey_table_len[i]); + if (err) + goto err_mem; + } + + if (dev->caps.uar_page_size * (dev->caps.num_uars - + dev->caps.reserved_uars) > + pci_resource_len(dev->pdev, 2)) { + mlx4_err(dev, "HCA reported UAR region size of 0x%x bigger than " + "PCI resource 2 size of 0x%llx, aborting.\n", + dev->caps.uar_page_size * dev->caps.num_uars, + (unsigned long long) pci_resource_len(dev->pdev, 2)); + err = -ENOMEM; + goto err_mem; + } + + if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_EQE_ENABLED) { + dev->caps.eqe_size = 64; + dev->caps.eqe_factor = 1; + } else { + dev->caps.eqe_size = 32; + dev->caps.eqe_factor = 0; + } + + if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_CQE_ENABLED) { + dev->caps.cqe_size = 64; + dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_64B_CQE; + } else { + dev->caps.cqe_size = 32; + } + + dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; + mlx4_warn(dev, "Timestamping is not supported in slave mode.\n"); + + slave_adjust_steering_mode(dev, &dev_cap, &hca_param); + return 0; + +err_mem: + kfree(dev->caps.qp0_tunnel); + kfree(dev->caps.qp0_proxy); + kfree(dev->caps.qp1_tunnel); + kfree(dev->caps.qp1_proxy); + dev->caps.qp0_tunnel = dev->caps.qp0_proxy = + dev->caps.qp1_tunnel = dev->caps.qp1_proxy = NULL; + + return err; } +static void mlx4_request_modules(struct mlx4_dev *dev) +{ + int port; + int has_ib_port = false; + int has_eth_port = false; +#define EN_DRV_NAME "mlx4_en" +#define IB_DRV_NAME "mlx4_ib" + + for (port = 1; port <= dev->caps.num_ports; port++) { + if (dev->caps.port_type[port] == MLX4_PORT_TYPE_IB) + has_ib_port = true; + else if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) + has_eth_port = true; + } + + if (has_ib_port) + request_module_nowait(IB_DRV_NAME); + if (has_eth_port) + request_module_nowait(EN_DRV_NAME); +} + /* * Change the port configuration of the device. * Every user of this function must hold the port mutex. */ int mlx4_change_port_types(struct mlx4_dev *dev, enum mlx4_port_type *port_types) { int err = 0; int change = 0; int port; for (port = 0; port < dev->caps.num_ports; port++) { /* Change the port type only if the new type is different * from the current, and not set to Auto */ - if (port_types[port] != dev->caps.port_type[port + 1]) { + if (port_types[port] != dev->caps.port_type[port + 1]) change = 1; - dev->caps.port_type[port + 1] = port_types[port]; - } } if (change) { mlx4_unregister_device(dev); for (port = 1; port <= dev->caps.num_ports; port++) { mlx4_CLOSE_PORT(dev, port); - err = mlx4_SET_PORT(dev, port); + dev->caps.port_type[port] = port_types[port - 1]; + err = mlx4_SET_PORT(dev, port, -1); if (err) { mlx4_err(dev, "Failed to set port %d, " "aborting\n", port); goto out; } } mlx4_set_port_mask(dev); - mlx4_save_config(dev); err = mlx4_register_device(dev); + if (err) { + mlx4_err(dev, "Failed to register device\n"); + goto out; + } + mlx4_request_modules(dev); } out: return err; } static ssize_t show_port_type(struct device *dev, struct device_attribute *attr, char *buf) { struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, port_attr); struct mlx4_dev *mdev = info->dev; char type[8]; sprintf(type, "%s", (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB) ? "ib" : "eth"); if (mdev->caps.possible_type[info->port] == MLX4_PORT_TYPE_AUTO) sprintf(buf, "auto (%s)\n", type); else sprintf(buf, "%s\n", type); return strlen(buf); } static ssize_t set_port_type(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, port_attr); struct mlx4_dev *mdev = info->dev; struct mlx4_priv *priv = mlx4_priv(mdev); enum mlx4_port_type types[MLX4_MAX_PORTS]; enum mlx4_port_type new_types[MLX4_MAX_PORTS]; int i; int err = 0; if (!strcmp(buf, "ib\n")) info->tmp_type = MLX4_PORT_TYPE_IB; else if (!strcmp(buf, "eth\n")) info->tmp_type = MLX4_PORT_TYPE_ETH; else if (!strcmp(buf, "auto\n")) info->tmp_type = MLX4_PORT_TYPE_AUTO; else { mlx4_err(mdev, "%s is not supported port type\n", buf); return -EINVAL; } + if ((info->tmp_type & mdev->caps.supported_type[info->port]) != + info->tmp_type) { + mlx4_err(mdev, "Requested port type for port %d is not supported on this HCA\n", + info->port); + return -EINVAL; + } + mlx4_stop_sense(mdev); mutex_lock(&priv->port_mutex); /* Possible type is always the one that was delivered */ mdev->caps.possible_type[info->port] = info->tmp_type; for (i = 0; i < mdev->caps.num_ports; i++) { types[i] = priv->port[i+1].tmp_type ? priv->port[i+1].tmp_type : mdev->caps.possible_type[i+1]; if (types[i] == MLX4_PORT_TYPE_AUTO) types[i] = mdev->caps.port_type[i+1]; } - if (priv->trig) { - if (++priv->changed_ports < mdev->caps.num_ports) - goto out; - else - priv->trig = priv->changed_ports = 0; - } - - if (!(mdev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) { + if (!(mdev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) && + !(mdev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT)) { for (i = 1; i <= mdev->caps.num_ports; i++) { if (mdev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) { mdev->caps.possible_type[i] = mdev->caps.port_type[i]; err = -EINVAL; } } } if (err) { mlx4_err(mdev, "Auto sensing is not supported on this HCA. " "Set only 'eth' or 'ib' for both ports " "(should be the same)\n"); goto out; } mlx4_do_sense_ports(mdev, new_types, types); err = mlx4_check_port_params(mdev, new_types); if (err) goto out; /* We are about to apply the changes after the configuration * was verified, no need to remember the temporary types * any more */ for (i = 0; i < mdev->caps.num_ports; i++) priv->port[i + 1].tmp_type = 0; err = mlx4_change_port_types(mdev, new_types); out: mlx4_start_sense(mdev); mutex_unlock(&priv->port_mutex); return err ? err : count; } -static ssize_t trigger_port(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) +enum ibta_mtu { + IB_MTU_256 = 1, + IB_MTU_512 = 2, + IB_MTU_1024 = 3, + IB_MTU_2048 = 4, + IB_MTU_4096 = 5 +}; + +static inline int int_to_ibta_mtu(int mtu) { - struct pci_dev *pdev = to_pci_dev(dev); - struct mlx4_dev *mdev = pci_get_drvdata(pdev); - struct mlx4_priv *priv = container_of(mdev, struct mlx4_priv, dev); + switch (mtu) { + case 256: return IB_MTU_256; + case 512: return IB_MTU_512; + case 1024: return IB_MTU_1024; + case 2048: return IB_MTU_2048; + case 4096: return IB_MTU_4096; + default: return -1; + } +} - if (!priv) - return -ENODEV; +static inline int ibta_mtu_to_int(enum ibta_mtu mtu) +{ + switch (mtu) { + case IB_MTU_256: return 256; + case IB_MTU_512: return 512; + case IB_MTU_1024: return 1024; + case IB_MTU_2048: return 2048; + case IB_MTU_4096: return 4096; + default: return -1; + } +} +static ssize_t show_port_ib_mtu(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, + port_mtu_attr); + struct mlx4_dev *mdev = info->dev; + + if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH) + mlx4_warn(mdev, "port level mtu is only used for IB ports\n"); + + sprintf(buf, "%d\n", + ibta_mtu_to_int(mdev->caps.port_ib_mtu[info->port])); + return strlen(buf); +} + +static ssize_t set_port_ib_mtu(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info, + port_mtu_attr); + struct mlx4_dev *mdev = info->dev; + struct mlx4_priv *priv = mlx4_priv(mdev); + int err, port, mtu, ibta_mtu = -1; + + if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH) { + mlx4_warn(mdev, "port level mtu is only used for IB ports\n"); + return -EINVAL; + } + + mtu = (int) simple_strtol(buf, NULL, 0); + ibta_mtu = int_to_ibta_mtu(mtu); + + if (ibta_mtu < 0) { + mlx4_err(mdev, "%s is invalid IBTA mtu\n", buf); + return -EINVAL; + } + + mdev->caps.port_ib_mtu[info->port] = ibta_mtu; + + mlx4_stop_sense(mdev); mutex_lock(&priv->port_mutex); - priv->trig = 1; + mlx4_unregister_device(mdev); + for (port = 1; port <= mdev->caps.num_ports; port++) { + mlx4_CLOSE_PORT(mdev, port); + err = mlx4_SET_PORT(mdev, port, -1); + if (err) { + mlx4_err(mdev, "Failed to set port %d, " + "aborting\n", port); + goto err_set_port; + } + } + err = mlx4_register_device(mdev); +err_set_port: mutex_unlock(&priv->port_mutex); - return count; + mlx4_start_sense(mdev); + return err ? err : count; } -DEVICE_ATTR(port_trigger, S_IWUGO, NULL, trigger_port); static int mlx4_load_fw(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); - int err; + int err, unmap_flag = 0; priv->fw.fw_icm = mlx4_alloc_icm(dev, priv->fw.fw_pages, GFP_HIGHUSER | __GFP_NOWARN, 0); if (!priv->fw.fw_icm) { mlx4_err(dev, "Couldn't allocate FW area, aborting.\n"); return -ENOMEM; } err = mlx4_MAP_FA(dev, priv->fw.fw_icm); if (err) { mlx4_err(dev, "MAP_FA command failed, aborting.\n"); goto err_free; } err = mlx4_RUN_FW(dev); if (err) { mlx4_err(dev, "RUN_FW command failed, aborting.\n"); goto err_unmap_fa; } return 0; err_unmap_fa: - mlx4_UNMAP_FA(dev); + unmap_flag = mlx4_UNMAP_FA(dev); + if (unmap_flag) + pr_warn("mlx4_core: mlx4_UNMAP_FA failed.\n"); err_free: - mlx4_free_icm(dev, priv->fw.fw_icm, 0); + if (!unmap_flag) + mlx4_free_icm(dev, priv->fw.fw_icm, 0); return err; } static int mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base, int cmpt_entry_sz) { struct mlx4_priv *priv = mlx4_priv(dev); int err; + int num_eqs; err = mlx4_init_icm_table(dev, &priv->qp_table.cmpt_table, cmpt_base + ((u64) (MLX4_CMPT_TYPE_QP * cmpt_entry_sz) << MLX4_CMPT_SHIFT), cmpt_entry_sz, dev->caps.num_qps, dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], 0, 0); if (err) goto err; err = mlx4_init_icm_table(dev, &priv->srq_table.cmpt_table, cmpt_base + ((u64) (MLX4_CMPT_TYPE_SRQ * cmpt_entry_sz) << MLX4_CMPT_SHIFT), cmpt_entry_sz, dev->caps.num_srqs, dev->caps.reserved_srqs, 0, 0); if (err) goto err_qp; err = mlx4_init_icm_table(dev, &priv->cq_table.cmpt_table, cmpt_base + ((u64) (MLX4_CMPT_TYPE_CQ * cmpt_entry_sz) << MLX4_CMPT_SHIFT), cmpt_entry_sz, dev->caps.num_cqs, dev->caps.reserved_cqs, 0, 0); if (err) goto err_srq; + num_eqs = (mlx4_is_master(dev)) ? dev->phys_caps.num_phys_eqs : + dev->caps.num_eqs; err = mlx4_init_icm_table(dev, &priv->eq_table.cmpt_table, cmpt_base + ((u64) (MLX4_CMPT_TYPE_EQ * cmpt_entry_sz) << MLX4_CMPT_SHIFT), - cmpt_entry_sz, - dev->caps.num_eqs, dev->caps.num_eqs, 0, 0); + cmpt_entry_sz, num_eqs, num_eqs, 0, 0); if (err) goto err_cq; return 0; err_cq: mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); err_srq: mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); err_qp: mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); err: return err; } static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap, struct mlx4_init_hca_param *init_hca, u64 icm_size) { struct mlx4_priv *priv = mlx4_priv(dev); u64 aux_pages; - int err; + int num_eqs; + int err, unmap_flag = 0; err = mlx4_SET_ICM_SIZE(dev, icm_size, &aux_pages); if (err) { mlx4_err(dev, "SET_ICM_SIZE command failed, aborting.\n"); return err; } mlx4_dbg(dev, "%lld KB of HCA context requires %lld KB aux memory.\n", (unsigned long long) icm_size >> 10, (unsigned long long) aux_pages << 2); priv->fw.aux_icm = mlx4_alloc_icm(dev, aux_pages, GFP_HIGHUSER | __GFP_NOWARN, 0); if (!priv->fw.aux_icm) { mlx4_err(dev, "Couldn't allocate aux memory, aborting.\n"); return -ENOMEM; } err = mlx4_MAP_ICM_AUX(dev, priv->fw.aux_icm); if (err) { mlx4_err(dev, "MAP_ICM_AUX command failed, aborting.\n"); goto err_free_aux; } err = mlx4_init_cmpt_table(dev, init_hca->cmpt_base, dev_cap->cmpt_entry_sz); if (err) { mlx4_err(dev, "Failed to map cMPT context memory, aborting.\n"); goto err_unmap_aux; } + + num_eqs = (mlx4_is_master(dev)) ? dev->phys_caps.num_phys_eqs : + dev->caps.num_eqs; err = mlx4_init_icm_table(dev, &priv->eq_table.table, init_hca->eqc_base, dev_cap->eqc_entry_sz, - dev->caps.num_eqs, dev->caps.num_eqs, - 0, 0); + num_eqs, num_eqs, 0, 0); if (err) { mlx4_err(dev, "Failed to map EQ context memory, aborting.\n"); goto err_unmap_cmpt; } /* * Reserved MTT entries must be aligned up to a cacheline * boundary, since the FW will write to them, while the driver * writes to all other MTT entries. (The variable * dev->caps.mtt_entry_sz below is really the MTT segment * size, not the raw entry size) */ dev->caps.reserved_mtts = ALIGN(dev->caps.reserved_mtts * dev->caps.mtt_entry_sz, dma_get_cache_alignment()) / dev->caps.mtt_entry_sz; err = mlx4_init_icm_table(dev, &priv->mr_table.mtt_table, init_hca->mtt_base, dev->caps.mtt_entry_sz, - dev->caps.num_mtt_segs, + dev->caps.num_mtts, dev->caps.reserved_mtts, 1, 0); if (err) { mlx4_err(dev, "Failed to map MTT context memory, aborting.\n"); goto err_unmap_eq; } err = mlx4_init_icm_table(dev, &priv->mr_table.dmpt_table, init_hca->dmpt_base, dev_cap->dmpt_entry_sz, dev->caps.num_mpts, dev->caps.reserved_mrws, 1, 1); if (err) { mlx4_err(dev, "Failed to map dMPT context memory, aborting.\n"); goto err_unmap_mtt; } err = mlx4_init_icm_table(dev, &priv->qp_table.qp_table, init_hca->qpc_base, dev_cap->qpc_entry_sz, dev->caps.num_qps, dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], 0, 0); if (err) { mlx4_err(dev, "Failed to map QP context memory, aborting.\n"); goto err_unmap_dmpt; } err = mlx4_init_icm_table(dev, &priv->qp_table.auxc_table, init_hca->auxc_base, dev_cap->aux_entry_sz, dev->caps.num_qps, dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], 0, 0); if (err) { mlx4_err(dev, "Failed to map AUXC context memory, aborting.\n"); goto err_unmap_qp; } err = mlx4_init_icm_table(dev, &priv->qp_table.altc_table, init_hca->altc_base, dev_cap->altc_entry_sz, dev->caps.num_qps, dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], 0, 0); if (err) { mlx4_err(dev, "Failed to map ALTC context memory, aborting.\n"); goto err_unmap_auxc; } err = mlx4_init_icm_table(dev, &priv->qp_table.rdmarc_table, init_hca->rdmarc_base, dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift, dev->caps.num_qps, dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], 0, 0); if (err) { mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n"); goto err_unmap_altc; } err = mlx4_init_icm_table(dev, &priv->cq_table.table, init_hca->cqc_base, dev_cap->cqc_entry_sz, dev->caps.num_cqs, dev->caps.reserved_cqs, 0, 0); if (err) { mlx4_err(dev, "Failed to map CQ context memory, aborting.\n"); goto err_unmap_rdmarc; } err = mlx4_init_icm_table(dev, &priv->srq_table.table, init_hca->srqc_base, dev_cap->srq_entry_sz, dev->caps.num_srqs, dev->caps.reserved_srqs, 0, 0); if (err) { mlx4_err(dev, "Failed to map SRQ context memory, aborting.\n"); goto err_unmap_cq; } /* - * It's not strictly required, but for simplicity just map the - * whole multicast group table now. The table isn't very big - * and it's a lot easier than trying to track ref counts. + * For flow steering device managed mode it is required to use + * mlx4_init_icm_table. For B0 steering mode it's not strictly + * required, but for simplicity just map the whole multicast + * group table now. The table isn't very big and it's a lot + * easier than trying to track ref counts. */ err = mlx4_init_icm_table(dev, &priv->mcg_table.table, - init_hca->mc_base, MLX4_MGM_ENTRY_SIZE, + init_hca->mc_base, + mlx4_get_mgm_entry_size(dev), dev->caps.num_mgms + dev->caps.num_amgms, dev->caps.num_mgms + dev->caps.num_amgms, 0, 0); if (err) { mlx4_err(dev, "Failed to map MCG context memory, aborting.\n"); goto err_unmap_srq; } return 0; err_unmap_srq: mlx4_cleanup_icm_table(dev, &priv->srq_table.table); err_unmap_cq: mlx4_cleanup_icm_table(dev, &priv->cq_table.table); err_unmap_rdmarc: mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table); err_unmap_altc: mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table); err_unmap_auxc: mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table); err_unmap_qp: mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table); err_unmap_dmpt: mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table); err_unmap_mtt: mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table); err_unmap_eq: mlx4_cleanup_icm_table(dev, &priv->eq_table.table); err_unmap_cmpt: mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table); mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); err_unmap_aux: - mlx4_UNMAP_ICM_AUX(dev); + unmap_flag = mlx4_UNMAP_ICM_AUX(dev); + if (unmap_flag) + pr_warn("mlx4_core: mlx4_UNMAP_ICM_AUX failed.\n"); err_free_aux: - mlx4_free_icm(dev, priv->fw.aux_icm, 0); + if (!unmap_flag) + mlx4_free_icm(dev, priv->fw.aux_icm, 0); return err; } static void mlx4_free_icms(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); mlx4_cleanup_icm_table(dev, &priv->mcg_table.table); mlx4_cleanup_icm_table(dev, &priv->srq_table.table); mlx4_cleanup_icm_table(dev, &priv->cq_table.table); mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table); mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table); mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table); mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table); mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table); mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table); mlx4_cleanup_icm_table(dev, &priv->eq_table.table); mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table); mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table); mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table); mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table); - mlx4_UNMAP_ICM_AUX(dev); - mlx4_free_icm(dev, priv->fw.aux_icm, 0); + if (!mlx4_UNMAP_ICM_AUX(dev)) + mlx4_free_icm(dev, priv->fw.aux_icm, 0); + else + pr_warn("mlx4_core: mlx4_UNMAP_ICM_AUX failed.\n"); } +static void mlx4_slave_exit(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + mutex_lock(&priv->cmd.slave_cmd_mutex); + if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_TIME)) + mlx4_warn(dev, "Failed to close slave function.\n"); + mutex_unlock(&priv->cmd.slave_cmd_mutex); +} + static int map_bf_area(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); resource_size_t bf_start; resource_size_t bf_len; int err = 0; - bf_start = pci_resource_start(dev->pdev, 2) + (dev->caps.num_uars << PAGE_SHIFT); - bf_len = pci_resource_len(dev->pdev, 2) - (dev->caps.num_uars << PAGE_SHIFT); + if (!dev->caps.bf_reg_size) + return -ENXIO; + + bf_start = pci_resource_start(dev->pdev, 2) + + (dev->caps.num_uars << PAGE_SHIFT); + bf_len = pci_resource_len(dev->pdev, 2) - + (dev->caps.num_uars << PAGE_SHIFT); priv->bf_mapping = io_mapping_create_wc(bf_start, bf_len); if (!priv->bf_mapping) err = -ENOMEM; return err; } static void unmap_bf_area(struct mlx4_dev *dev) { if (mlx4_priv(dev)->bf_mapping) io_mapping_free(mlx4_priv(dev)->bf_mapping); } +int mlx4_read_clock(struct mlx4_dev *dev) +{ + u32 clockhi, clocklo, clockhi1; + cycle_t cycles; + int i; + struct mlx4_priv *priv = mlx4_priv(dev); + + if (!priv->clock_mapping) + return -ENOTSUPP; + + for (i = 0; i < 10; i++) { + clockhi = swab32(readl(priv->clock_mapping)); + clocklo = swab32(readl(priv->clock_mapping + 4)); + clockhi1 = swab32(readl(priv->clock_mapping)); + if (clockhi == clockhi1) + break; + } + + cycles = (u64) clockhi << 32 | (u64) clocklo; + + return cycles; +} +EXPORT_SYMBOL_GPL(mlx4_read_clock); + + +static int map_internal_clock(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + priv->clock_mapping = ioremap(pci_resource_start(dev->pdev, + priv->fw.clock_bar) + + priv->fw.clock_offset, MLX4_CLOCK_SIZE); + + if (!priv->clock_mapping) + return -ENOMEM; + + return 0; +} + + +int mlx4_get_internal_clock_params(struct mlx4_dev *dev, + struct mlx4_clock_params *params) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + if (mlx4_is_slave(dev)) + return -ENOTSUPP; + if (!params) + return -EINVAL; + + params->bar = priv->fw.clock_bar; + params->offset = priv->fw.clock_offset; + params->size = MLX4_CLOCK_SIZE; + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_get_internal_clock_params); + +static void unmap_internal_clock(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + if (priv->clock_mapping) + iounmap(priv->clock_mapping); +} + static void mlx4_close_hca(struct mlx4_dev *dev) { + unmap_internal_clock(dev); unmap_bf_area(dev); - mlx4_CLOSE_HCA(dev, 0); - mlx4_free_icms(dev); - mlx4_UNMAP_FA(dev); - mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm, 0); + if (mlx4_is_slave(dev)) { + mlx4_slave_exit(dev); + } else { + mlx4_CLOSE_HCA(dev, 0); + mlx4_free_icms(dev); + + if (!mlx4_UNMAP_FA(dev)) + mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm, 0); + else + pr_warn("mlx4_core: mlx4_UNMAP_FA failed.\n"); + } } +static int mlx4_init_slave(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + u64 dma = (u64) priv->mfunc.vhcr_dma; + int num_of_reset_retries = NUM_OF_RESET_RETRIES; + int ret_from_reset = 0; + u32 slave_read; + u32 cmd_channel_ver; + + mutex_lock(&priv->cmd.slave_cmd_mutex); + priv->cmd.max_cmds = 1; + mlx4_warn(dev, "Sending reset\n"); + ret_from_reset = mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, + MLX4_COMM_TIME); + /* if we are in the middle of flr the slave will try + * NUM_OF_RESET_RETRIES times before leaving.*/ + if (ret_from_reset) { + if (MLX4_DELAY_RESET_SLAVE == ret_from_reset) { + msleep(SLEEP_TIME_IN_RESET); + while (ret_from_reset && num_of_reset_retries) { + mlx4_warn(dev, "slave is currently in the" + "middle of FLR. retrying..." + "(try num:%d)\n", + (NUM_OF_RESET_RETRIES - + num_of_reset_retries + 1)); + ret_from_reset = + mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, + 0, MLX4_COMM_TIME); + num_of_reset_retries = num_of_reset_retries - 1; + } + } else + goto err; + } + + /* check the driver version - the slave I/F revision + * must match the master's */ + slave_read = swab32(readl(&priv->mfunc.comm->slave_read)); + cmd_channel_ver = mlx4_comm_get_version(); + + if (MLX4_COMM_GET_IF_REV(cmd_channel_ver) != + MLX4_COMM_GET_IF_REV(slave_read)) { + mlx4_err(dev, "slave driver version is not supported" + " by the master\n"); + goto err; + } + + mlx4_warn(dev, "Sending vhcr0\n"); + if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR0, dma >> 48, + MLX4_COMM_TIME)) + goto err; + if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR1, dma >> 32, + MLX4_COMM_TIME)) + goto err; + if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR2, dma >> 16, + MLX4_COMM_TIME)) + goto err; + if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR_EN, dma, MLX4_COMM_TIME)) + goto err; + + mutex_unlock(&priv->cmd.slave_cmd_mutex); + return 0; + +err: + mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, 0); + mutex_unlock(&priv->cmd.slave_cmd_mutex); + return -EIO; +} + +static void mlx4_parav_master_pf_caps(struct mlx4_dev *dev) +{ + int i; + + for (i = 1; i <= dev->caps.num_ports; i++) { + if (dev->caps.port_type[i] == MLX4_PORT_TYPE_ETH) + dev->caps.gid_table_len[i] = + mlx4_get_slave_num_gids(dev, 0); + else + dev->caps.gid_table_len[i] = 1; + dev->caps.pkey_table_len[i] = + dev->phys_caps.pkey_phys_table_len[i] - 1; + } +} + +static int choose_log_fs_mgm_entry_size(int qp_per_entry) +{ + int i = MLX4_MIN_MGM_LOG_ENTRY_SIZE; + + for (i = MLX4_MIN_MGM_LOG_ENTRY_SIZE; i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE; + i++) { + if (qp_per_entry <= 4 * ((1 << i) / 16 - 2)) + break; + } + + return (i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE) ? i : -1; +} + +static void choose_steering_mode(struct mlx4_dev *dev, + struct mlx4_dev_cap *dev_cap) +{ + int nvfs; + + mlx4_get_val(num_vfs.dbdf2val.tbl, pci_physfn(dev->pdev), 0, &nvfs); + if (high_rate_steer && !mlx4_is_mfunc(dev)) { + dev->caps.flags &= ~(MLX4_DEV_CAP_FLAG_VEP_MC_STEER | + MLX4_DEV_CAP_FLAG_VEP_UC_STEER); + dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_FS_EN; + } + + if (mlx4_log_num_mgm_entry_size == -1 && + dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_FS_EN && + (!mlx4_is_mfunc(dev) || + (dev_cap->fs_max_num_qp_per_entry >= (nvfs + 1))) && + choose_log_fs_mgm_entry_size(dev_cap->fs_max_num_qp_per_entry) >= + MLX4_MIN_MGM_LOG_ENTRY_SIZE) { + dev->oper_log_mgm_entry_size = + choose_log_fs_mgm_entry_size(dev_cap->fs_max_num_qp_per_entry); + dev->caps.steering_mode = MLX4_STEERING_MODE_DEVICE_MANAGED; + dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry; + } else { + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER && + dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) + dev->caps.steering_mode = MLX4_STEERING_MODE_B0; + else { + dev->caps.steering_mode = MLX4_STEERING_MODE_A0; + + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER || + dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) + mlx4_warn(dev, "Must have both UC_STEER and MC_STEER flags " + "set to use B0 steering. Falling back to A0 steering mode.\n"); + } + dev->oper_log_mgm_entry_size = + mlx4_log_num_mgm_entry_size > 0 ? + mlx4_log_num_mgm_entry_size : + MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE; + dev->caps.num_qp_per_mgm = mlx4_get_qp_per_mgm(dev); + } + mlx4_dbg(dev, "Steering mode is: %s, oper_log_mgm_entry_size = %d, " + "log_num_mgm_entry_size = %d\n", + mlx4_steering_mode_str(dev->caps.steering_mode), + dev->oper_log_mgm_entry_size, mlx4_log_num_mgm_entry_size); +} + static int mlx4_init_hca(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_dev_cap *dev_cap = NULL; struct mlx4_adapter adapter; - struct mlx4_dev_cap dev_cap; struct mlx4_mod_stat_cfg mlx4_cfg; struct mlx4_profile profile; struct mlx4_init_hca_param init_hca; - struct mlx4_port_config *config; u64 icm_size; int err; - int i; - err = mlx4_QUERY_FW(dev); - if (err) { - if (err == -EACCES) - mlx4_info(dev, "non-primary physical function, skipping.\n"); - else - mlx4_err(dev, "QUERY_FW command failed, aborting.\n"); - return err; - } + if (!mlx4_is_slave(dev)) { + err = mlx4_QUERY_FW(dev); + if (err) { + if (err == -EACCES) + mlx4_info(dev, "non-primary physical function, skipping.\n"); + else + mlx4_err(dev, "QUERY_FW command failed, aborting.\n"); + return err; + } - err = mlx4_load_fw(dev); - if (err) { - mlx4_err(dev, "Failed to start FW, aborting.\n"); - return err; - } + err = mlx4_load_fw(dev); + if (err) { + mlx4_err(dev, "Failed to start FW, aborting.\n"); + return err; + } - mlx4_cfg.log_pg_sz_m = 1; - mlx4_cfg.log_pg_sz = 0; - err = mlx4_MOD_STAT_CFG(dev, &mlx4_cfg); - if (err) - mlx4_warn(dev, "Failed to override log_pg_sz parameter\n"); + mlx4_cfg.log_pg_sz_m = 1; + mlx4_cfg.log_pg_sz = 0; + err = mlx4_MOD_STAT_CFG(dev, &mlx4_cfg); + if (err) + mlx4_warn(dev, "Failed to override log_pg_sz parameter\n"); - err = mlx4_dev_cap(dev, &dev_cap); - if (err) { - mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); - goto err_stop_fw; - } + dev_cap = kzalloc(sizeof *dev_cap, GFP_KERNEL); + if (!dev_cap) { + mlx4_err(dev, "Failed to allocate memory for dev_cap\n"); + err = -ENOMEM; + goto err_stop_fw; + } - process_mod_param_profile(); - profile = default_profile; + err = mlx4_dev_cap(dev, dev_cap); + if (err) { + mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n"); + goto err_stop_fw; + } - list_for_each_entry(config, &config_list, list) { - if (config->pdev == dev->pdev) { - for (i = 1; i <= dev->caps.num_ports; i++) { - dev->caps.possible_type[i] = config->port_type[i]; - if (config->port_type[i] != MLX4_PORT_TYPE_AUTO) - dev->caps.port_type[i] = config->port_type[i]; - } + choose_steering_mode(dev, dev_cap); + + if (mlx4_is_master(dev)) + mlx4_parav_master_pf_caps(dev); + + process_mod_param_profile(&profile); + if (dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) + profile.num_mcg = MLX4_FS_NUM_MCG; + + icm_size = mlx4_make_profile(dev, &profile, dev_cap, + &init_hca); + if ((long long) icm_size < 0) { + err = icm_size; + goto err_stop_fw; } - } - mlx4_set_port_mask(dev); - icm_size = mlx4_make_profile(dev, &profile, &dev_cap, &init_hca); - if ((long long) icm_size < 0) { - err = icm_size; - goto err_stop_fw; - } + dev->caps.max_fmr_maps = (1 << (32 - ilog2(dev->caps.num_mpts))) - 1; - if (map_bf_area(dev)) - mlx4_dbg(dev, "Kernel support for blue flame is not available for kernels < 2.6.28\n"); + init_hca.log_uar_sz = ilog2(dev->caps.num_uars); + init_hca.uar_page_sz = PAGE_SHIFT - 12; - init_hca.log_uar_sz = ilog2(dev->caps.num_uars); + err = mlx4_init_icm(dev, dev_cap, &init_hca, icm_size); + if (err) + goto err_stop_fw; - err = mlx4_init_icm(dev, &dev_cap, &init_hca, icm_size); - if (err) - goto err_stop_fw; + init_hca.mw_enable = 1; - err = mlx4_INIT_HCA(dev, &init_hca); - if (err) { - mlx4_err(dev, "INIT_HCA command failed, aborting.\n"); - goto err_free_icm; + err = mlx4_INIT_HCA(dev, &init_hca); + if (err) { + mlx4_err(dev, "INIT_HCA command failed, aborting.\n"); + goto err_free_icm; + } + + /* + * Read HCA frequency by QUERY_HCA command + */ + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) { + memset(&init_hca, 0, sizeof(init_hca)); + err = mlx4_QUERY_HCA(dev, &init_hca); + if (err) { + mlx4_err(dev, "QUERY_HCA command failed, disable timestamp.\n"); + dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; + } else { + dev->caps.hca_core_clock = + init_hca.hca_core_clock; + } + + /* In case we got HCA frequency 0 - disable timestamping + * to avoid dividing by zero + */ + if (!dev->caps.hca_core_clock) { + dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; + mlx4_err(dev, "HCA frequency is 0. Timestamping is not supported."); + } else if (map_internal_clock(dev)) { + /* Map internal clock, + * in case of failure disable timestamping + */ + dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS; + mlx4_err(dev, "Failed to map internal clock. Timestamping is not supported.\n"); + } + } + } else { + err = mlx4_init_slave(dev); + if (err) { + mlx4_err(dev, "Failed to initialize slave\n"); + return err; + } + + err = mlx4_slave_cap(dev); + if (err) { + mlx4_err(dev, "Failed to obtain slave caps\n"); + goto err_close; + } } + if (map_bf_area(dev)) + mlx4_dbg(dev, "Failed to map blue flame area\n"); + + /* Only the master set the ports, all the rest got it from it.*/ + if (!mlx4_is_slave(dev)) + mlx4_set_port_mask(dev); + err = mlx4_QUERY_ADAPTER(dev, &adapter); if (err) { mlx4_err(dev, "QUERY_ADAPTER command failed, aborting.\n"); - goto err_close; + goto unmap_bf; } priv->eq_table.inta_pin = adapter.inta_pin; memcpy(dev->board_id, adapter.board_id, sizeof dev->board_id); + memcpy(dev->vsd, adapter.vsd, sizeof(dev->vsd)); + dev->vsd_vendor_id = adapter.vsd_vendor_id; + if (!mlx4_is_slave(dev)) + kfree(dev_cap); + return 0; +unmap_bf: + if (!mlx4_is_slave(dev)) + unmap_internal_clock(dev); + unmap_bf_area(dev); + + if (mlx4_is_slave(dev)) { + kfree(dev->caps.qp0_tunnel); + kfree(dev->caps.qp0_proxy); + kfree(dev->caps.qp1_tunnel); + kfree(dev->caps.qp1_proxy); + } + err_close: - mlx4_CLOSE_HCA(dev, 0); + if (mlx4_is_slave(dev)) + mlx4_slave_exit(dev); + else + mlx4_CLOSE_HCA(dev, 0); err_free_icm: - mlx4_free_icms(dev); + if (!mlx4_is_slave(dev)) + mlx4_free_icms(dev); err_stop_fw: - unmap_bf_area(dev); - mlx4_UNMAP_FA(dev); - mlx4_free_icm(dev, priv->fw.fw_icm, 0); - + if (!mlx4_is_slave(dev)) { + if (!mlx4_UNMAP_FA(dev)) + mlx4_free_icm(dev, priv->fw.fw_icm, 0); + else + pr_warn("mlx4_core: mlx4_UNMAP_FA failed.\n"); + kfree(dev_cap); + } return err; } static int mlx4_init_counters_table(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); - int err; - int nent; + int nent_pow2, port_indx, vf_index, num_counters; + int res, index = 0; + struct counter_index *new_counter_index; - switch (dev->caps.counters_mode) { - case MLX4_CUNTERS_BASIC: - nent = dev->caps.max_basic_counters; - break; - case MLX4_CUNTERS_EXT: - nent = dev->caps.max_ext_counters; - break; - default: + + if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS)) return -ENOENT; + + if (!mlx4_is_slave(dev) && + dev->caps.max_counters == dev->caps.max_extended_counters) { + res = mlx4_cmd(dev, MLX4_IF_STATE_EXTENDED, 0, 0, + MLX4_CMD_SET_IF_STAT, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + if (res) { + mlx4_err(dev, "Failed to set extended counters (err=%d)\n", res); + return res; + } } - err = mlx4_bitmap_init(&priv->counters_bitmap, nent, nent - 1, 0, 0); - if (err) - return err; + mutex_init(&priv->counters_table.mutex); + + if (mlx4_is_slave(dev)) { + for (port_indx = 0; port_indx < dev->caps.num_ports; port_indx++) { + INIT_LIST_HEAD(&priv->counters_table.global_port_list[port_indx]); + if (dev->caps.def_counter_index[port_indx] != 0xFF) { + new_counter_index = kmalloc(sizeof(struct counter_index), GFP_KERNEL); + if (!new_counter_index) + return -ENOMEM; + new_counter_index->index = dev->caps.def_counter_index[port_indx]; + list_add_tail(&new_counter_index->list, &priv->counters_table.global_port_list[port_indx]); + } + } + mlx4_dbg(dev, "%s: slave allocated %d counters for %d ports\n", + __func__, dev->caps.num_ports, dev->caps.num_ports); + return 0; + } + + nent_pow2 = roundup_pow_of_two(dev->caps.max_counters); + + for (port_indx = 0; port_indx < dev->caps.num_ports; port_indx++) { + INIT_LIST_HEAD(&priv->counters_table.global_port_list[port_indx]); + /* allocating 2 counters per port for PFs */ + /* For the PF, the ETH default counters are 0,2; */ + /* and the RoCE default counters are 1,3 */ + for (num_counters = 0; num_counters < 2; num_counters++, index++) { + new_counter_index = kmalloc(sizeof(struct counter_index), GFP_KERNEL); + if (!new_counter_index) + return -ENOMEM; + new_counter_index->index = index; + list_add_tail(&new_counter_index->list, + &priv->counters_table.global_port_list[port_indx]); + } + } + + if (mlx4_is_master(dev)) { + for (vf_index = 0; vf_index < dev->num_vfs; vf_index++) { + for (port_indx = 0; port_indx < dev->caps.num_ports; port_indx++) { + INIT_LIST_HEAD(&priv->counters_table.vf_list[vf_index][port_indx]); + new_counter_index = kmalloc(sizeof(struct counter_index), GFP_KERNEL); + if (!new_counter_index) + return -ENOMEM; + if (index < nent_pow2 - 2) { + new_counter_index->index = index; + index++; + } else { + new_counter_index->index = MLX4_SINK_COUNTER_INDEX; + } + + list_add_tail(&new_counter_index->list, + &priv->counters_table.vf_list[vf_index][port_indx]); + } + } + + res = mlx4_bitmap_init(&priv->counters_table.bitmap, + nent_pow2, nent_pow2 - 1, + index, 1); + mlx4_dbg(dev, "%s: master allocated %d counters for %d VFs\n", + __func__, index, dev->num_vfs); + } else { + res = mlx4_bitmap_init(&priv->counters_table.bitmap, + nent_pow2, nent_pow2 - 1, + index, 1); + mlx4_dbg(dev, "%s: native allocated %d counters for %d ports\n", + __func__, index, dev->caps.num_ports); + } + return 0; + } static void mlx4_cleanup_counters_table(struct mlx4_dev *dev) { - switch (dev->caps.counters_mode) { - case MLX4_CUNTERS_BASIC: - case MLX4_CUNTERS_EXT: - mlx4_bitmap_cleanup(&mlx4_priv(dev)->counters_bitmap); - break; - default: - break; + struct mlx4_priv *priv = mlx4_priv(dev); + int i, j; + struct counter_index *port, *tmp_port; + struct counter_index *vf, *tmp_vf; + + mutex_lock(&priv->counters_table.mutex); + + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS) { + for (i = 0; i < dev->caps.num_ports; i++) { + list_for_each_entry_safe(port, tmp_port, + &priv->counters_table.global_port_list[i], + list) { + list_del(&port->list); + kfree(port); + } + } + if (!mlx4_is_slave(dev)) { + for (i = 0; i < dev->num_vfs; i++) { + for (j = 0; j < dev->caps.num_ports; j++) { + list_for_each_entry_safe(vf, tmp_vf, + &priv->counters_table.vf_list[i][j], + list) { + /* clear the counter statistic */ + if (__mlx4_clear_if_stat(dev, vf->index)) + mlx4_dbg(dev, "%s: reset counter %d failed\n", + __func__, vf->index); + list_del(&vf->list); + kfree(vf); + } + } + } + mlx4_bitmap_cleanup(&priv->counters_table.bitmap); + } } + mutex_unlock(&priv->counters_table.mutex); } -int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx) +int __mlx4_slave_counters_free(struct mlx4_dev *dev, int slave) { struct mlx4_priv *priv = mlx4_priv(dev); + int i, first; + struct counter_index *vf, *tmp_vf; - switch (dev->caps.counters_mode) { - case MLX4_CUNTERS_BASIC: - case MLX4_CUNTERS_EXT: - *idx = mlx4_bitmap_alloc(&priv->counters_bitmap); - if (*idx == -1) - return -ENOMEM; + /* clean VF's counters for the next useg */ + if (slave > 0 && slave <= dev->num_vfs) { + mlx4_dbg(dev, "%s: free counters of slave(%d)\n" + , __func__, slave); + + mutex_lock(&priv->counters_table.mutex); + for (i = 0; i < dev->caps.num_ports; i++) { + first = 0; + list_for_each_entry_safe(vf, tmp_vf, + &priv->counters_table.vf_list[slave - 1][i], + list) { + /* clear the counter statistic */ + if (__mlx4_clear_if_stat(dev, vf->index)) + mlx4_dbg(dev, "%s: reset counter %d failed\n", + __func__, vf->index); + if (first++ && vf->index != MLX4_SINK_COUNTER_INDEX) { + mlx4_dbg(dev, "%s: delete counter index %d for slave %d and port %d\n" + , __func__, vf->index, slave, i + 1); + mlx4_bitmap_free(&priv->counters_table.bitmap, vf->index, MLX4_USE_RR); + list_del(&vf->list); + kfree(vf); + } else { + mlx4_dbg(dev, "%s: can't delete default counter index %d for slave %d and port %d\n" + , __func__, vf->index, slave, i + 1); + } + } + } + mutex_unlock(&priv->counters_table.mutex); + } + + return 0; +} + +int __mlx4_counter_alloc(struct mlx4_dev *dev, int slave, int port, u32 *idx) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct counter_index *new_counter_index; + + if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS)) + return -ENOENT; + + if ((slave > MLX4_MAX_NUM_VF) || (slave < 0) || + (port < 0) || (port > MLX4_MAX_PORTS)) { + mlx4_dbg(dev, "%s: invalid slave(%d) or port(%d) index\n", + __func__, slave, port); + return -EINVAL; + } + + /* handle old guest request does not support request by port index */ + if (port == 0) { + *idx = MLX4_SINK_COUNTER_INDEX; + mlx4_dbg(dev, "%s: allocated default counter index %d for slave %d port %d\n" + , __func__, *idx, slave, port); return 0; - default: - return -ENOMEM; } + + mutex_lock(&priv->counters_table.mutex); + + *idx = mlx4_bitmap_alloc(&priv->counters_table.bitmap); + /* if no resources return the default counter of the slave and port */ + if (*idx == -1) { + if (slave == 0) { /* its the ethernet counter ?????? */ + new_counter_index = list_entry(priv->counters_table.global_port_list[port - 1].next, + struct counter_index, + list); + } else { + new_counter_index = list_entry(priv->counters_table.vf_list[slave - 1][port - 1].next, + struct counter_index, + list); + } + + *idx = new_counter_index->index; + mlx4_dbg(dev, "%s: allocated defualt counter index %d for slave %d port %d\n" + , __func__, *idx, slave, port); + goto out; + } + + if (slave == 0) { /* native or master */ + new_counter_index = kmalloc(sizeof(struct counter_index), GFP_KERNEL); + if (!new_counter_index) + goto no_mem; + new_counter_index->index = *idx; + list_add_tail(&new_counter_index->list, &priv->counters_table.global_port_list[port - 1]); + } else { + new_counter_index = kmalloc(sizeof(struct counter_index), GFP_KERNEL); + if (!new_counter_index) + goto no_mem; + new_counter_index->index = *idx; + list_add_tail(&new_counter_index->list, &priv->counters_table.vf_list[slave - 1][port - 1]); + } + + mlx4_dbg(dev, "%s: allocated counter index %d for slave %d port %d\n" + , __func__, *idx, slave, port); +out: + mutex_unlock(&priv->counters_table.mutex); + return 0; + +no_mem: + mlx4_bitmap_free(&priv->counters_table.bitmap, *idx, MLX4_USE_RR); + mutex_unlock(&priv->counters_table.mutex); + *idx = MLX4_SINK_COUNTER_INDEX; + mlx4_dbg(dev, "%s: failed err (%d)\n" + , __func__, -ENOMEM); + return -ENOMEM; } + +int mlx4_counter_alloc(struct mlx4_dev *dev, u8 port, u32 *idx) +{ + u64 out_param; + int err; + struct mlx4_priv *priv = mlx4_priv(dev); + struct counter_index *new_counter_index, *c_index; + + if (mlx4_is_mfunc(dev)) { + err = mlx4_cmd_imm(dev, 0, &out_param, + ((u32) port) << 8 | (u32) RES_COUNTER, + RES_OP_RESERVE, MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (!err) { + *idx = get_param_l(&out_param); + if (*idx == MLX4_SINK_COUNTER_INDEX) + return -ENOSPC; + + mutex_lock(&priv->counters_table.mutex); + c_index = list_entry(priv->counters_table.global_port_list[port - 1].next, + struct counter_index, + list); + mutex_unlock(&priv->counters_table.mutex); + if (c_index->index == *idx) + return -EEXIST; + + if (mlx4_is_slave(dev)) { + new_counter_index = kmalloc(sizeof(struct counter_index), GFP_KERNEL); + if (!new_counter_index) { + mlx4_counter_free(dev, port, *idx); + return -ENOMEM; + } + new_counter_index->index = *idx; + mutex_lock(&priv->counters_table.mutex); + list_add_tail(&new_counter_index->list, &priv->counters_table.global_port_list[port - 1]); + mutex_unlock(&priv->counters_table.mutex); + mlx4_dbg(dev, "%s: allocated counter index %d for port %d\n" + , __func__, *idx, port); + } + } + return err; + } + return __mlx4_counter_alloc(dev, 0, port, idx); +} EXPORT_SYMBOL_GPL(mlx4_counter_alloc); -void mlx4_counter_free(struct mlx4_dev *dev, u32 idx) +void __mlx4_counter_free(struct mlx4_dev *dev, int slave, int port, u32 idx) { - switch (dev->caps.counters_mode) { - case MLX4_CUNTERS_BASIC: - case MLX4_CUNTERS_EXT: - mlx4_bitmap_free(&mlx4_priv(dev)->counters_bitmap, idx); + /* check if native or slave and deletes acordingly */ + struct mlx4_priv *priv = mlx4_priv(dev); + struct counter_index *pf, *tmp_pf; + struct counter_index *vf, *tmp_vf; + int first; + + + if (idx == MLX4_SINK_COUNTER_INDEX) { + mlx4_dbg(dev, "%s: try to delete default counter index %d for port %d\n" + , __func__, idx, port); + return; + } + + if ((slave > MLX4_MAX_NUM_VF) || (slave < 0) || + (port < 0) || (port > MLX4_MAX_PORTS)) { + mlx4_warn(dev, "%s: deletion failed due to invalid slave(%d) or port(%d) index\n" + , __func__, slave, idx); + return; + } + + mutex_lock(&priv->counters_table.mutex); + if (slave == 0) { + first = 0; + list_for_each_entry_safe(pf, tmp_pf, + &priv->counters_table.global_port_list[port - 1], + list) { + /* the first 2 counters are reserved */ + if (pf->index == idx) { + /* clear the counter statistic */ + if (__mlx4_clear_if_stat(dev, pf->index)) + mlx4_dbg(dev, "%s: reset counter %d failed\n", + __func__, pf->index); + if (1 < first && idx != MLX4_SINK_COUNTER_INDEX) { + list_del(&pf->list); + kfree(pf); + mlx4_dbg(dev, "%s: delete counter index %d for native device (%d) port %d\n" + , __func__, idx, slave, port); + mlx4_bitmap_free(&priv->counters_table.bitmap, idx, MLX4_USE_RR); + goto out; + } else { + mlx4_dbg(dev, "%s: can't delete default counter index %d for native device (%d) port %d\n" + , __func__, idx, slave, port); + goto out; + } + } + first++; + } + mlx4_dbg(dev, "%s: can't delete counter index %d for native device (%d) port %d\n" + , __func__, idx, slave, port); + } else { + first = 0; + list_for_each_entry_safe(vf, tmp_vf, + &priv->counters_table.vf_list[slave - 1][port - 1], + list) { + /* the first element is reserved */ + if (vf->index == idx) { + /* clear the counter statistic */ + if (__mlx4_clear_if_stat(dev, vf->index)) + mlx4_dbg(dev, "%s: reset counter %d failed\n", + __func__, vf->index); + if (first) { + list_del(&vf->list); + kfree(vf); + mlx4_dbg(dev, "%s: delete counter index %d for slave %d port %d\n", + __func__, idx, slave, port); + mlx4_bitmap_free(&priv->counters_table.bitmap, idx, MLX4_USE_RR); + goto out; + } else { + mlx4_dbg(dev, "%s: can't delete default slave (%d) counter index %d for port %d\n" + , __func__, slave, idx, port); + goto out; + } + } + first++; + } + mlx4_dbg(dev, "%s: can't delete slave (%d) counter index %d for port %d\n" + , __func__, slave, idx, port); + } + +out: + mutex_unlock(&priv->counters_table.mutex); +} + +void mlx4_counter_free(struct mlx4_dev *dev, u8 port, u32 idx) +{ + u64 in_param = 0; + struct mlx4_priv *priv = mlx4_priv(dev); + struct counter_index *counter, *tmp_counter; + int first = 0; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, idx); + mlx4_cmd(dev, in_param, + ((u32) port) << 8 | (u32) RES_COUNTER, + RES_OP_RESERVE, + MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + + if (mlx4_is_slave(dev) && idx != MLX4_SINK_COUNTER_INDEX) { + mutex_lock(&priv->counters_table.mutex); + list_for_each_entry_safe(counter, tmp_counter, + &priv->counters_table.global_port_list[port - 1], + list) { + if (counter->index == idx && first++) { + list_del(&counter->list); + kfree(counter); + mlx4_dbg(dev, "%s: delete counter index %d for port %d\n" + , __func__, idx, port); + mutex_unlock(&priv->counters_table.mutex); + return; + } + } + mutex_unlock(&priv->counters_table.mutex); + } + return; - default: - return; } + __mlx4_counter_free(dev, 0, port, idx); } EXPORT_SYMBOL_GPL(mlx4_counter_free); +int __mlx4_clear_if_stat(struct mlx4_dev *dev, + u8 counter_index) +{ + struct mlx4_cmd_mailbox *if_stat_mailbox = NULL; + int err = 0; + u32 if_stat_in_mod = (counter_index & 0xff) | (1 << 31); + + if (counter_index == MLX4_SINK_COUNTER_INDEX) + return -EINVAL; + + if (mlx4_is_slave(dev)) + return 0; + + if_stat_mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(if_stat_mailbox)) { + err = PTR_ERR(if_stat_mailbox); + return err; + } + + err = mlx4_cmd_box(dev, 0, if_stat_mailbox->dma, if_stat_in_mod, 0, + MLX4_CMD_QUERY_IF_STAT, MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, if_stat_mailbox); + return err; +} + +u8 mlx4_get_default_counter_index(struct mlx4_dev *dev, int slave, int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct counter_index *new_counter_index; + + if (dev->caps.port_type[port] == MLX4_PORT_TYPE_IB) { + mlx4_dbg(dev, "%s: return counter index %d for slave %d port (MLX4_PORT_TYPE_IB) %d\n", + __func__, MLX4_SINK_COUNTER_INDEX, slave, port); + return (u8)MLX4_SINK_COUNTER_INDEX; + } + + mutex_lock(&priv->counters_table.mutex); + if (slave == 0) { + new_counter_index = list_entry(priv->counters_table.global_port_list[port - 1].next, + struct counter_index, + list); + } else { + new_counter_index = list_entry(priv->counters_table.vf_list[slave - 1][port - 1].next, + struct counter_index, + list); + } + mutex_unlock(&priv->counters_table.mutex); + + mlx4_dbg(dev, "%s: return counter index %d for slave %d port %d\n", + __func__, new_counter_index->index, slave, port); + + + return (u8)new_counter_index->index; +} + +int mlx4_get_vport_ethtool_stats(struct mlx4_dev *dev, int port, + struct mlx4_en_vport_stats *vport_stats, + int reset) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_cmd_mailbox *if_stat_mailbox = NULL; + union mlx4_counter *counter; + int err = 0; + u32 if_stat_in_mod; + struct counter_index *vport, *tmp_vport; + + if (!vport_stats) + return -EINVAL; + + if_stat_mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(if_stat_mailbox)) { + err = PTR_ERR(if_stat_mailbox); + return err; + } + + mutex_lock(&priv->counters_table.mutex); + list_for_each_entry_safe(vport, tmp_vport, + &priv->counters_table.global_port_list[port - 1], + list) { + if (vport->index == MLX4_SINK_COUNTER_INDEX) + continue; + + memset(if_stat_mailbox->buf, 0, sizeof(union mlx4_counter)); + if_stat_in_mod = (vport->index & 0xff) | ((reset & 1) << 31); + err = mlx4_cmd_box(dev, 0, if_stat_mailbox->dma, + if_stat_in_mod, 0, + MLX4_CMD_QUERY_IF_STAT, + MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_NATIVE); + if (err) { + mlx4_dbg(dev, "%s: failed to read statistics for counter index %d\n", + __func__, vport->index); + goto if_stat_out; + } + counter = (union mlx4_counter *)if_stat_mailbox->buf; + if ((counter->control.cnt_mode & 0xf) == 1) { + vport_stats->rx_broadcast_packets += be64_to_cpu(counter->ext.counters[0].IfRxBroadcastFrames); + vport_stats->rx_unicast_packets += be64_to_cpu(counter->ext.counters[0].IfRxUnicastFrames); + vport_stats->rx_multicast_packets += be64_to_cpu(counter->ext.counters[0].IfRxMulticastFrames); + vport_stats->tx_broadcast_packets += be64_to_cpu(counter->ext.counters[0].IfTxBroadcastFrames); + vport_stats->tx_unicast_packets += be64_to_cpu(counter->ext.counters[0].IfTxUnicastFrames); + vport_stats->tx_multicast_packets += be64_to_cpu(counter->ext.counters[0].IfTxMulticastFrames); + vport_stats->rx_broadcast_bytes += be64_to_cpu(counter->ext.counters[0].IfRxBroadcastOctets); + vport_stats->rx_unicast_bytes += be64_to_cpu(counter->ext.counters[0].IfRxUnicastOctets); + vport_stats->rx_multicast_bytes += be64_to_cpu(counter->ext.counters[0].IfRxMulticastOctets); + vport_stats->tx_broadcast_bytes += be64_to_cpu(counter->ext.counters[0].IfTxBroadcastOctets); + vport_stats->tx_unicast_bytes += be64_to_cpu(counter->ext.counters[0].IfTxUnicastOctets); + vport_stats->tx_multicast_bytes += be64_to_cpu(counter->ext.counters[0].IfTxMulticastOctets); + vport_stats->rx_errors += be64_to_cpu(counter->ext.counters[0].IfRxErrorFrames); + vport_stats->rx_dropped += be64_to_cpu(counter->ext.counters[0].IfRxNoBufferFrames); + vport_stats->tx_errors += be64_to_cpu(counter->ext.counters[0].IfTxDroppedFrames); + } + } + +if_stat_out: + mutex_unlock(&priv->counters_table.mutex); + mlx4_free_cmd_mailbox(dev, if_stat_mailbox); + + return err; +} +EXPORT_SYMBOL_GPL(mlx4_get_vport_ethtool_stats); + static int mlx4_setup_hca(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int err; int port; __be32 ib_port_default_caps; err = mlx4_init_uar_table(dev); if (err) { mlx4_err(dev, "Failed to initialize " - "user access region table, aborting.\n"); + "user access region table (err=%d), aborting.\n", + err); return err; } err = mlx4_uar_alloc(dev, &priv->driver_uar); if (err) { - mlx4_err(dev, "Failed to allocate driver access region, " - "aborting.\n"); + mlx4_err(dev, "Failed to allocate driver access region " + "(err=%d), aborting.\n", err); goto err_uar_table_free; } - priv->kar = ioremap(priv->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE); + priv->kar = ioremap((phys_addr_t) priv->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE); if (!priv->kar) { mlx4_err(dev, "Couldn't map kernel access region, " "aborting.\n"); err = -ENOMEM; goto err_uar_free; } err = mlx4_init_pd_table(dev); if (err) { mlx4_err(dev, "Failed to initialize " - "protection domain table, aborting.\n"); + "protection domain table (err=%d), aborting.\n", err); goto err_kar_unmap; } err = mlx4_init_xrcd_table(dev); if (err) { - mlx4_err(dev, "Failed to initialize extended " - "reliably connected domain table, aborting.\n"); + mlx4_err(dev, "Failed to initialize " + "reliable connection domain table (err=%d), " + "aborting.\n", err); goto err_pd_table_free; } err = mlx4_init_mr_table(dev); if (err) { mlx4_err(dev, "Failed to initialize " - "memory region table, aborting.\n"); + "memory region table (err=%d), aborting.\n", err); goto err_xrcd_table_free; } + if (!mlx4_is_slave(dev)) { + err = mlx4_init_mcg_table(dev); + if (err) { + mlx4_err(dev, "Failed to initialize " + "multicast group table (err=%d), aborting.\n", + err); + goto err_mr_table_free; + } + } + err = mlx4_init_eq_table(dev); if (err) { mlx4_err(dev, "Failed to initialize " - "event queue table, aborting.\n"); - goto err_mr_table_free; + "event queue table (err=%d), aborting.\n", err); + goto err_mcg_table_free; } err = mlx4_cmd_use_events(dev); if (err) { mlx4_err(dev, "Failed to switch to event-driven " - "firmware commands, aborting.\n"); + "firmware commands (err=%d), aborting.\n", err); goto err_eq_table_free; } err = mlx4_NOP(dev); if (err) { if (dev->flags & MLX4_FLAG_MSI_X) { mlx4_warn(dev, "NOP command failed to generate MSI-X " "interrupt IRQ %d).\n", priv->eq_table.eq[dev->caps.num_comp_vectors].irq); mlx4_warn(dev, "Trying again without MSI-X.\n"); } else { mlx4_err(dev, "NOP command failed to generate interrupt " "(IRQ %d), aborting.\n", priv->eq_table.eq[dev->caps.num_comp_vectors].irq); mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n"); } goto err_cmd_poll; } mlx4_dbg(dev, "NOP command IRQ test passed\n"); err = mlx4_init_cq_table(dev); if (err) { mlx4_err(dev, "Failed to initialize " - "completion queue table, aborting.\n"); + "completion queue table (err=%d), aborting.\n", err); goto err_cmd_poll; } err = mlx4_init_srq_table(dev); if (err) { mlx4_err(dev, "Failed to initialize " - "shared receive queue table, aborting.\n"); + "shared receive queue table (err=%d), aborting.\n", + err); goto err_cq_table_free; } err = mlx4_init_qp_table(dev); if (err) { mlx4_err(dev, "Failed to initialize " - "queue pair table, aborting.\n"); + "queue pair table (err=%d), aborting.\n", err); goto err_srq_table_free; } - err = mlx4_init_mcg_table(dev); - if (err) { - mlx4_err(dev, "Failed to initialize " - "multicast group table, aborting.\n"); - goto err_qp_table_free; - } - err = mlx4_init_counters_table(dev); if (err && err != -ENOENT) { - mlx4_err(dev, "Failed to initialize counters table, aborting.\n"); - goto err_mcg_table_free; + mlx4_err(dev, "Failed to initialize counters table (err=%d), " + "aborting.\n", err); + goto err_qp_table_free; } - for (port = 1; port <= dev->caps.num_ports; port++) { - ib_port_default_caps = 0; - err = mlx4_get_port_ib_caps(dev, port, &ib_port_default_caps); - if (err) - mlx4_warn(dev, "failed to get port %d default " - "ib capabilities (%d). Continuing with " - "caps = 0\n", port, err); - dev->caps.ib_port_def_cap[port] = ib_port_default_caps; - err = mlx4_SET_PORT(dev, port); - if (err) { - mlx4_err(dev, "Failed to set port %d, aborting\n", - port); - goto err_counters_table_free; + if (!mlx4_is_slave(dev)) { + for (port = 1; port <= dev->caps.num_ports; port++) { + ib_port_default_caps = 0; + err = mlx4_get_port_ib_caps(dev, port, + &ib_port_default_caps); + if (err) + mlx4_warn(dev, "failed to get port %d default " + "ib capabilities (%d). Continuing " + "with caps = 0\n", port, err); + dev->caps.ib_port_def_cap[port] = ib_port_default_caps; + + /* initialize per-slave default ib port capabilities */ + if (mlx4_is_master(dev)) { + int i; + for (i = 0; i < dev->num_slaves; i++) { + if (i == mlx4_master_func_num(dev)) + continue; + priv->mfunc.master.slave_state[i].ib_cap_mask[port] = + ib_port_default_caps; + } + } + + dev->caps.port_ib_mtu[port] = IB_MTU_4096; + + err = mlx4_SET_PORT(dev, port, mlx4_is_master(dev) ? + dev->caps.pkey_table_len[port] : -1); + if (err) { + mlx4_err(dev, "Failed to set port %d (err=%d), " + "aborting\n", port, err); + goto err_counters_table_free; + } } } return 0; err_counters_table_free: mlx4_cleanup_counters_table(dev); -err_mcg_table_free: - mlx4_cleanup_mcg_table(dev); - err_qp_table_free: mlx4_cleanup_qp_table(dev); err_srq_table_free: mlx4_cleanup_srq_table(dev); err_cq_table_free: mlx4_cleanup_cq_table(dev); err_cmd_poll: mlx4_cmd_use_polling(dev); err_eq_table_free: mlx4_cleanup_eq_table(dev); +err_mcg_table_free: + if (!mlx4_is_slave(dev)) + mlx4_cleanup_mcg_table(dev); + err_mr_table_free: mlx4_cleanup_mr_table(dev); err_xrcd_table_free: mlx4_cleanup_xrcd_table(dev); err_pd_table_free: mlx4_cleanup_pd_table(dev); err_kar_unmap: iounmap(priv->kar); err_uar_free: mlx4_uar_free(dev, &priv->driver_uar); err_uar_table_free: mlx4_cleanup_uar_table(dev); return err; } static void mlx4_enable_msi_x(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); struct msix_entry *entries; - int nreq; + int nreq = min_t(int, dev->caps.num_ports * + min_t(int, num_possible_cpus() + 1, MAX_MSIX_P_PORT) + + MSIX_LEGACY_SZ, MAX_MSIX); int err; int i; if (msi_x) { nreq = min_t(int, dev->caps.num_eqs - dev->caps.reserved_eqs, - num_possible_cpus() + 1); + nreq); + + if (msi_x > 1 && !mlx4_is_mfunc(dev)) + nreq = min_t(int, nreq, msi_x); + entries = kcalloc(nreq, sizeof *entries, GFP_KERNEL); if (!entries) goto no_msi; for (i = 0; i < nreq; ++i) entries[i].entry = i; retry: err = pci_enable_msix(dev->pdev, entries, nreq); if (err) { /* Try again if at least 2 vectors are available */ if (err > 1) { mlx4_info(dev, "Requested %d vectors, " "but only %d MSI-X vectors available, " "trying again\n", nreq, err); nreq = err; goto retry; } kfree(entries); goto no_msi; } - dev->caps.num_comp_vectors = nreq - 1; + if (nreq < + MSIX_LEGACY_SZ + dev->caps.num_ports * MIN_MSIX_P_PORT) { + /*Working in legacy mode , all EQ's shared*/ + dev->caps.comp_pool = 0; + dev->caps.num_comp_vectors = nreq - 1; + } else { + dev->caps.comp_pool = nreq - MSIX_LEGACY_SZ; + dev->caps.num_comp_vectors = MSIX_LEGACY_SZ - 1; + } for (i = 0; i < nreq; ++i) priv->eq_table.eq[i].irq = entries[i].vector; dev->flags |= MLX4_FLAG_MSI_X; kfree(entries); return; } no_msi: dev->caps.num_comp_vectors = 1; + dev->caps.comp_pool = 0; for (i = 0; i < 2; ++i) priv->eq_table.eq[i].irq = dev->pdev->irq; } static int mlx4_init_port_info(struct mlx4_dev *dev, int port) { struct mlx4_port_info *info = &mlx4_priv(dev)->port[port]; int err = 0; info->dev = dev; info->port = port; - mlx4_init_mac_table(dev, &info->mac_table); - mlx4_init_vlan_table(dev, &info->vlan_table); + if (!mlx4_is_slave(dev)) { + mlx4_init_mac_table(dev, &info->mac_table); + mlx4_init_vlan_table(dev, &info->vlan_table); + info->base_qpn = mlx4_get_base_qpn(dev, port); + } sprintf(info->dev_name, "mlx4_port%d", port); info->port_attr.attr.name = info->dev_name; - info->port_attr.attr.mode = S_IRUGO | S_IWUSR; + if (mlx4_is_mfunc(dev)) + info->port_attr.attr.mode = S_IRUGO; + else { + info->port_attr.attr.mode = S_IRUGO | S_IWUSR; + info->port_attr.store = set_port_type; + } info->port_attr.show = show_port_type; - info->port_attr.store = set_port_type; + sysfs_attr_init(&info->port_attr.attr); err = device_create_file(&dev->pdev->dev, &info->port_attr); if (err) { mlx4_err(dev, "Failed to create file for port %d\n", port); info->port = -1; } + sprintf(info->dev_mtu_name, "mlx4_port%d_mtu", port); + info->port_mtu_attr.attr.name = info->dev_mtu_name; + if (mlx4_is_mfunc(dev)) + info->port_mtu_attr.attr.mode = S_IRUGO; + else { + info->port_mtu_attr.attr.mode = S_IRUGO | S_IWUSR; + info->port_mtu_attr.store = set_port_ib_mtu; + } + info->port_mtu_attr.show = show_port_ib_mtu; + sysfs_attr_init(&info->port_mtu_attr.attr); + + err = device_create_file(&dev->pdev->dev, &info->port_mtu_attr); + if (err) { + mlx4_err(dev, "Failed to create mtu file for port %d\n", port); + device_remove_file(&info->dev->pdev->dev, &info->port_attr); + info->port = -1; + } + return err; } static void mlx4_cleanup_port_info(struct mlx4_port_info *info) { if (info->port < 0) return; device_remove_file(&info->dev->pdev->dev, &info->port_attr); + device_remove_file(&info->dev->pdev->dev, &info->port_mtu_attr); } -static int mlx4_init_trigger(struct mlx4_priv *priv) +static int mlx4_init_steering(struct mlx4_dev *dev) { - memcpy(&priv->trigger_attr, &dev_attr_port_trigger, - sizeof(struct device_attribute)); - return device_create_file(&priv->dev.pdev->dev, &priv->trigger_attr); + struct mlx4_priv *priv = mlx4_priv(dev); + int num_entries = dev->caps.num_ports; + int i, j; + + priv->steer = kzalloc(sizeof(struct mlx4_steer) * num_entries, GFP_KERNEL); + if (!priv->steer) + return -ENOMEM; + + for (i = 0; i < num_entries; i++) + for (j = 0; j < MLX4_NUM_STEERS; j++) { + INIT_LIST_HEAD(&priv->steer[i].promisc_qps[j]); + INIT_LIST_HEAD(&priv->steer[i].steer_entries[j]); + } + return 0; } -static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) +static void mlx4_clear_steering(struct mlx4_dev *dev) { + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_steer_index *entry, *tmp_entry; + struct mlx4_promisc_qp *pqp, *tmp_pqp; + int num_entries = dev->caps.num_ports; + int i, j; + + for (i = 0; i < num_entries; i++) { + for (j = 0; j < MLX4_NUM_STEERS; j++) { + list_for_each_entry_safe(pqp, tmp_pqp, + &priv->steer[i].promisc_qps[j], + list) { + list_del(&pqp->list); + kfree(pqp); + } + list_for_each_entry_safe(entry, tmp_entry, + &priv->steer[i].steer_entries[j], + list) { + list_del(&entry->list); + list_for_each_entry_safe(pqp, tmp_pqp, + &entry->duplicates, + list) { + list_del(&pqp->list); + kfree(pqp); + } + kfree(entry); + } + } + } + kfree(priv->steer); +} + +static int extended_func_num(struct pci_dev *pdev) +{ + return PCI_SLOT(pdev->devfn) * 8 + PCI_FUNC(pdev->devfn); +} + +#define MLX4_OWNER_BASE 0x8069c +#define MLX4_OWNER_SIZE 4 + +static int mlx4_get_ownership(struct mlx4_dev *dev) +{ + void __iomem *owner; + u32 ret; + + if (pci_channel_offline(dev->pdev)) + return -EIO; + + owner = ioremap(pci_resource_start(dev->pdev, 0) + MLX4_OWNER_BASE, + MLX4_OWNER_SIZE); + if (!owner) { + mlx4_err(dev, "Failed to obtain ownership bit\n"); + return -ENOMEM; + } + + ret = readl(owner); + iounmap(owner); + return (int) !!ret; +} + +static void mlx4_free_ownership(struct mlx4_dev *dev) +{ + void __iomem *owner; + + if (pci_channel_offline(dev->pdev)) + return; + + owner = ioremap(pci_resource_start(dev->pdev, 0) + MLX4_OWNER_BASE, + MLX4_OWNER_SIZE); + if (!owner) { + mlx4_err(dev, "Failed to obtain ownership bit\n"); + return; + } + writel(0, owner); + msleep(1000); + iounmap(owner); +} + +static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data) +{ struct mlx4_priv *priv; struct mlx4_dev *dev; int err; int port; - int i; + int nvfs, prb_vf; - printk(KERN_INFO PFX "Initializing %s\n", - pci_name(pdev)); + pr_info(DRV_NAME ": Initializing %s\n", pci_name(pdev)); err = pci_enable_device(pdev); if (err) { dev_err(&pdev->dev, "Cannot enable PCI device, " "aborting.\n"); return err; } + mlx4_get_val(num_vfs.dbdf2val.tbl, pci_physfn(pdev), 0, &nvfs); + mlx4_get_val(probe_vf.dbdf2val.tbl, pci_physfn(pdev), 0, &prb_vf); + if (nvfs > MLX4_MAX_NUM_VF) { + dev_err(&pdev->dev, "There are more VF's (%d) than allowed(%d)\n", + nvfs, MLX4_MAX_NUM_VF); + return -EINVAL; + } + + if (nvfs < 0) { + dev_err(&pdev->dev, "num_vfs module parameter cannot be negative\n"); + return -EINVAL; + } /* - * Check for BARs. We expect 0: 1MB + * Check for BARs. */ - if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) || - pci_resource_len(pdev, 0) != 1 << 20) { - dev_err(&pdev->dev, "Missing DCS, aborting.\n"); + if (!(pci_dev_data & MLX4_PCI_DEV_IS_VF) && + !(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { + dev_err(&pdev->dev, "Missing DCS, aborting." + "(driver_data: 0x%x, pci_resource_flags(pdev, 0):0x%x)\n", + pci_dev_data, pci_resource_flags(pdev, 0)); err = -ENODEV; goto err_disable_pdev; } if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) { dev_err(&pdev->dev, "Missing UAR, aborting.\n"); err = -ENODEV; goto err_disable_pdev; } - err = pci_request_region(pdev, 0, DRV_NAME); + err = pci_request_regions(pdev, DRV_NAME); if (err) { - dev_err(&pdev->dev, "Cannot request control region, aborting.\n"); + dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); goto err_disable_pdev; } - err = pci_request_region(pdev, 2, DRV_NAME); - if (err) { - dev_err(&pdev->dev, "Cannot request UAR region, aborting.\n"); - goto err_release_bar0; - } - pci_set_master(pdev); err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); if (err) { dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n"); err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); if (err) { dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n"); - goto err_release_bar2; + goto err_release_regions; } } err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); if (err) { dev_warn(&pdev->dev, "Warning: couldn't set 64-bit " "consistent PCI DMA mask.\n"); err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); if (err) { dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, " "aborting.\n"); - goto err_release_bar2; + goto err_release_regions; } } + /* Allow large DMA segments, up to the firmware limit of 1 GB */ + dma_set_max_seg_size(&pdev->dev, 1024 * 1024 * 1024); + priv = kzalloc(sizeof *priv, GFP_KERNEL); if (!priv) { dev_err(&pdev->dev, "Device struct alloc failed, " "aborting.\n"); err = -ENOMEM; - goto err_release_bar2; + goto err_release_regions; } dev = &priv->dev; dev->pdev = pdev; + INIT_LIST_HEAD(&priv->dev_list); INIT_LIST_HEAD(&priv->ctx_list); spin_lock_init(&priv->ctx_lock); mutex_init(&priv->port_mutex); INIT_LIST_HEAD(&priv->pgdir_list); mutex_init(&priv->pgdir_mutex); - for (i = 0; i < MLX4_MAX_PORTS; ++i) - priv->iboe_counter_index[i] = -1; INIT_LIST_HEAD(&priv->bf_list); mutex_init(&priv->bf_mutex); - /* - * Now reset the HCA before we touch the PCI capabilities or - * attempt a firmware command, since a boot ROM may have left - * the HCA in an undefined state. - */ - err = mlx4_reset(dev); - if (err) { - mlx4_err(dev, "Failed to reset HCA, aborting.\n"); - goto err_free_dev; + dev->rev_id = pdev->revision; + dev->numa_node = dev_to_node(&pdev->dev); + /* Detect if this device is a virtual function */ + if (pci_dev_data & MLX4_PCI_DEV_IS_VF) { + /* When acting as pf, we normally skip vfs unless explicitly + * requested to probe them. */ + if (nvfs && extended_func_num(pdev) > prb_vf) { + mlx4_warn(dev, "Skipping virtual function:%d\n", + extended_func_num(pdev)); + err = -ENODEV; + goto err_free_dev; + } + mlx4_warn(dev, "Detected virtual function - running in slave mode\n"); + dev->flags |= MLX4_FLAG_SLAVE; + } else { + /* We reset the device and enable SRIOV only for physical + * devices. Try to claim ownership on the device; + * if already taken, skip -- do not allow multiple PFs */ + err = mlx4_get_ownership(dev); + if (err) { + if (err < 0) + goto err_free_dev; + else { + mlx4_warn(dev, "Multiple PFs not yet supported." + " Skipping PF.\n"); + err = -EINVAL; + goto err_free_dev; + } + } + + if (nvfs) { + mlx4_warn(dev, "Enabling SR-IOV with %d VFs\n", nvfs); + err = pci_enable_sriov(pdev, nvfs); + if (err) { + mlx4_err(dev, "Failed to enable SR-IOV, continuing without SR-IOV (err = %d).\n", + err); + err = 0; + } else { + mlx4_warn(dev, "Running in master mode\n"); + dev->flags |= MLX4_FLAG_SRIOV | + MLX4_FLAG_MASTER; + dev->num_vfs = nvfs; + } + } + + atomic_set(&priv->opreq_count, 0); + INIT_WORK(&priv->opreq_task, mlx4_opreq_action); + + /* + * Now reset the HCA before we touch the PCI capabilities or + * attempt a firmware command, since a boot ROM may have left + * the HCA in an undefined state. + */ + err = mlx4_reset(dev); + if (err) { + mlx4_err(dev, "Failed to reset HCA, aborting.\n"); + goto err_sriov; + } } - if (mlx4_cmd_init(dev)) { +slave_start: + err = mlx4_cmd_init(dev); + if (err) { mlx4_err(dev, "Failed to init command interface, aborting.\n"); - goto err_free_dev; + goto err_sriov; } + /* In slave functions, the communication channel must be initialized + * before posting commands. Also, init num_slaves before calling + * mlx4_init_hca */ + if (mlx4_is_mfunc(dev)) { + if (mlx4_is_master(dev)) + dev->num_slaves = MLX4_MAX_NUM_SLAVES; + else { + dev->num_slaves = 0; + err = mlx4_multi_func_init(dev); + if (err) { + mlx4_err(dev, "Failed to init slave mfunc" + " interface, aborting.\n"); + goto err_cmd; + } + } + } + err = mlx4_init_hca(dev); - if (err) - goto err_cmd; + if (err) { + if (err == -EACCES) { + /* Not primary Physical function + * Running in slave mode */ + mlx4_cmd_cleanup(dev); + dev->flags |= MLX4_FLAG_SLAVE; + dev->flags &= ~MLX4_FLAG_MASTER; + goto slave_start; + } else + goto err_mfunc; + } + /* In master functions, the communication channel must be initialized + * after obtaining its address from fw */ + if (mlx4_is_master(dev)) { + err = mlx4_multi_func_init(dev); + if (err) { + mlx4_err(dev, "Failed to init master mfunc" + "interface, aborting.\n"); + goto err_close; + } + } + err = mlx4_alloc_eq_table(dev); if (err) - goto err_close; + goto err_master_mfunc; + priv->msix_ctl.pool_bm = 0; + mutex_init(&priv->msix_ctl.pool_lock); + mlx4_enable_msi_x(dev); + if ((mlx4_is_mfunc(dev)) && + !(dev->flags & MLX4_FLAG_MSI_X)) { + err = -ENOSYS; + mlx4_err(dev, "INTx is not supported in multi-function mode." + " aborting.\n"); + goto err_free_eq; + } + if (!mlx4_is_slave(dev)) { + err = mlx4_init_steering(dev); + if (err) + goto err_free_eq; + } + err = mlx4_setup_hca(dev); - if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X)) { + if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X) && + !mlx4_is_mfunc(dev)) { dev->flags &= ~MLX4_FLAG_MSI_X; + dev->caps.num_comp_vectors = 1; + dev->caps.comp_pool = 0; pci_disable_msix(pdev); err = mlx4_setup_hca(dev); } if (err) - goto err_free_eq; + goto err_steer; + mlx4_init_quotas(dev); + for (port = 1; port <= dev->caps.num_ports; port++) { err = mlx4_init_port_info(dev, port); if (err) goto err_port; } err = mlx4_register_device(dev); if (err) goto err_port; - err = mlx4_init_trigger(priv); - if (err) - goto err_register; + mlx4_request_modules(dev); - err = mlx4_sense_init(dev); - if (err) - goto err_trigger; - + mlx4_sense_init(dev); mlx4_start_sense(dev); + priv->pci_dev_data = pci_dev_data; pci_set_drvdata(pdev, dev); return 0; -err_trigger: - device_remove_file(&dev->pdev->dev, &priv->trigger_attr); -err_register: - mlx4_unregister_device(dev); err_port: for (--port; port >= 1; --port) mlx4_cleanup_port_info(&priv->port[port]); mlx4_cleanup_counters_table(dev); - mlx4_cleanup_mcg_table(dev); mlx4_cleanup_qp_table(dev); mlx4_cleanup_srq_table(dev); mlx4_cleanup_cq_table(dev); mlx4_cmd_use_polling(dev); mlx4_cleanup_eq_table(dev); + mlx4_cleanup_mcg_table(dev); mlx4_cleanup_mr_table(dev); mlx4_cleanup_xrcd_table(dev); mlx4_cleanup_pd_table(dev); mlx4_cleanup_uar_table(dev); +err_steer: + if (!mlx4_is_slave(dev)) + mlx4_clear_steering(dev); + err_free_eq: mlx4_free_eq_table(dev); +err_master_mfunc: + if (mlx4_is_master(dev)) { + mlx4_free_resource_tracker(dev, RES_TR_FREE_STRUCTS_ONLY); + mlx4_multi_func_cleanup(dev); + } + + if (mlx4_is_slave(dev)) { + kfree(dev->caps.qp0_tunnel); + kfree(dev->caps.qp0_proxy); + kfree(dev->caps.qp1_tunnel); + kfree(dev->caps.qp1_proxy); + } + err_close: if (dev->flags & MLX4_FLAG_MSI_X) pci_disable_msix(pdev); mlx4_close_hca(dev); +err_mfunc: + if (mlx4_is_slave(dev)) + mlx4_multi_func_cleanup(dev); + err_cmd: mlx4_cmd_cleanup(dev); +err_sriov: + if (dev->flags & MLX4_FLAG_SRIOV) + pci_disable_sriov(pdev); + + if (!mlx4_is_slave(dev)) + mlx4_free_ownership(dev); + err_free_dev: kfree(priv); -err_release_bar2: - pci_release_region(pdev, 2); +err_release_regions: + pci_release_regions(pdev); -err_release_bar0: - pci_release_region(pdev, 0); - err_disable_pdev: pci_disable_device(pdev); pci_set_drvdata(pdev, NULL); return err; } static int __devinit mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) { - static int mlx4_version_printed; + printk_once(KERN_INFO "%s", mlx4_version); - if (!mlx4_version_printed) { - printk(KERN_INFO "%s", mlx4_version); - ++mlx4_version_printed; - } - - return __mlx4_init_one(pdev, id); + return __mlx4_init_one(pdev, id->driver_data); } static void mlx4_remove_one(struct pci_dev *pdev) { struct mlx4_dev *dev = pci_get_drvdata(pdev); struct mlx4_priv *priv = mlx4_priv(dev); int p; if (dev) { - mlx4_sense_cleanup(dev); + /* in SRIOV it is not allowed to unload the pf's + * driver while there are alive vf's */ + if (mlx4_is_master(dev)) { + if (mlx4_how_many_lives_vf(dev)) + mlx4_err(dev, "Removing PF when there are assigned VF's !!!\n"); + } + mlx4_stop_sense(dev); mlx4_unregister_device(dev); - device_remove_file(&dev->pdev->dev, &priv->trigger_attr); for (p = 1; p <= dev->caps.num_ports; p++) { mlx4_cleanup_port_info(&priv->port[p]); mlx4_CLOSE_PORT(dev, p); } - mlx4_cleanup_counters_table(dev); - mlx4_cleanup_mcg_table(dev); + if (mlx4_is_master(dev)) + mlx4_free_resource_tracker(dev, + RES_TR_FREE_SLAVES_ONLY); + + mlx4_cleanup_counters_table(dev); mlx4_cleanup_qp_table(dev); mlx4_cleanup_srq_table(dev); mlx4_cleanup_cq_table(dev); mlx4_cmd_use_polling(dev); mlx4_cleanup_eq_table(dev); + mlx4_cleanup_mcg_table(dev); mlx4_cleanup_mr_table(dev); mlx4_cleanup_xrcd_table(dev); mlx4_cleanup_pd_table(dev); + if (mlx4_is_master(dev)) + mlx4_free_resource_tracker(dev, + RES_TR_FREE_STRUCTS_ONLY); + iounmap(priv->kar); mlx4_uar_free(dev, &priv->driver_uar); mlx4_cleanup_uar_table(dev); + if (!mlx4_is_slave(dev)) + mlx4_clear_steering(dev); mlx4_free_eq_table(dev); + if (mlx4_is_master(dev)) + mlx4_multi_func_cleanup(dev); mlx4_close_hca(dev); + if (mlx4_is_slave(dev)) + mlx4_multi_func_cleanup(dev); mlx4_cmd_cleanup(dev); if (dev->flags & MLX4_FLAG_MSI_X) pci_disable_msix(pdev); + if (dev->flags & MLX4_FLAG_SRIOV) { + mlx4_warn(dev, "Disabling SR-IOV\n"); + pci_disable_sriov(pdev); + } + if (!mlx4_is_slave(dev)) + mlx4_free_ownership(dev); + + kfree(dev->caps.qp0_tunnel); + kfree(dev->caps.qp0_proxy); + kfree(dev->caps.qp1_tunnel); + kfree(dev->caps.qp1_proxy); + kfree(priv); - pci_release_region(pdev, 2); - pci_release_region(pdev, 0); + pci_release_regions(pdev); pci_disable_device(pdev); pci_set_drvdata(pdev, NULL); } } +static int restore_current_port_types(struct mlx4_dev *dev, + enum mlx4_port_type *types, + enum mlx4_port_type *poss_types) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int err, i; + + mlx4_stop_sense(dev); + mutex_lock(&priv->port_mutex); + for (i = 0; i < dev->caps.num_ports; i++) + dev->caps.possible_type[i + 1] = poss_types[i]; + err = mlx4_change_port_types(dev, types); + mlx4_start_sense(dev); + mutex_unlock(&priv->port_mutex); + return err; +} + int mlx4_restart_one(struct pci_dev *pdev) { + struct mlx4_dev *dev = pci_get_drvdata(pdev); + struct mlx4_priv *priv = mlx4_priv(dev); + enum mlx4_port_type curr_type[MLX4_MAX_PORTS]; + enum mlx4_port_type poss_type[MLX4_MAX_PORTS]; + int pci_dev_data, err, i; + + pci_dev_data = priv->pci_dev_data; + for (i = 0; i < dev->caps.num_ports; i++) { + curr_type[i] = dev->caps.port_type[i + 1]; + poss_type[i] = dev->caps.possible_type[i + 1]; + } + mlx4_remove_one(pdev); - return __mlx4_init_one(pdev, NULL); + err = __mlx4_init_one(pdev, pci_dev_data); + if (err) + return err; + + dev = pci_get_drvdata(pdev); + err = restore_current_port_types(dev, curr_type, poss_type); + if (err) + mlx4_err(dev, "mlx4_restart_one: could not restore original port types (%d)\n", + err); + return 0; } -static struct pci_device_id mlx4_pci_table[] = { - { PCI_VDEVICE(MELLANOX, 0x6340) }, /* MT25408 "Hermon" SDR */ - { PCI_VDEVICE(MELLANOX, 0x634a) }, /* MT25408 "Hermon" DDR */ - { PCI_VDEVICE(MELLANOX, 0x6354) }, /* MT25408 "Hermon" QDR */ - { PCI_VDEVICE(MELLANOX, 0x6732) }, /* MT25408 "Hermon" DDR PCIe gen2 */ - { PCI_VDEVICE(MELLANOX, 0x673c) }, /* MT25408 "Hermon" QDR PCIe gen2 */ - { PCI_VDEVICE(MELLANOX, 0x6368) }, /* MT25408 "Hermon" EN 10GigE */ - { PCI_VDEVICE(MELLANOX, 0x6750) }, /* MT25408 "Hermon" EN 10GigE PCIe gen2 */ - { PCI_VDEVICE(MELLANOX, 0x6372) }, /* MT25458 ConnectX EN 10GBASE-T 10GigE */ - { PCI_VDEVICE(MELLANOX, 0x675a) }, /* MT25458 ConnectX EN 10GBASE-T+Gen2 10GigE */ - { PCI_VDEVICE(MELLANOX, 0x6764) }, /* MT26468 ConnectX EN 10GigE PCIe gen2 */ - { PCI_VDEVICE(MELLANOX, 0x6746) }, /* MT26438 ConnectX VPI PCIe 2.0 5GT/s - IB QDR / 10GigE Virt+ */ - { PCI_VDEVICE(MELLANOX, 0x676e) }, /* MT26478 ConnectX EN 40GigE PCIe 2.0 5GT/s */ - { PCI_VDEVICE(MELLANOX, 0x6778) }, /* MT26488 ConnectX VPI PCIe 2.0 5GT/s - IB DDR / 10GigE Virt+ */ - { PCI_VDEVICE(MELLANOX, 0x1000) }, - { PCI_VDEVICE(MELLANOX, 0x1001) }, - { PCI_VDEVICE(MELLANOX, 0x1002) }, - { PCI_VDEVICE(MELLANOX, 0x1003) }, - { PCI_VDEVICE(MELLANOX, 0x1004) }, - { PCI_VDEVICE(MELLANOX, 0x1005) }, - { PCI_VDEVICE(MELLANOX, 0x1006) }, - { PCI_VDEVICE(MELLANOX, 0x1007) }, - { PCI_VDEVICE(MELLANOX, 0x1008) }, - { PCI_VDEVICE(MELLANOX, 0x1009) }, - { PCI_VDEVICE(MELLANOX, 0x100a) }, - { PCI_VDEVICE(MELLANOX, 0x100b) }, - { PCI_VDEVICE(MELLANOX, 0x100c) }, - { PCI_VDEVICE(MELLANOX, 0x100d) }, - { PCI_VDEVICE(MELLANOX, 0x100e) }, - { PCI_VDEVICE(MELLANOX, 0x100f) }, +static DEFINE_PCI_DEVICE_TABLE(mlx4_pci_table) = { + /* MT25408 "Hermon" SDR */ + { PCI_VDEVICE(MELLANOX, 0x6340), MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25408 "Hermon" DDR */ + { PCI_VDEVICE(MELLANOX, 0x634a), MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25408 "Hermon" QDR */ + { PCI_VDEVICE(MELLANOX, 0x6354), MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25408 "Hermon" DDR PCIe gen2 */ + { PCI_VDEVICE(MELLANOX, 0x6732), MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25408 "Hermon" QDR PCIe gen2 */ + { PCI_VDEVICE(MELLANOX, 0x673c), MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25408 "Hermon" EN 10GigE */ + { PCI_VDEVICE(MELLANOX, 0x6368), MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25408 "Hermon" EN 10GigE PCIe gen2 */ + { PCI_VDEVICE(MELLANOX, 0x6750), MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25458 ConnectX EN 10GBASE-T 10GigE */ + { PCI_VDEVICE(MELLANOX, 0x6372), MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25458 ConnectX EN 10GBASE-T+Gen2 10GigE */ + { PCI_VDEVICE(MELLANOX, 0x675a), MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT26468 ConnectX EN 10GigE PCIe gen2*/ + { PCI_VDEVICE(MELLANOX, 0x6764), MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT26438 ConnectX EN 40GigE PCIe gen2 5GT/s */ + { PCI_VDEVICE(MELLANOX, 0x6746), MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT26478 ConnectX2 40GigE PCIe gen2 */ + { PCI_VDEVICE(MELLANOX, 0x676e), MLX4_PCI_DEV_FORCE_SENSE_PORT }, + /* MT25400 Family [ConnectX-2 Virtual Function] */ + { PCI_VDEVICE(MELLANOX, 0x1002), MLX4_PCI_DEV_IS_VF }, + /* MT27500 Family [ConnectX-3] */ + { PCI_VDEVICE(MELLANOX, 0x1003), 0 }, + /* MT27500 Family [ConnectX-3 Virtual Function] */ + { PCI_VDEVICE(MELLANOX, 0x1004), MLX4_PCI_DEV_IS_VF }, + { PCI_VDEVICE(MELLANOX, 0x1005), 0 }, /* MT27510 Family */ + { PCI_VDEVICE(MELLANOX, 0x1006), 0 }, /* MT27511 Family */ + { PCI_VDEVICE(MELLANOX, 0x1007), 0 }, /* MT27520 Family */ + { PCI_VDEVICE(MELLANOX, 0x1008), 0 }, /* MT27521 Family */ + { PCI_VDEVICE(MELLANOX, 0x1009), 0 }, /* MT27530 Family */ + { PCI_VDEVICE(MELLANOX, 0x100a), 0 }, /* MT27531 Family */ + { PCI_VDEVICE(MELLANOX, 0x100b), 0 }, /* MT27540 Family */ + { PCI_VDEVICE(MELLANOX, 0x100c), 0 }, /* MT27541 Family */ + { PCI_VDEVICE(MELLANOX, 0x100d), 0 }, /* MT27550 Family */ + { PCI_VDEVICE(MELLANOX, 0x100e), 0 }, /* MT27551 Family */ + { PCI_VDEVICE(MELLANOX, 0x100f), 0 }, /* MT27560 Family */ + { PCI_VDEVICE(MELLANOX, 0x1010), 0 }, /* MT27561 Family */ { 0, } }; MODULE_DEVICE_TABLE(pci, mlx4_pci_table); +static pci_ers_result_t mlx4_pci_err_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + mlx4_remove_one(pdev); + + return state == pci_channel_io_perm_failure ? + PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t mlx4_pci_slot_reset(struct pci_dev *pdev) +{ + int ret = __mlx4_init_one(pdev, 0); + + return ret ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; +} + +static const struct pci_error_handlers mlx4_err_handler = { + .error_detected = mlx4_pci_err_detected, + .slot_reset = mlx4_pci_slot_reset, +}; + +static int suspend(struct pci_dev *pdev, pm_message_t state) +{ + mlx4_remove_one(pdev); + + return 0; +} + +static int resume(struct pci_dev *pdev) +{ + return __mlx4_init_one(pdev, 0); +} + static struct pci_driver mlx4_driver = { .name = DRV_NAME, .id_table = mlx4_pci_table, .probe = mlx4_init_one, - .remove = __devexit_p(mlx4_remove_one) + .remove = __devexit_p(mlx4_remove_one), + .suspend = suspend, + .resume = resume, + .err_handler = &mlx4_err_handler, }; static int __init mlx4_verify_params(void) { + int status; + + status = update_defaults(&port_type_array); + if (status == INVALID_STR) { + if (mlx4_fill_dbdf2val_tbl(&port_type_array.dbdf2val)) + return -1; + } else if (status == INVALID_DATA) { + return -1; + } + + status = update_defaults(&num_vfs); + if (status == INVALID_STR) { + if (mlx4_fill_dbdf2val_tbl(&num_vfs.dbdf2val)) + return -1; + } else if (status == INVALID_DATA) { + return -1; + } + + status = update_defaults(&probe_vf); + if (status == INVALID_STR) { + if (mlx4_fill_dbdf2val_tbl(&probe_vf.dbdf2val)) + return -1; + } else if (status == INVALID_DATA) { + return -1; + } + + if (msi_x < 0) { + pr_warn("mlx4_core: bad msi_x: %d\n", msi_x); + return -1; + } + if ((log_num_mac < 0) || (log_num_mac > 7)) { - printk(KERN_WARNING "mlx4_core: bad num_mac: %d\n", log_num_mac); + pr_warning("mlx4_core: bad num_mac: %d\n", log_num_mac); return -1; } - if (log_mtts_per_seg == 0) - log_mtts_per_seg = ilog2(MLX4_MTT_ENTRY_PER_SEG); - if ((log_mtts_per_seg < 1) || (log_mtts_per_seg > 7)) { - printk(KERN_WARNING "mlx4_core: bad log_mtts_per_seg: %d\n", log_mtts_per_seg); + if (log_num_vlan != 0) + pr_warning("mlx4_core: log_num_vlan - obsolete module param, using %d\n", + MLX4_LOG_NUM_VLANS); + + if (mlx4_set_4k_mtu != -1) + pr_warning("mlx4_core: set_4k_mtu - obsolete module param\n"); + + if ((log_mtts_per_seg < 0) || (log_mtts_per_seg > 7)) { + pr_warning("mlx4_core: bad log_mtts_per_seg: %d\n", log_mtts_per_seg); return -1; } + if (mlx4_log_num_mgm_entry_size != -1 && + (mlx4_log_num_mgm_entry_size < MLX4_MIN_MGM_LOG_ENTRY_SIZE || + mlx4_log_num_mgm_entry_size > MLX4_MAX_MGM_LOG_ENTRY_SIZE)) { + pr_warning("mlx4_core: mlx4_log_num_mgm_entry_size (%d) not " + "in legal range (-1 or %d..%d)\n", + mlx4_log_num_mgm_entry_size, + MLX4_MIN_MGM_LOG_ENTRY_SIZE, + MLX4_MAX_MGM_LOG_ENTRY_SIZE); + return -1; + } + + if (mod_param_profile.num_qp < 18 || mod_param_profile.num_qp > 23) { + pr_warning("mlx4_core: bad log_num_qp: %d\n", + mod_param_profile.num_qp); + return -1; + } + + if (mod_param_profile.num_srq < 10) { + pr_warning("mlx4_core: too low log_num_srq: %d\n", + mod_param_profile.num_srq); + return -1; + } + + if (mod_param_profile.num_cq < 10) { + pr_warning("mlx4_core: too low log_num_cq: %d\n", + mod_param_profile.num_cq); + return -1; + } + + if (mod_param_profile.num_mpt < 10) { + pr_warning("mlx4_core: too low log_num_mpt: %d\n", + mod_param_profile.num_mpt); + return -1; + } + + if (mod_param_profile.num_mtt_segs && + mod_param_profile.num_mtt_segs < 15) { + pr_warning("mlx4_core: too low log_num_mtt: %d\n", + mod_param_profile.num_mtt_segs); + return -1; + } + + if (mod_param_profile.num_mtt_segs > MLX4_MAX_LOG_NUM_MTT) { + pr_warning("mlx4_core: too high log_num_mtt: %d\n", + mod_param_profile.num_mtt_segs); + return -1; + } return 0; } static int __init mlx4_init(void) { int ret; - mutex_init(&drv_mutex); - if (mlx4_verify_params()) return -EINVAL; mlx4_catas_init(); mlx4_wq = create_singlethread_workqueue("mlx4"); if (!mlx4_wq) return -ENOMEM; + if (enable_sys_tune) + sys_tune_init(); + ret = pci_register_driver(&mlx4_driver); - return ret < 0 ? ret : 0; + if (ret < 0) + goto err; + + return 0; + +err: + if (enable_sys_tune) + sys_tune_fini(); + + destroy_workqueue(mlx4_wq); + + return ret; } static void __exit mlx4_cleanup(void) { - mutex_lock(&drv_mutex); - mlx4_config_cleanup(); + if (enable_sys_tune) + sys_tune_fini(); + pci_unregister_driver(&mlx4_driver); - mutex_unlock(&drv_mutex); destroy_workqueue(mlx4_wq); } module_init_order(mlx4_init, SI_ORDER_MIDDLE); module_exit(mlx4_cleanup); -#undef MODULE_VERSION #include static int mlx4_evhand(module_t mod, int event, void *arg) { - return (0); + return (0); } static moduledata_t mlx4_mod = { - .name = "mlx4", - .evhand = mlx4_evhand, + .name = "mlx4", + .evhand = mlx4_evhand, }; MODULE_VERSION(mlx4, 1); DECLARE_MODULE(mlx4, mlx4_mod, SI_SUB_OFED_PREINIT, SI_ORDER_ANY); Index: stable/9/sys/ofed/drivers/net/mlx4/mcg.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/mcg.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/mcg.c (revision 273864) @@ -1,366 +1,1541 @@ /* * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include #include -#include +#include #include -#include +#include #include "mlx4.h" -#define MGM_QPN_MASK 0x00FFFFFF -#define MGM_BLCK_LB_BIT 30 -struct mlx4_mgm { - __be32 next_gid_index; - __be32 members_count; - u32 reserved[2]; - u8 gid[16]; - __be32 qp[MLX4_QP_PER_MGM]; -}; - static const u8 zero_gid[16]; /* automatically initialized to 0 */ -static int mlx4_READ_MCG(struct mlx4_dev *dev, int index, - struct mlx4_cmd_mailbox *mailbox) +int mlx4_get_mgm_entry_size(struct mlx4_dev *dev) { + return 1 << dev->oper_log_mgm_entry_size; +} + +int mlx4_get_qp_per_mgm(struct mlx4_dev *dev) +{ + return 4 * (mlx4_get_mgm_entry_size(dev) / 16 - 2); +} + +static int mlx4_QP_FLOW_STEERING_ATTACH(struct mlx4_dev *dev, + struct mlx4_cmd_mailbox *mailbox, + u32 size, + u64 *reg_id) +{ + u64 imm; + int err = 0; + + err = mlx4_cmd_imm(dev, mailbox->dma, &imm, size, 0, + MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + return err; + *reg_id = imm; + + return err; +} + +static int mlx4_QP_FLOW_STEERING_DETACH(struct mlx4_dev *dev, u64 regid) +{ + int err = 0; + + err = mlx4_cmd(dev, regid, 0, 0, + MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + + return err; +} + +static int mlx4_READ_ENTRY(struct mlx4_dev *dev, int index, + struct mlx4_cmd_mailbox *mailbox) +{ return mlx4_cmd_box(dev, 0, mailbox->dma, index, 0, MLX4_CMD_READ_MCG, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); } -static int mlx4_WRITE_MCG(struct mlx4_dev *dev, int index, - struct mlx4_cmd_mailbox *mailbox) +static int mlx4_WRITE_ENTRY(struct mlx4_dev *dev, int index, + struct mlx4_cmd_mailbox *mailbox) { return mlx4_cmd(dev, mailbox->dma, index, 0, MLX4_CMD_WRITE_MCG, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); } -static int mlx4_MGID_HASH(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, - u16 *hash) +static int mlx4_WRITE_PROMISC(struct mlx4_dev *dev, u8 port, u8 steer, + struct mlx4_cmd_mailbox *mailbox) { + u32 in_mod; + + in_mod = (u32) port << 16 | steer << 1; + return mlx4_cmd(dev, mailbox->dma, in_mod, 0x1, + MLX4_CMD_WRITE_MCG, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); +} + +static int mlx4_GID_HASH(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, + u16 *hash, u8 op_mod) +{ u64 imm; int err; - err = mlx4_cmd_imm(dev, mailbox->dma, &imm, 0, 0, MLX4_CMD_MGID_HASH, - MLX4_CMD_TIME_CLASS_A); + err = mlx4_cmd_imm(dev, mailbox->dma, &imm, 0, op_mod, + MLX4_CMD_MGID_HASH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); if (!err) *hash = imm; return err; } +static struct mlx4_promisc_qp *get_promisc_qp(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, + u32 qpn) +{ + struct mlx4_steer *s_steer; + struct mlx4_promisc_qp *pqp; + + if (port < 1 || port > dev->caps.num_ports) + return NULL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + + list_for_each_entry(pqp, &s_steer->promisc_qps[steer], list) { + if (pqp->qpn == qpn) + return pqp; + } + /* not found */ + return NULL; +} + /* + * Add new entry to steering data structure. + * All promisc QPs should be added as well + */ +static int new_steering_entry(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, + unsigned int index, u32 qpn) +{ + struct mlx4_steer *s_steer; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mgm *mgm; + u32 members_count; + struct mlx4_steer_index *new_entry; + struct mlx4_promisc_qp *pqp; + struct mlx4_promisc_qp *dqp = NULL; + u32 prot; + int err; + + if (port < 1 || port > dev->caps.num_ports) + return -EINVAL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + new_entry = kzalloc(sizeof *new_entry, GFP_KERNEL); + if (!new_entry) + return -ENOMEM; + + INIT_LIST_HEAD(&new_entry->duplicates); + new_entry->index = index; + list_add_tail(&new_entry->list, &s_steer->steer_entries[steer]); + + /* If the given qpn is also a promisc qp, + * it should be inserted to duplicates list + */ + pqp = get_promisc_qp(dev, port, steer, qpn); + if (pqp) { + dqp = kmalloc(sizeof *dqp, GFP_KERNEL); + if (!dqp) { + err = -ENOMEM; + goto out_alloc; + } + dqp->qpn = qpn; + list_add_tail(&dqp->list, &new_entry->duplicates); + } + + /* if no promisc qps for this vep, we are done */ + if (list_empty(&s_steer->promisc_qps[steer])) + return 0; + + /* now need to add all the promisc qps to the new + * steering entry, as they should also receive the packets + * destined to this address */ + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = -ENOMEM; + goto out_alloc; + } + mgm = mailbox->buf; + + err = mlx4_READ_ENTRY(dev, index, mailbox); + if (err) + goto out_mailbox; + + members_count = be32_to_cpu(mgm->members_count) & 0xffffff; + prot = be32_to_cpu(mgm->members_count) >> 30; + list_for_each_entry(pqp, &s_steer->promisc_qps[steer], list) { + /* don't add already existing qpn */ + if (pqp->qpn == qpn) + continue; + if (members_count == dev->caps.num_qp_per_mgm) { + /* out of space */ + err = -ENOMEM; + goto out_mailbox; + } + + /* add the qpn */ + mgm->qp[members_count++] = cpu_to_be32(pqp->qpn & MGM_QPN_MASK); + } + /* update the qps count and update the entry with all the promisc qps*/ + mgm->members_count = cpu_to_be32(members_count | (prot << 30)); + err = mlx4_WRITE_ENTRY(dev, index, mailbox); + +out_mailbox: + mlx4_free_cmd_mailbox(dev, mailbox); + if (!err) + return 0; +out_alloc: + if (dqp) { + list_del(&dqp->list); + kfree(dqp); + } + list_del(&new_entry->list); + kfree(new_entry); + return err; +} + +/* update the data structures with existing steering entry */ +static int existing_steering_entry(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, + unsigned int index, u32 qpn) +{ + struct mlx4_steer *s_steer; + struct mlx4_steer_index *tmp_entry, *entry = NULL; + struct mlx4_promisc_qp *pqp; + struct mlx4_promisc_qp *dqp; + + if (port < 1 || port > dev->caps.num_ports) + return -EINVAL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + + pqp = get_promisc_qp(dev, port, steer, qpn); + if (!pqp) + return 0; /* nothing to do */ + + list_for_each_entry(tmp_entry, &s_steer->steer_entries[steer], list) { + if (tmp_entry->index == index) { + entry = tmp_entry; + break; + } + } + if (unlikely(!entry)) { + mlx4_warn(dev, "Steering entry at index %x is not registered\n", index); + return -EINVAL; + } + + /* the given qpn is listed as a promisc qpn + * we need to add it as a duplicate to this entry + * for future references */ + list_for_each_entry(dqp, &entry->duplicates, list) { + if (qpn == dqp->qpn) + return 0; /* qp is already duplicated */ + } + + /* add the qp as a duplicate on this index */ + dqp = kmalloc(sizeof *dqp, GFP_KERNEL); + if (!dqp) + return -ENOMEM; + dqp->qpn = qpn; + list_add_tail(&dqp->list, &entry->duplicates); + + return 0; +} + +/* Check whether a qpn is a duplicate on steering entry + * If so, it should not be removed from mgm */ +static bool check_duplicate_entry(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, + unsigned int index, u32 qpn) +{ + struct mlx4_steer *s_steer; + struct mlx4_steer_index *tmp_entry, *entry = NULL; + struct mlx4_promisc_qp *dqp, *tmp_dqp; + + if (port < 1 || port > dev->caps.num_ports) + return NULL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + + /* if qp is not promisc, it cannot be duplicated */ + if (!get_promisc_qp(dev, port, steer, qpn)) + return false; + + /* The qp is promisc qp so it is a duplicate on this index + * Find the index entry, and remove the duplicate */ + list_for_each_entry(tmp_entry, &s_steer->steer_entries[steer], list) { + if (tmp_entry->index == index) { + entry = tmp_entry; + break; + } + } + if (unlikely(!entry)) { + mlx4_warn(dev, "Steering entry for index %x is not registered\n", index); + return false; + } + list_for_each_entry_safe(dqp, tmp_dqp, &entry->duplicates, list) { + if (dqp->qpn == qpn) { + list_del(&dqp->list); + kfree(dqp); + } + } + return true; +} + +/* + * returns true if all the QPs != tqpn contained in this entry + * are Promisc QPs. return false otherwise. + */ +static bool promisc_steering_entry(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, + unsigned int index, u32 tqpn, u32 *members_count) +{ + struct mlx4_steer *s_steer; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mgm *mgm; + u32 m_count; + bool ret = false; + int i; + + if (port < 1 || port > dev->caps.num_ports) + return false; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return false; + mgm = mailbox->buf; + + if (mlx4_READ_ENTRY(dev, index, mailbox)) + goto out; + m_count = be32_to_cpu(mgm->members_count) & 0xffffff; + if (members_count) + *members_count = m_count; + + for (i = 0; i < m_count; i++) { + u32 qpn = be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK; + if (!get_promisc_qp(dev, port, steer, qpn) && qpn != tqpn) { + /* the qp is not promisc, the entry can't be removed */ + goto out; + } + } + ret = true; +out: + mlx4_free_cmd_mailbox(dev, mailbox); + return ret; +} + +/* IF a steering entry contains only promisc QPs, it can be removed. */ +static bool can_remove_steering_entry(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, + unsigned int index, u32 tqpn) +{ + struct mlx4_steer *s_steer; + struct mlx4_steer_index *entry = NULL, *tmp_entry; + u32 members_count; + bool ret = false; + + if (port < 1 || port > dev->caps.num_ports) + return NULL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + + if (!promisc_steering_entry(dev, port, steer, index, tqpn, &members_count)) + goto out; + + /* All the qps currently registered for this entry are promiscuous, + * Checking for duplicates */ + ret = true; + list_for_each_entry_safe(entry, tmp_entry, &s_steer->steer_entries[steer], list) { + if (entry->index == index) { + if (list_empty(&entry->duplicates) || members_count == 1) { + struct mlx4_promisc_qp *pqp, *tmp_pqp; + /* + * If there is only 1 entry in duplicates than + * this is the QP we want to delete, going over + * the list and deleting the entry. + */ + list_del(&entry->list); + list_for_each_entry_safe(pqp, tmp_pqp, + &entry->duplicates, + list) { + list_del(&pqp->list); + kfree(pqp); + } + kfree(entry); + } else { + /* This entry contains duplicates so it shouldn't be removed */ + ret = false; + goto out; + } + } + } + +out: + return ret; +} + +static int add_promisc_qp(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, u32 qpn) +{ + struct mlx4_steer *s_steer; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mgm *mgm; + struct mlx4_steer_index *entry; + struct mlx4_promisc_qp *pqp; + struct mlx4_promisc_qp *dqp; + u32 members_count; + u32 prot; + int i; + bool found; + int err; + struct mlx4_priv *priv = mlx4_priv(dev); + + if (port < 1 || port > dev->caps.num_ports) + return -EINVAL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + + mutex_lock(&priv->mcg_table.mutex); + + if (get_promisc_qp(dev, port, steer, qpn)) { + err = 0; /* Noting to do, already exists */ + goto out_mutex; + } + + pqp = kmalloc(sizeof *pqp, GFP_KERNEL); + if (!pqp) { + err = -ENOMEM; + goto out_mutex; + } + pqp->qpn = qpn; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = -ENOMEM; + goto out_alloc; + } + mgm = mailbox->buf; + + if (!(mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)) { + /* the promisc qp needs to be added for each one of the steering + * entries, if it already exists, needs to be added as a duplicate + * for this entry */ + list_for_each_entry(entry, &s_steer->steer_entries[steer], list) { + err = mlx4_READ_ENTRY(dev, entry->index, mailbox); + if (err) + goto out_mailbox; + + members_count = be32_to_cpu(mgm->members_count) & 0xffffff; + prot = be32_to_cpu(mgm->members_count) >> 30; + found = false; + for (i = 0; i < members_count; i++) { + if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qpn) { + /* Entry already exists, add to duplicates */ + dqp = kmalloc(sizeof *dqp, GFP_KERNEL); + if (!dqp) { + err = -ENOMEM; + goto out_mailbox; + } + dqp->qpn = qpn; + list_add_tail(&dqp->list, &entry->duplicates); + found = true; + } + } + if (!found) { + /* Need to add the qpn to mgm */ + if (members_count == dev->caps.num_qp_per_mgm) { + /* entry is full */ + err = -ENOMEM; + goto out_mailbox; + } + mgm->qp[members_count++] = cpu_to_be32(qpn & MGM_QPN_MASK); + mgm->members_count = cpu_to_be32(members_count | (prot << 30)); + err = mlx4_WRITE_ENTRY(dev, entry->index, mailbox); + if (err) + goto out_mailbox; + } + } + } + + /* add the new qpn to list of promisc qps */ + list_add_tail(&pqp->list, &s_steer->promisc_qps[steer]); + /* now need to add all the promisc qps to default entry */ + memset(mgm, 0, sizeof *mgm); + members_count = 0; + list_for_each_entry(dqp, &s_steer->promisc_qps[steer], list) { + if (members_count == dev->caps.num_qp_per_mgm) { + /* entry is full */ + err = -ENOMEM; + goto out_list; + } + mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK); + } + mgm->members_count = cpu_to_be32(members_count | MLX4_PROT_ETH << 30); + + err = mlx4_WRITE_PROMISC(dev, port, steer, mailbox); + if (err) + goto out_list; + + mlx4_free_cmd_mailbox(dev, mailbox); + mutex_unlock(&priv->mcg_table.mutex); + return 0; + +out_list: + list_del(&pqp->list); +out_mailbox: + mlx4_free_cmd_mailbox(dev, mailbox); +out_alloc: + kfree(pqp); +out_mutex: + mutex_unlock(&priv->mcg_table.mutex); + return err; +} + +static int remove_promisc_qp(struct mlx4_dev *dev, u8 port, + enum mlx4_steer_type steer, u32 qpn) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_steer *s_steer; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mgm *mgm; + struct mlx4_steer_index *entry, *tmp_entry; + struct mlx4_promisc_qp *pqp; + struct mlx4_promisc_qp *dqp; + u32 members_count; + bool found; + bool back_to_list = false; + int i, loc = -1; + int err; + + if (port < 1 || port > dev->caps.num_ports) + return -EINVAL; + + s_steer = &mlx4_priv(dev)->steer[port - 1]; + mutex_lock(&priv->mcg_table.mutex); + + pqp = get_promisc_qp(dev, port, steer, qpn); + if (unlikely(!pqp)) { + mlx4_warn(dev, "QP %x is not promiscuous QP\n", qpn); + /* nothing to do */ + err = 0; + goto out_mutex; + } + + /*remove from list of promisc qps */ + list_del(&pqp->list); + + /* set the default entry not to include the removed one */ + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = -ENOMEM; + back_to_list = true; + goto out_list; + } + mgm = mailbox->buf; + memset(mgm, 0, sizeof *mgm); + members_count = 0; + list_for_each_entry(dqp, &s_steer->promisc_qps[steer], list) + mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK); + mgm->members_count = cpu_to_be32(members_count | MLX4_PROT_ETH << 30); + + err = mlx4_WRITE_PROMISC(dev, port, steer, mailbox); + if (err) + goto out_mailbox; + + if (!(mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)) { + /* remove the qp from all the steering entries*/ + list_for_each_entry_safe(entry, tmp_entry, &s_steer->steer_entries[steer], list) { + found = false; + list_for_each_entry(dqp, &entry->duplicates, list) { + if (dqp->qpn == qpn) { + found = true; + break; + } + } + if (found) { + /* a duplicate, no need to change the mgm, + * only update the duplicates list */ + list_del(&dqp->list); + kfree(dqp); + } else { + err = mlx4_READ_ENTRY(dev, entry->index, mailbox); + if (err) + goto out_mailbox; + members_count = be32_to_cpu(mgm->members_count) & 0xffffff; + if (!members_count) { + mlx4_warn(dev, "QP %06x wasn't found in entry %x mcount=0." + " deleting entry...\n", qpn, entry->index); + list_del(&entry->list); + kfree(entry); + continue; + } + + for (i = 0; i < members_count; ++i) + if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qpn) { + loc = i; + break; + } + + if (loc < 0) { + mlx4_err(dev, "QP %06x wasn't found in entry %d\n", + qpn, entry->index); + err = -EINVAL; + goto out_mailbox; + } + + /* copy the last QP in this MGM over removed QP */ + mgm->qp[loc] = mgm->qp[members_count - 1]; + mgm->qp[members_count - 1] = 0; + mgm->members_count = cpu_to_be32(--members_count | + (MLX4_PROT_ETH << 30)); + + err = mlx4_WRITE_ENTRY(dev, entry->index, mailbox); + if (err) + goto out_mailbox; + } + } + } + +out_mailbox: + mlx4_free_cmd_mailbox(dev, mailbox); +out_list: + if (back_to_list) + list_add_tail(&pqp->list, &s_steer->promisc_qps[steer]); + else + kfree(pqp); +out_mutex: + mutex_unlock(&priv->mcg_table.mutex); + return err; +} + +/* * Caller must hold MCG table semaphore. gid and mgm parameters must * be properly aligned for command interface. * * Returns 0 unless a firmware command error occurs. * * If GID is found in MGM or MGM is empty, *index = *hash, *prev = -1 * and *mgm holds MGM entry. * * if GID is found in AMGM, *index = index in AMGM, *prev = index of * previous entry in hash chain and *mgm holds AMGM entry. * * If no AMGM exists for given gid, *index = -1, *prev = index of last * entry in hash chain and *mgm holds end of hash chain. */ -static int find_mgm(struct mlx4_dev *dev, - u8 *gid, enum mlx4_mcast_prot prot, - struct mlx4_cmd_mailbox *mgm_mailbox, - u16 *hash, int *prev, int *index) +static int find_entry(struct mlx4_dev *dev, u8 port, + u8 *gid, enum mlx4_protocol prot, + struct mlx4_cmd_mailbox *mgm_mailbox, + int *prev, int *index) { struct mlx4_cmd_mailbox *mailbox; struct mlx4_mgm *mgm = mgm_mailbox->buf; u8 *mgid; int err; + u16 hash; + u8 op_mod = (prot == MLX4_PROT_ETH) ? + !!(dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) : 0; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return -ENOMEM; mgid = mailbox->buf; memcpy(mgid, gid, 16); - err = mlx4_MGID_HASH(dev, mailbox, hash); + err = mlx4_GID_HASH(dev, mailbox, &hash, op_mod); mlx4_free_cmd_mailbox(dev, mailbox); if (err) return err; if (0) - mlx4_dbg(dev, "Hash for %pI6 is %04x\n", gid, *hash); + mlx4_dbg(dev, "Hash for %pI6 is %04x\n", gid, hash); - *index = *hash; + *index = hash; *prev = -1; do { - err = mlx4_READ_MCG(dev, *index, mgm_mailbox); + err = mlx4_READ_ENTRY(dev, *index, mgm_mailbox); if (err) return err; - if (!memcmp(mgm->gid, zero_gid, 16)) { - if (*index != *hash) { + if (!(be32_to_cpu(mgm->members_count) & 0xffffff)) { + if (*index != hash) { mlx4_err(dev, "Found zero MGID in AMGM.\n"); err = -EINVAL; } return err; } if (!memcmp(mgm->gid, gid, 16) && - (prot == be32_to_cpu(mgm->members_count) >> 30)) + be32_to_cpu(mgm->members_count) >> 30 == prot) return err; *prev = *index; *index = be32_to_cpu(mgm->next_gid_index) >> 6; } while (*index); *index = -1; return err; } -int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - int block_mcast_loopback, enum mlx4_mcast_prot prot) +static const u8 __promisc_mode[] = { + [MLX4_FS_REGULAR] = 0x0, + [MLX4_FS_ALL_DEFAULT] = 0x1, + [MLX4_FS_MC_DEFAULT] = 0x3, + [MLX4_FS_UC_SNIFFER] = 0x4, + [MLX4_FS_MC_SNIFFER] = 0x5, +}; + +int map_sw_to_hw_steering_mode(struct mlx4_dev *dev, + enum mlx4_net_trans_promisc_mode flow_type) { + if (flow_type >= MLX4_FS_MODE_NUM || flow_type < 0) { + mlx4_err(dev, "Invalid flow type. type = %d\n", flow_type); + return -EINVAL; + } + return __promisc_mode[flow_type]; +} +EXPORT_SYMBOL_GPL(map_sw_to_hw_steering_mode); + +static void trans_rule_ctrl_to_hw(struct mlx4_net_trans_rule *ctrl, + struct mlx4_net_trans_rule_hw_ctrl *hw) +{ + u8 flags = 0; + + flags = ctrl->queue_mode == MLX4_NET_TRANS_Q_LIFO ? 1 : 0; + flags |= ctrl->exclusive ? (1 << 2) : 0; + flags |= ctrl->allow_loopback ? (1 << 3) : 0; + + hw->flags = flags; + hw->type = __promisc_mode[ctrl->promisc_mode]; + hw->prio = cpu_to_be16(ctrl->priority); + hw->port = ctrl->port; + hw->qpn = cpu_to_be32(ctrl->qpn); +} + +const u16 __sw_id_hw[] = { + [MLX4_NET_TRANS_RULE_ID_ETH] = 0xE001, + [MLX4_NET_TRANS_RULE_ID_IB] = 0xE005, + [MLX4_NET_TRANS_RULE_ID_IPV6] = 0xE003, + [MLX4_NET_TRANS_RULE_ID_IPV4] = 0xE002, + [MLX4_NET_TRANS_RULE_ID_TCP] = 0xE004, + [MLX4_NET_TRANS_RULE_ID_UDP] = 0xE006 +}; + +int map_sw_to_hw_steering_id(struct mlx4_dev *dev, + enum mlx4_net_trans_rule_id id) +{ + if (id >= MLX4_NET_TRANS_RULE_NUM || id < 0) { + mlx4_err(dev, "Invalid network rule id. id = %d\n", id); + return -EINVAL; + } + return __sw_id_hw[id]; +} +EXPORT_SYMBOL_GPL(map_sw_to_hw_steering_id); + +static const int __rule_hw_sz[] = { + [MLX4_NET_TRANS_RULE_ID_ETH] = + sizeof(struct mlx4_net_trans_rule_hw_eth), + [MLX4_NET_TRANS_RULE_ID_IB] = + sizeof(struct mlx4_net_trans_rule_hw_ib), + [MLX4_NET_TRANS_RULE_ID_IPV6] = 0, + [MLX4_NET_TRANS_RULE_ID_IPV4] = + sizeof(struct mlx4_net_trans_rule_hw_ipv4), + [MLX4_NET_TRANS_RULE_ID_TCP] = + sizeof(struct mlx4_net_trans_rule_hw_tcp_udp), + [MLX4_NET_TRANS_RULE_ID_UDP] = + sizeof(struct mlx4_net_trans_rule_hw_tcp_udp) +}; + +int hw_rule_sz(struct mlx4_dev *dev, + enum mlx4_net_trans_rule_id id) +{ + if (id >= MLX4_NET_TRANS_RULE_NUM || id < 0) { + mlx4_err(dev, "Invalid network rule id. id = %d\n", id); + return -EINVAL; + } + + return __rule_hw_sz[id]; +} +EXPORT_SYMBOL_GPL(hw_rule_sz); + +static int parse_trans_rule(struct mlx4_dev *dev, struct mlx4_spec_list *spec, + struct _rule_hw *rule_hw) +{ + if (hw_rule_sz(dev, spec->id) < 0) + return -EINVAL; + memset(rule_hw, 0, hw_rule_sz(dev, spec->id)); + rule_hw->id = cpu_to_be16(__sw_id_hw[spec->id]); + rule_hw->size = hw_rule_sz(dev, spec->id) >> 2; + + switch (spec->id) { + case MLX4_NET_TRANS_RULE_ID_ETH: + memcpy(rule_hw->eth.dst_mac, spec->eth.dst_mac, ETH_ALEN); + memcpy(rule_hw->eth.dst_mac_msk, spec->eth.dst_mac_msk, + ETH_ALEN); + memcpy(rule_hw->eth.src_mac, spec->eth.src_mac, ETH_ALEN); + memcpy(rule_hw->eth.src_mac_msk, spec->eth.src_mac_msk, + ETH_ALEN); + if (spec->eth.ether_type_enable) { + rule_hw->eth.ether_type_enable = 1; + rule_hw->eth.ether_type = spec->eth.ether_type; + } + rule_hw->eth.vlan_tag = spec->eth.vlan_id; + rule_hw->eth.vlan_tag_msk = spec->eth.vlan_id_msk; + break; + + case MLX4_NET_TRANS_RULE_ID_IB: + rule_hw->ib.l3_qpn = spec->ib.l3_qpn; + rule_hw->ib.qpn_mask = spec->ib.qpn_msk; + memcpy(&rule_hw->ib.dst_gid, &spec->ib.dst_gid, 16); + memcpy(&rule_hw->ib.dst_gid_msk, &spec->ib.dst_gid_msk, 16); + break; + + case MLX4_NET_TRANS_RULE_ID_IPV6: + return -EOPNOTSUPP; + + case MLX4_NET_TRANS_RULE_ID_IPV4: + rule_hw->ipv4.src_ip = spec->ipv4.src_ip; + rule_hw->ipv4.src_ip_msk = spec->ipv4.src_ip_msk; + rule_hw->ipv4.dst_ip = spec->ipv4.dst_ip; + rule_hw->ipv4.dst_ip_msk = spec->ipv4.dst_ip_msk; + break; + + case MLX4_NET_TRANS_RULE_ID_TCP: + case MLX4_NET_TRANS_RULE_ID_UDP: + rule_hw->tcp_udp.dst_port = spec->tcp_udp.dst_port; + rule_hw->tcp_udp.dst_port_msk = spec->tcp_udp.dst_port_msk; + rule_hw->tcp_udp.src_port = spec->tcp_udp.src_port; + rule_hw->tcp_udp.src_port_msk = spec->tcp_udp.src_port_msk; + break; + + default: + return -EINVAL; + } + + return __rule_hw_sz[spec->id]; +} + +static void mlx4_err_rule(struct mlx4_dev *dev, char *str, + struct mlx4_net_trans_rule *rule) +{ +#define BUF_SIZE 256 + struct mlx4_spec_list *cur; + char buf[BUF_SIZE]; + int len = 0; + + mlx4_err(dev, "%s", str); + len += snprintf(buf + len, BUF_SIZE - len, + "port = %d prio = 0x%x qp = 0x%x ", + rule->port, rule->priority, rule->qpn); + + list_for_each_entry(cur, &rule->list, list) { + switch (cur->id) { + case MLX4_NET_TRANS_RULE_ID_ETH: + len += snprintf(buf + len, BUF_SIZE - len, + "dmac = %pM ", &cur->eth.dst_mac); + if (cur->eth.ether_type) + len += snprintf(buf + len, BUF_SIZE - len, + "ethertype = 0x%x ", + be16_to_cpu(cur->eth.ether_type)); + if (cur->eth.vlan_id) + len += snprintf(buf + len, BUF_SIZE - len, + "vlan-id = %d ", + be16_to_cpu(cur->eth.vlan_id)); + break; + + case MLX4_NET_TRANS_RULE_ID_IPV4: + if (cur->ipv4.src_ip) + len += snprintf(buf + len, BUF_SIZE - len, + "src-ip = %pI4 ", + &cur->ipv4.src_ip); + if (cur->ipv4.dst_ip) + len += snprintf(buf + len, BUF_SIZE - len, + "dst-ip = %pI4 ", + &cur->ipv4.dst_ip); + break; + + case MLX4_NET_TRANS_RULE_ID_TCP: + case MLX4_NET_TRANS_RULE_ID_UDP: + if (cur->tcp_udp.src_port) + len += snprintf(buf + len, BUF_SIZE - len, + "src-port = %d ", + be16_to_cpu(cur->tcp_udp.src_port)); + if (cur->tcp_udp.dst_port) + len += snprintf(buf + len, BUF_SIZE - len, + "dst-port = %d ", + be16_to_cpu(cur->tcp_udp.dst_port)); + break; + + case MLX4_NET_TRANS_RULE_ID_IB: + len += snprintf(buf + len, BUF_SIZE - len, + "dst-gid = %pI6\n", cur->ib.dst_gid); + len += snprintf(buf + len, BUF_SIZE - len, + "dst-gid-mask = %pI6\n", + cur->ib.dst_gid_msk); + break; + + case MLX4_NET_TRANS_RULE_ID_IPV6: + break; + + default: + break; + } + } + len += snprintf(buf + len, BUF_SIZE - len, "\n"); + mlx4_err(dev, "%s", buf); + + if (len >= BUF_SIZE) + mlx4_err(dev, "Network rule error message was truncated, print buffer is too small.\n"); +} + +int mlx4_flow_attach(struct mlx4_dev *dev, + struct mlx4_net_trans_rule *rule, u64 *reg_id) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_spec_list *cur; + u32 size = 0; + int ret; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + memset(mailbox->buf, 0, sizeof(struct mlx4_net_trans_rule_hw_ctrl)); + trans_rule_ctrl_to_hw(rule, mailbox->buf); + + size += sizeof(struct mlx4_net_trans_rule_hw_ctrl); + + list_for_each_entry(cur, &rule->list, list) { + ret = parse_trans_rule(dev, cur, mailbox->buf + size); + if (ret < 0) { + mlx4_free_cmd_mailbox(dev, mailbox); + return -EINVAL; + } + size += ret; + } + + ret = mlx4_QP_FLOW_STEERING_ATTACH(dev, mailbox, size >> 2, reg_id); + if (ret == -ENOMEM) + mlx4_err_rule(dev, + "mcg table is full. Fail to register network rule.\n", + rule); + else if (ret) + mlx4_err_rule(dev, "Fail to register network rule.\n", rule); + + mlx4_free_cmd_mailbox(dev, mailbox); + + return ret; +} +EXPORT_SYMBOL_GPL(mlx4_flow_attach); + +int mlx4_flow_detach(struct mlx4_dev *dev, u64 reg_id) +{ + int err; + + err = mlx4_QP_FLOW_STEERING_DETACH(dev, reg_id); + if (err) + mlx4_err(dev, "Fail to detach network rule. registration id = 0x%llx\n", + (unsigned long long)reg_id); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_flow_detach); + +int mlx4_FLOW_STEERING_IB_UC_QP_RANGE(struct mlx4_dev *dev, u32 min_range_qpn, u32 max_range_qpn) +{ + int err; + u64 in_param; + + in_param = ((u64) min_range_qpn) << 32; + in_param |= ((u64) max_range_qpn) & 0xFFFFFFFF; + + err = mlx4_cmd(dev, in_param, 0, 0, + MLX4_FLOW_STEERING_IB_UC_QP_RANGE, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); + + return err; +} +EXPORT_SYMBOL_GPL(mlx4_FLOW_STEERING_IB_UC_QP_RANGE); + +int mlx4_qp_attach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + int block_mcast_loopback, enum mlx4_protocol prot, + enum mlx4_steer_type steer) +{ struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cmd_mailbox *mailbox; struct mlx4_mgm *mgm; u32 members_count; - u16 hash; int index, prev; int link = 0; int i; int err; + u8 port = gid[5]; + u8 new_entry = 0; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); mgm = mailbox->buf; mutex_lock(&priv->mcg_table.mutex); - - err = find_mgm(dev, gid, prot, mailbox, &hash, &prev, &index); + err = find_entry(dev, port, gid, prot, + mailbox, &prev, &index); if (err) goto out; if (index != -1) { - if (!memcmp(mgm->gid, zero_gid, 16)) + if (!(be32_to_cpu(mgm->members_count) & 0xffffff)) { + new_entry = 1; memcpy(mgm->gid, gid, 16); + } } else { link = 1; index = mlx4_bitmap_alloc(&priv->mcg_table.bitmap); if (index == -1) { mlx4_err(dev, "No AMGM entries left\n"); err = -ENOMEM; goto out; } index += dev->caps.num_mgms; + new_entry = 1; memset(mgm, 0, sizeof *mgm); memcpy(mgm->gid, gid, 16); } members_count = be32_to_cpu(mgm->members_count) & 0xffffff; - if (members_count == MLX4_QP_PER_MGM) { + if (members_count == dev->caps.num_qp_per_mgm) { mlx4_err(dev, "MGM at index %x is full.\n", index); err = -ENOMEM; goto out; } for (i = 0; i < members_count; ++i) if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) { mlx4_dbg(dev, "QP %06x already a member of MGM\n", qp->qpn); err = 0; goto out; } mgm->qp[members_count++] = cpu_to_be32((qp->qpn & MGM_QPN_MASK) | (!!mlx4_blck_lb << MGM_BLCK_LB_BIT)); - mgm->members_count = cpu_to_be32(members_count | ((u32) prot << 30)); + mgm->members_count = cpu_to_be32(members_count | (u32) prot << 30); - err = mlx4_WRITE_MCG(dev, index, mailbox); + err = mlx4_WRITE_ENTRY(dev, index, mailbox); if (err) goto out; + /* if !link, still add the new entry. */ if (!link) - goto out; + goto skip_link; - err = mlx4_READ_MCG(dev, prev, mailbox); + err = mlx4_READ_ENTRY(dev, prev, mailbox); if (err) goto out; mgm->next_gid_index = cpu_to_be32(index << 6); - err = mlx4_WRITE_MCG(dev, prev, mailbox); + err = mlx4_WRITE_ENTRY(dev, prev, mailbox); if (err) goto out; +skip_link: + if (prot == MLX4_PROT_ETH) { + /* manage the steering entry for promisc mode */ + if (new_entry) + new_steering_entry(dev, port, steer, index, qp->qpn); + else + existing_steering_entry(dev, port, steer, + index, qp->qpn); + } + out: if (err && link && index != -1) { if (index < dev->caps.num_mgms) mlx4_warn(dev, "Got AMGM index %d < %d", index, dev->caps.num_mgms); else mlx4_bitmap_free(&priv->mcg_table.bitmap, - index - dev->caps.num_mgms); + index - dev->caps.num_mgms, MLX4_USE_RR); } mutex_unlock(&priv->mcg_table.mutex); mlx4_free_cmd_mailbox(dev, mailbox); return err; } -EXPORT_SYMBOL_GPL(mlx4_multicast_attach); -int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - enum mlx4_mcast_prot prot) +int mlx4_qp_detach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + enum mlx4_protocol prot, enum mlx4_steer_type steer) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cmd_mailbox *mailbox; struct mlx4_mgm *mgm; u32 members_count; - u16 hash; int prev, index; - int i, loc; + int i, loc = -1; int err; + u8 port = gid[5]; + bool removed_entry = false; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); mgm = mailbox->buf; mutex_lock(&priv->mcg_table.mutex); - err = find_mgm(dev, gid, prot, mailbox, &hash, &prev, &index); + err = find_entry(dev, port, gid, prot, + mailbox, &prev, &index); if (err) goto out; if (index == -1) { mlx4_err(dev, "MGID %pI6 not found\n", gid); err = -EINVAL; goto out; } + /* + if this QP is also a promisc QP, it shouldn't be removed only if + at least one none promisc QP is also attached to this MCG + */ + if (prot == MLX4_PROT_ETH && + check_duplicate_entry(dev, port, steer, index, qp->qpn) && + !promisc_steering_entry(dev, port, steer, index, qp->qpn, NULL)) + goto out; + members_count = be32_to_cpu(mgm->members_count) & 0xffffff; - for (loc = -1, i = 0; i < members_count; ++i) - if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) + for (i = 0; i < members_count; ++i) + if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) { loc = i; + break; + } if (loc == -1) { mlx4_err(dev, "QP %06x not found in MGM\n", qp->qpn); err = -EINVAL; goto out; } + /* copy the last QP in this MGM over removed QP */ + mgm->qp[loc] = mgm->qp[members_count - 1]; + mgm->qp[members_count - 1] = 0; + mgm->members_count = cpu_to_be32(--members_count | (u32) prot << 30); - mgm->members_count = cpu_to_be32(--members_count | ((u32) prot << 30)); - mgm->qp[loc] = mgm->qp[i - 1]; - mgm->qp[i - 1] = 0; - - if (i != 1) { - err = mlx4_WRITE_MCG(dev, index, mailbox); + if (prot == MLX4_PROT_ETH) + removed_entry = can_remove_steering_entry(dev, port, steer, + index, qp->qpn); + if (members_count && (prot != MLX4_PROT_ETH || !removed_entry)) { + err = mlx4_WRITE_ENTRY(dev, index, mailbox); goto out; } + /* We are going to delete the entry, members count should be 0 */ + mgm->members_count = cpu_to_be32((u32) prot << 30); + if (prev == -1) { /* Remove entry from MGM */ int amgm_index = be32_to_cpu(mgm->next_gid_index) >> 6; if (amgm_index) { - err = mlx4_READ_MCG(dev, amgm_index, mailbox); + err = mlx4_READ_ENTRY(dev, amgm_index, mailbox); if (err) goto out; } else memset(mgm->gid, 0, 16); - err = mlx4_WRITE_MCG(dev, index, mailbox); + err = mlx4_WRITE_ENTRY(dev, index, mailbox); if (err) goto out; if (amgm_index) { if (amgm_index < dev->caps.num_mgms) mlx4_warn(dev, "MGM entry %d had AMGM index %d < %d", index, amgm_index, dev->caps.num_mgms); else mlx4_bitmap_free(&priv->mcg_table.bitmap, - amgm_index - dev->caps.num_mgms); + amgm_index - dev->caps.num_mgms, MLX4_USE_RR); } } else { /* Remove entry from AMGM */ int cur_next_index = be32_to_cpu(mgm->next_gid_index) >> 6; - err = mlx4_READ_MCG(dev, prev, mailbox); + err = mlx4_READ_ENTRY(dev, prev, mailbox); if (err) goto out; mgm->next_gid_index = cpu_to_be32(cur_next_index << 6); - err = mlx4_WRITE_MCG(dev, prev, mailbox); + err = mlx4_WRITE_ENTRY(dev, prev, mailbox); if (err) goto out; if (index < dev->caps.num_mgms) mlx4_warn(dev, "entry %d had next AMGM index %d < %d", prev, index, dev->caps.num_mgms); else mlx4_bitmap_free(&priv->mcg_table.bitmap, - index - dev->caps.num_mgms); + index - dev->caps.num_mgms, MLX4_USE_RR); } out: mutex_unlock(&priv->mcg_table.mutex); mlx4_free_cmd_mailbox(dev, mailbox); return err; } + +static int mlx4_QP_ATTACH(struct mlx4_dev *dev, struct mlx4_qp *qp, + u8 gid[16], u8 attach, u8 block_loopback, + enum mlx4_protocol prot) +{ + struct mlx4_cmd_mailbox *mailbox; + int err = 0; + int qpn; + + if (!mlx4_is_mfunc(dev)) + return -EBADF; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + memcpy(mailbox->buf, gid, 16); + qpn = qp->qpn; + qpn |= (prot << 28); + if (attach && block_loopback) + qpn |= (1 << 31); + + err = mlx4_cmd(dev, mailbox->dma, qpn, attach, + MLX4_CMD_QP_ATTACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} + +int mlx4_trans_to_dmfs_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, + u8 gid[16], u8 port, + int block_mcast_loopback, + enum mlx4_protocol prot, u64 *reg_id) +{ + struct mlx4_spec_list spec = { {NULL} }; + __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); + + struct mlx4_net_trans_rule rule = { + .queue_mode = MLX4_NET_TRANS_Q_FIFO, + .exclusive = 0, + .promisc_mode = MLX4_FS_REGULAR, + .priority = MLX4_DOMAIN_NIC, + }; + + rule.allow_loopback = !block_mcast_loopback; + rule.port = port; + rule.qpn = qp->qpn; + INIT_LIST_HEAD(&rule.list); + + switch (prot) { + case MLX4_PROT_ETH: + spec.id = MLX4_NET_TRANS_RULE_ID_ETH; + memcpy(spec.eth.dst_mac, &gid[10], ETH_ALEN); + memcpy(spec.eth.dst_mac_msk, &mac_mask, ETH_ALEN); + break; + + case MLX4_PROT_IB_IPV6: + spec.id = MLX4_NET_TRANS_RULE_ID_IB; + memcpy(spec.ib.dst_gid, gid, 16); + memset(&spec.ib.dst_gid_msk, 0xff, 16); + break; + default: + return -EINVAL; + } + list_add_tail(&spec.list, &rule.list); + + return mlx4_flow_attach(dev, &rule, reg_id); +} + +int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + u8 port, int block_mcast_loopback, + enum mlx4_protocol prot, u64 *reg_id) +{ + enum mlx4_steer_type steer; + steer = (is_valid_ether_addr(&gid[10])) ? MLX4_UC_STEER : MLX4_MC_STEER; + + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_A0: + if (prot == MLX4_PROT_ETH) + return 0; + + case MLX4_STEERING_MODE_B0: + if (prot == MLX4_PROT_ETH) + gid[7] |= (steer << 1); + + if (mlx4_is_mfunc(dev)) + return mlx4_QP_ATTACH(dev, qp, gid, 1, + block_mcast_loopback, prot); + return mlx4_qp_attach_common(dev, qp, gid, + block_mcast_loopback, prot, + MLX4_MC_STEER); + + case MLX4_STEERING_MODE_DEVICE_MANAGED: + return mlx4_trans_to_dmfs_attach(dev, qp, gid, port, + block_mcast_loopback, + prot, reg_id); + default: + return -EINVAL; + } +} +EXPORT_SYMBOL_GPL(mlx4_multicast_attach); + +int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + enum mlx4_protocol prot, u64 reg_id) +{ + enum mlx4_steer_type steer; + steer = (is_valid_ether_addr(&gid[10])) ? MLX4_UC_STEER : MLX4_MC_STEER; + + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_A0: + if (prot == MLX4_PROT_ETH) + return 0; + + case MLX4_STEERING_MODE_B0: + if (prot == MLX4_PROT_ETH) + gid[7] |= (steer << 1); + + if (mlx4_is_mfunc(dev)) + return mlx4_QP_ATTACH(dev, qp, gid, 0, 0, prot); + + return mlx4_qp_detach_common(dev, qp, gid, prot, + MLX4_MC_STEER); + + case MLX4_STEERING_MODE_DEVICE_MANAGED: + return mlx4_flow_detach(dev, reg_id); + + default: + return -EINVAL; + } +} EXPORT_SYMBOL_GPL(mlx4_multicast_detach); +int mlx4_flow_steer_promisc_add(struct mlx4_dev *dev, u8 port, + u32 qpn, enum mlx4_net_trans_promisc_mode mode) +{ + struct mlx4_net_trans_rule rule; + u64 *regid_p; + + switch (mode) { + case MLX4_FS_ALL_DEFAULT: + regid_p = &dev->regid_promisc_array[port]; + break; + case MLX4_FS_MC_DEFAULT: + regid_p = &dev->regid_allmulti_array[port]; + break; + default: + return -1; + } + + if (*regid_p != 0) + return -1; + + rule.promisc_mode = mode; + rule.port = port; + rule.qpn = qpn; + INIT_LIST_HEAD(&rule.list); + mlx4_err(dev, "going promisc on %x\n", port); + + return mlx4_flow_attach(dev, &rule, regid_p); +} +EXPORT_SYMBOL_GPL(mlx4_flow_steer_promisc_add); + +int mlx4_flow_steer_promisc_remove(struct mlx4_dev *dev, u8 port, + enum mlx4_net_trans_promisc_mode mode) +{ + int ret; + u64 *regid_p; + + switch (mode) { + case MLX4_FS_ALL_DEFAULT: + regid_p = &dev->regid_promisc_array[port]; + break; + case MLX4_FS_MC_DEFAULT: + regid_p = &dev->regid_allmulti_array[port]; + break; + default: + return -1; + } + + if (*regid_p == 0) + return -1; + + ret = mlx4_flow_detach(dev, *regid_p); + if (ret == 0) + *regid_p = 0; + + return ret; +} +EXPORT_SYMBOL_GPL(mlx4_flow_steer_promisc_remove); + +int mlx4_unicast_attach(struct mlx4_dev *dev, + struct mlx4_qp *qp, u8 gid[16], + int block_mcast_loopback, enum mlx4_protocol prot) +{ + if (prot == MLX4_PROT_ETH) + gid[7] |= (MLX4_UC_STEER << 1); + + if (mlx4_is_mfunc(dev)) + return mlx4_QP_ATTACH(dev, qp, gid, 1, + block_mcast_loopback, prot); + + return mlx4_qp_attach_common(dev, qp, gid, block_mcast_loopback, + prot, MLX4_UC_STEER); +} +EXPORT_SYMBOL_GPL(mlx4_unicast_attach); + +int mlx4_unicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, + u8 gid[16], enum mlx4_protocol prot) +{ + if (prot == MLX4_PROT_ETH) + gid[7] |= (MLX4_UC_STEER << 1); + + if (mlx4_is_mfunc(dev)) + return mlx4_QP_ATTACH(dev, qp, gid, 0, 0, prot); + + return mlx4_qp_detach_common(dev, qp, gid, prot, MLX4_UC_STEER); +} +EXPORT_SYMBOL_GPL(mlx4_unicast_detach); + +int mlx4_PROMISC_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + u32 qpn = (u32) vhcr->in_param & 0xffffffff; + u8 port = vhcr->in_param >> 62; + enum mlx4_steer_type steer = vhcr->in_modifier; + + /* Promiscuous unicast is not allowed in mfunc for VFs */ + if ((slave != dev->caps.function) && (steer == MLX4_UC_STEER)) + return 0; + + if (vhcr->op_modifier) + return add_promisc_qp(dev, port, steer, qpn); + else + return remove_promisc_qp(dev, port, steer, qpn); +} + +static int mlx4_PROMISC(struct mlx4_dev *dev, u32 qpn, + enum mlx4_steer_type steer, u8 add, u8 port) +{ + return mlx4_cmd(dev, (u64) qpn | (u64) port << 62, (u32) steer, add, + MLX4_CMD_PROMISC, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); +} + +int mlx4_multicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port) +{ + if (mlx4_is_mfunc(dev)) + return mlx4_PROMISC(dev, qpn, MLX4_MC_STEER, 1, port); + + return add_promisc_qp(dev, port, MLX4_MC_STEER, qpn); +} +EXPORT_SYMBOL_GPL(mlx4_multicast_promisc_add); + +int mlx4_multicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port) +{ + if (mlx4_is_mfunc(dev)) + return mlx4_PROMISC(dev, qpn, MLX4_MC_STEER, 0, port); + + return remove_promisc_qp(dev, port, MLX4_MC_STEER, qpn); +} +EXPORT_SYMBOL_GPL(mlx4_multicast_promisc_remove); + +int mlx4_unicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port) +{ + if (mlx4_is_mfunc(dev)) + return mlx4_PROMISC(dev, qpn, MLX4_UC_STEER, 1, port); + + return add_promisc_qp(dev, port, MLX4_UC_STEER, qpn); +} +EXPORT_SYMBOL_GPL(mlx4_unicast_promisc_add); + +int mlx4_unicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port) +{ + if (mlx4_is_mfunc(dev)) + return mlx4_PROMISC(dev, qpn, MLX4_UC_STEER, 0, port); + + return remove_promisc_qp(dev, port, MLX4_UC_STEER, qpn); +} +EXPORT_SYMBOL_GPL(mlx4_unicast_promisc_remove); + int mlx4_init_mcg_table(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); int err; + /* No need for mcg_table when fw managed the mcg table*/ + if (dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) + return 0; err = mlx4_bitmap_init(&priv->mcg_table.bitmap, dev->caps.num_amgms, dev->caps.num_amgms - 1, 0, 0); if (err) return err; mutex_init(&priv->mcg_table.mutex); return 0; } void mlx4_cleanup_mcg_table(struct mlx4_dev *dev) { - mlx4_bitmap_cleanup(&mlx4_priv(dev)->mcg_table.bitmap); + if (dev->caps.steering_mode != + MLX4_STEERING_MODE_DEVICE_MANAGED) + mlx4_bitmap_cleanup(&mlx4_priv(dev)->mcg_table.bitmap); } Index: stable/9/sys/ofed/drivers/net/mlx4/mlx4.h =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/mlx4.h (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/mlx4.h (revision 273864) @@ -1,427 +1,1320 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2004 Voltaire, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef MLX4_H #define MLX4_H #include #include +#include #include +#include #include - +#include #include #include #include +#include #define DRV_NAME "mlx4_core" #define PFX DRV_NAME ": " -#define DRV_VERSION "1.0-ofed1.5.2" -#define DRV_RELDATE "August 4, 2010" +#define DRV_VERSION "2.1" +#define DRV_RELDATE __DATE__ +#define DRV_STACK_NAME "Linux-MLNX_OFED" +#define DRV_STACK_VERSION "2.1" +#define DRV_NAME_FOR_FW DRV_STACK_NAME","DRV_STACK_VERSION + +#define MLX4_FS_UDP_UC_EN (1 << 1) +#define MLX4_FS_TCP_UC_EN (1 << 2) +#define MLX4_FS_NUM_OF_L2_ADDR 8 +#define MLX4_FS_MGM_LOG_ENTRY_SIZE 7 +#define MLX4_FS_NUM_MCG (1 << 17) + +struct mlx4_set_port_prio2tc_context { + u8 prio2tc[4]; +}; + +struct mlx4_port_scheduler_tc_cfg_be { + __be16 pg; + __be16 bw_precentage; + __be16 max_bw_units; /* 3-100Mbps, 4-1Gbps, other values - reserved */ + __be16 max_bw_value; +}; + +struct mlx4_set_port_scheduler_context { + struct mlx4_port_scheduler_tc_cfg_be tc[MLX4_NUM_TC]; +}; + enum { MLX4_HCR_BASE = 0x80680, MLX4_HCR_SIZE = 0x0001c, - MLX4_CLR_INT_SIZE = 0x00008 + MLX4_CLR_INT_SIZE = 0x00008, + MLX4_SLAVE_COMM_BASE = 0x0, + MLX4_COMM_PAGESIZE = 0x1000, + MLX4_CLOCK_SIZE = 0x00008 }; enum { - MLX4_MGM_ENTRY_SIZE = 0x100, - MLX4_QP_PER_MGM = 4 * (MLX4_MGM_ENTRY_SIZE / 16 - 2), - MLX4_MTT_ENTRY_PER_SEG = 8 + MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE = 10, + MLX4_MIN_MGM_LOG_ENTRY_SIZE = 7, + MLX4_MAX_MGM_LOG_ENTRY_SIZE = 12, + MLX4_MAX_QP_PER_MGM = 4 * ((1 << MLX4_MAX_MGM_LOG_ENTRY_SIZE)/16 - 2), }; enum { MLX4_NUM_PDS = 1 << 15 }; enum { MLX4_CMPT_TYPE_QP = 0, MLX4_CMPT_TYPE_SRQ = 1, MLX4_CMPT_TYPE_CQ = 2, MLX4_CMPT_TYPE_EQ = 3, MLX4_CMPT_NUM_TYPE }; enum { MLX4_CMPT_SHIFT = 24, MLX4_NUM_CMPTS = MLX4_CMPT_NUM_TYPE << MLX4_CMPT_SHIFT }; +enum mlx4_mpt_state { + MLX4_MPT_DISABLED = 0, + MLX4_MPT_EN_HW, + MLX4_MPT_EN_SW +}; + +#define MLX4_COMM_TIME 10000 +enum { + MLX4_COMM_CMD_RESET, + MLX4_COMM_CMD_VHCR0, + MLX4_COMM_CMD_VHCR1, + MLX4_COMM_CMD_VHCR2, + MLX4_COMM_CMD_VHCR_EN, + MLX4_COMM_CMD_VHCR_POST, + MLX4_COMM_CMD_FLR = 254 +}; + +/*The flag indicates that the slave should delay the RESET cmd*/ +#define MLX4_DELAY_RESET_SLAVE 0xbbbbbbb +/*indicates how many retries will be done if we are in the middle of FLR*/ +#define NUM_OF_RESET_RETRIES 10 +#define SLEEP_TIME_IN_RESET (2 * 1000) +enum mlx4_resource { + RES_QP, + RES_CQ, + RES_SRQ, + RES_XRCD, + RES_MPT, + RES_MTT, + RES_MAC, + RES_VLAN, + RES_NPORT_ID, + RES_COUNTER, + RES_FS_RULE, + RES_EQ, + MLX4_NUM_OF_RESOURCE_TYPE +}; + +enum mlx4_alloc_mode { + RES_OP_RESERVE, + RES_OP_RESERVE_AND_MAP, + RES_OP_MAP_ICM, +}; + +enum mlx4_res_tracker_free_type { + RES_TR_FREE_ALL, + RES_TR_FREE_SLAVES_ONLY, + RES_TR_FREE_STRUCTS_ONLY, +}; + +/* + *Virtual HCR structures. + * mlx4_vhcr is the sw representation, in machine endianess + * + * mlx4_vhcr_cmd is the formalized structure, the one that is passed + * to FW to go through communication channel. + * It is big endian, and has the same structure as the physical HCR + * used by command interface + */ +struct mlx4_vhcr { + u64 in_param; + u64 out_param; + u32 in_modifier; + u32 errno; + u16 op; + u16 token; + u8 op_modifier; + u8 e_bit; +}; + +struct mlx4_vhcr_cmd { + __be64 in_param; + __be32 in_modifier; + u32 reserved1; + __be64 out_param; + __be16 token; + u16 reserved; + u8 status; + u8 flags; + __be16 opcode; +} __packed; + +struct mlx4_cmd_info { + u16 opcode; + bool has_inbox; + bool has_outbox; + bool out_is_imm; + bool encode_slave_id; + bool skip_err_print; + int (*verify)(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox); + int (*wrapper)(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +}; + +enum { + MLX4_DEBUG_MASK_CMD_TIME = 0x100, +}; + #ifdef CONFIG_MLX4_DEBUG extern int mlx4_debug_level; #else /* CONFIG_MLX4_DEBUG */ #define mlx4_debug_level (0) #endif /* CONFIG_MLX4_DEBUG */ #define mlx4_dbg(mdev, format, arg...) \ - do { \ - if (mlx4_debug_level) \ - dev_printk(KERN_DEBUG, &mdev->pdev->dev, format, ## arg); \ - } while (0) +do { \ + if (mlx4_debug_level) \ + dev_printk(KERN_DEBUG, &mdev->pdev->dev, format, ##arg); \ +} while (0) #define mlx4_err(mdev, format, arg...) \ - dev_err(&mdev->pdev->dev, format, ## arg) + dev_err(&mdev->pdev->dev, format, ##arg) #define mlx4_info(mdev, format, arg...) \ - dev_info(&mdev->pdev->dev, format, ## arg) + dev_info(&mdev->pdev->dev, format, ##arg) #define mlx4_warn(mdev, format, arg...) \ - dev_warn(&mdev->pdev->dev, format, ## arg) + dev_warn(&mdev->pdev->dev, format, ##arg) +extern int mlx4_log_num_mgm_entry_size; +extern int log_mtts_per_seg; extern int mlx4_blck_lb; +extern int mlx4_set_4k_mtu; +#define MLX4_MAX_NUM_SLAVES (MLX4_MAX_NUM_PF + MLX4_MAX_NUM_VF) +#define ALL_SLAVES 0xff + struct mlx4_bitmap { u32 last; u32 top; u32 max; u32 reserved_top; u32 mask; u32 avail; spinlock_t lock; unsigned long *table; }; struct mlx4_buddy { unsigned long **bits; unsigned int *num_free; - int max_order; + u32 max_order; spinlock_t lock; }; struct mlx4_icm; struct mlx4_icm_table { u64 virt; int num_icm; - int num_obj; + u32 num_obj; int obj_size; int lowmem; int coherent; struct mutex mutex; struct mlx4_icm **icm; }; +#define MLX4_MPT_FLAG_SW_OWNS (0xfUL << 28) +#define MLX4_MPT_FLAG_FREE (0x3UL << 28) +#define MLX4_MPT_FLAG_MIO (1 << 17) +#define MLX4_MPT_FLAG_BIND_ENABLE (1 << 15) +#define MLX4_MPT_FLAG_PHYSICAL (1 << 9) +#define MLX4_MPT_FLAG_REGION (1 << 8) + +#define MLX4_MPT_PD_FLAG_FAST_REG (1 << 27) +#define MLX4_MPT_PD_FLAG_RAE (1 << 28) +#define MLX4_MPT_PD_FLAG_EN_INV (3 << 24) + +#define MLX4_MPT_QP_FLAG_BOUND_QP (1 << 7) + +#define MLX4_MPT_STATUS_SW 0xF0 +#define MLX4_MPT_STATUS_HW 0x00 + +/* + * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits. + */ +struct mlx4_mpt_entry { + __be32 flags; + __be32 qpn; + __be32 key; + __be32 pd_flags; + __be64 start; + __be64 length; + __be32 lkey; + __be32 win_cnt; + u8 reserved1[3]; + u8 mtt_rep; + __be64 mtt_addr; + __be32 mtt_sz; + __be32 entity_size; + __be32 first_byte_offset; +} __packed; + +/* + * Must be packed because start is 64 bits but only aligned to 32 bits. + */ +struct mlx4_eq_context { + __be32 flags; + u16 reserved1[3]; + __be16 page_offset; + u8 log_eq_size; + u8 reserved2[4]; + u8 eq_period; + u8 reserved3; + u8 eq_max_count; + u8 reserved4[3]; + u8 intr; + u8 log_page_size; + u8 reserved5[2]; + u8 mtt_base_addr_h; + __be32 mtt_base_addr_l; + u32 reserved6[2]; + __be32 consumer_index; + __be32 producer_index; + u32 reserved7[4]; +}; + +struct mlx4_cq_context { + __be32 flags; + u16 reserved1[3]; + __be16 page_offset; + __be32 logsize_usrpage; + __be16 cq_period; + __be16 cq_max_count; + u8 reserved2[3]; + u8 comp_eqn; + u8 log_page_size; + u8 reserved3[2]; + u8 mtt_base_addr_h; + __be32 mtt_base_addr_l; + __be32 last_notified_index; + __be32 solicit_producer_index; + __be32 consumer_index; + __be32 producer_index; + u32 reserved4[2]; + __be64 db_rec_addr; +}; + +struct mlx4_srq_context { + __be32 state_logsize_srqn; + u8 logstride; + u8 reserved1; + __be16 xrcd; + __be32 pg_offset_cqn; + u32 reserved2; + u8 log_page_size; + u8 reserved3[2]; + u8 mtt_base_addr_h; + __be32 mtt_base_addr_l; + __be32 pd; + __be16 limit_watermark; + __be16 wqe_cnt; + u16 reserved4; + __be16 wqe_counter; + u32 reserved5; + __be64 db_rec_addr; +}; + struct mlx4_eq { struct mlx4_dev *dev; void __iomem *doorbell; int eqn; u32 cons_index; u16 irq; u16 have_irq; int nent; - int load; struct mlx4_buf_list *page_list; struct mlx4_mtt mtt; }; +struct mlx4_slave_eqe { + u8 type; + u8 port; + u32 param; +}; + +struct mlx4_slave_event_eq_info { + int eqn; + u16 token; +}; + struct mlx4_profile { int num_qp; int rdmarc_per_qp; int num_srq; int num_cq; int num_mcg; int num_mpt; - int num_mtt; + unsigned num_mtt_segs; }; struct mlx4_fw { u64 clr_int_base; u64 catas_offset; + u64 comm_base; + u64 clock_offset; struct mlx4_icm *fw_icm; struct mlx4_icm *aux_icm; u32 catas_size; u16 fw_pages; u8 clr_int_bar; u8 catas_bar; + u8 comm_bar; + u8 clock_bar; }; +struct mlx4_comm { + u32 slave_write; + u32 slave_read; +}; + +enum { + MLX4_MCAST_CONFIG = 0, + MLX4_MCAST_DISABLE = 1, + MLX4_MCAST_ENABLE = 2, +}; + +#define VLAN_FLTR_SIZE 128 + +struct mlx4_vlan_fltr { + __be32 entry[VLAN_FLTR_SIZE]; +}; + +struct mlx4_mcast_entry { + struct list_head list; + u64 addr; +}; + +struct mlx4_promisc_qp { + struct list_head list; + u32 qpn; +}; + +struct mlx4_steer_index { + struct list_head list; + unsigned int index; + struct list_head duplicates; +}; + +#define MLX4_EVENT_TYPES_NUM 64 + +struct mlx4_slave_state { + u8 comm_toggle; + u8 last_cmd; + u8 init_port_mask; + bool active; + bool old_vlan_api; + u8 function; + dma_addr_t vhcr_dma; + u16 mtu[MLX4_MAX_PORTS + 1]; + __be32 ib_cap_mask[MLX4_MAX_PORTS + 1]; + struct mlx4_slave_eqe eq[MLX4_MFUNC_MAX_EQES]; + struct list_head mcast_filters[MLX4_MAX_PORTS + 1]; + struct mlx4_vlan_fltr *vlan_filter[MLX4_MAX_PORTS + 1]; + /* event type to eq number lookup */ + struct mlx4_slave_event_eq_info event_eq[MLX4_EVENT_TYPES_NUM]; + u16 eq_pi; + u16 eq_ci; + spinlock_t lock; + /*initialized via the kzalloc*/ + u8 is_slave_going_down; + u32 cookie; + enum slave_port_state port_state[MLX4_MAX_PORTS + 1]; +}; + +#define MLX4_VGT 4095 +#define NO_INDX (-1) + + +struct mlx4_vport_state { + u64 mac; + u16 default_vlan; + u8 default_qos; + u32 tx_rate; + bool spoofchk; + u32 link_state; +}; + +struct mlx4_vf_admin_state { + struct mlx4_vport_state vport[MLX4_MAX_PORTS + 1]; +}; + +struct mlx4_vport_oper_state { + struct mlx4_vport_state state; + int mac_idx; + int vlan_idx; +}; +struct mlx4_vf_oper_state { + struct mlx4_vport_oper_state vport[MLX4_MAX_PORTS + 1]; +}; + +struct slave_list { + struct mutex mutex; + struct list_head res_list[MLX4_NUM_OF_RESOURCE_TYPE]; +}; + +struct resource_allocator { + spinlock_t alloc_lock; + union { + int res_reserved; + int res_port_rsvd[MLX4_MAX_PORTS]; + }; + union { + int res_free; + int res_port_free[MLX4_MAX_PORTS]; + }; + int *quota; + int *allocated; + int *guaranteed; +}; + +struct mlx4_resource_tracker { + spinlock_t lock; + /* tree for each resources */ + struct rb_root res_tree[MLX4_NUM_OF_RESOURCE_TYPE]; + /* num_of_slave's lists, one per slave */ + struct slave_list *slave_list; + struct resource_allocator res_alloc[MLX4_NUM_OF_RESOURCE_TYPE]; +}; + +#define SLAVE_EVENT_EQ_SIZE 128 +struct mlx4_slave_event_eq { + u32 eqn; + u32 cons; + u32 prod; + spinlock_t event_lock; + struct mlx4_eqe event_eqe[SLAVE_EVENT_EQ_SIZE]; +}; + +struct mlx4_master_qp0_state { + int proxy_qp0_active; + int qp0_active; + int port_active; +}; + +struct mlx4_mfunc_master_ctx { + struct mlx4_slave_state *slave_state; + struct mlx4_vf_admin_state *vf_admin; + struct mlx4_vf_oper_state *vf_oper; + struct mlx4_master_qp0_state qp0_state[MLX4_MAX_PORTS + 1]; + int init_port_ref[MLX4_MAX_PORTS + 1]; + u16 max_mtu[MLX4_MAX_PORTS + 1]; + int disable_mcast_ref[MLX4_MAX_PORTS + 1]; + struct mlx4_resource_tracker res_tracker; + struct workqueue_struct *comm_wq; + struct work_struct comm_work; + struct work_struct arm_comm_work; + struct work_struct slave_event_work; + struct work_struct slave_flr_event_work; + spinlock_t slave_state_lock; + __be32 comm_arm_bit_vector[4]; + struct mlx4_eqe cmd_eqe; + struct mlx4_slave_event_eq slave_eq; + struct mutex gen_eqe_mutex[MLX4_MFUNC_MAX]; +}; + +struct mlx4_mfunc { + struct mlx4_comm __iomem *comm; + struct mlx4_vhcr_cmd *vhcr; + dma_addr_t vhcr_dma; + + struct mlx4_mfunc_master_ctx master; +}; + +#define MGM_QPN_MASK 0x00FFFFFF +#define MGM_BLCK_LB_BIT 30 + +struct mlx4_mgm { + __be32 next_gid_index; + __be32 members_count; + u32 reserved[2]; + u8 gid[16]; + __be32 qp[MLX4_MAX_QP_PER_MGM]; +}; + struct mlx4_cmd { struct pci_pool *pool; void __iomem *hcr; struct mutex hcr_mutex; + struct mutex slave_cmd_mutex; struct semaphore poll_sem; struct semaphore event_sem; int max_cmds; spinlock_t context_lock; int free_head; struct mlx4_cmd_context *context; u16 token_mask; u8 use_events; u8 toggle; + u8 comm_toggle; }; +enum { + MLX4_VF_IMMED_VLAN_FLAG_VLAN = 1 << 0, + MLX4_VF_IMMED_VLAN_FLAG_QOS = 1 << 1, +}; +struct mlx4_vf_immed_vlan_work { + struct work_struct work; + struct mlx4_priv *priv; + int flags; + int slave; + int vlan_ix; + int orig_vlan_ix; + u8 port; + u8 qos; + u16 vlan_id; + u16 orig_vlan_id; +}; + + struct mlx4_uar_table { struct mlx4_bitmap bitmap; }; struct mlx4_mr_table { struct mlx4_bitmap mpt_bitmap; struct mlx4_buddy mtt_buddy; u64 mtt_base; u64 mpt_base; struct mlx4_icm_table mtt_table; struct mlx4_icm_table dmpt_table; }; struct mlx4_cq_table { struct mlx4_bitmap bitmap; spinlock_t lock; + rwlock_t cq_table_lock; struct radix_tree_root tree; struct mlx4_icm_table table; struct mlx4_icm_table cmpt_table; }; struct mlx4_eq_table { struct mlx4_bitmap bitmap; char *irq_names; void __iomem *clr_int; void __iomem **uar_map; u32 clr_mask; struct mlx4_eq *eq; struct mlx4_icm_table table; struct mlx4_icm_table cmpt_table; int have_irq; u8 inta_pin; }; struct mlx4_srq_table { struct mlx4_bitmap bitmap; spinlock_t lock; + struct radix_tree_root tree; struct mlx4_icm_table table; struct mlx4_icm_table cmpt_table; }; struct mlx4_qp_table { struct mlx4_bitmap bitmap; u32 rdmarc_base; int rdmarc_shift; spinlock_t lock; struct mlx4_icm_table qp_table; struct mlx4_icm_table auxc_table; struct mlx4_icm_table altc_table; struct mlx4_icm_table rdmarc_table; struct mlx4_icm_table cmpt_table; }; struct mlx4_mcg_table { struct mutex mutex; struct mlx4_bitmap bitmap; struct mlx4_icm_table table; }; struct mlx4_catas_err { u32 __iomem *map; struct timer_list timer; struct list_head list; }; #define MLX4_MAX_MAC_NUM 128 #define MLX4_MAC_TABLE_SIZE (MLX4_MAX_MAC_NUM << 3) struct mlx4_mac_table { __be64 entries[MLX4_MAX_MAC_NUM]; int refs[MLX4_MAX_MAC_NUM]; struct mutex mutex; int total; int max; }; #define MLX4_MAX_VLAN_NUM 128 #define MLX4_VLAN_TABLE_SIZE (MLX4_MAX_VLAN_NUM << 2) struct mlx4_vlan_table { __be32 entries[MLX4_MAX_VLAN_NUM]; int refs[MLX4_MAX_VLAN_NUM]; struct mutex mutex; int total; int max; }; +#define SET_PORT_GEN_ALL_VALID 0x7 +#define SET_PORT_PROMISC_SHIFT 31 +#define SET_PORT_MC_PROMISC_SHIFT 30 + +enum { + MCAST_DIRECT_ONLY = 0, + MCAST_DIRECT = 1, + MCAST_DEFAULT = 2 +}; + + +struct mlx4_set_port_general_context { + u8 reserved[3]; + u8 flags; + u16 reserved2; + __be16 mtu; + u8 pptx; + u8 pfctx; + u16 reserved3; + u8 pprx; + u8 pfcrx; + u16 reserved4; +}; + +struct mlx4_set_port_rqp_calc_context { + __be32 base_qpn; + u8 rererved; + u8 n_mac; + u8 n_vlan; + u8 n_prio; + u8 reserved2[3]; + u8 mac_miss; + u8 intra_no_vlan; + u8 no_vlan; + u8 intra_vlan_miss; + u8 vlan_miss; + u8 reserved3[3]; + u8 no_vlan_prio; + __be32 promisc; + __be32 mcast; +}; + struct mlx4_port_info { struct mlx4_dev *dev; int port; char dev_name[16]; struct device_attribute port_attr; enum mlx4_port_type tmp_type; + char dev_mtu_name[16]; + struct device_attribute port_mtu_attr; struct mlx4_mac_table mac_table; struct mlx4_vlan_table vlan_table; + int base_qpn; }; struct mlx4_sense { struct mlx4_dev *dev; u8 do_sense_port[MLX4_MAX_PORTS + 1]; u8 sense_allowed[MLX4_MAX_PORTS + 1]; struct delayed_work sense_poll; - struct workqueue_struct *sense_wq; - u32 resched; }; -extern struct mutex drv_mutex; +struct mlx4_msix_ctl { + u64 pool_bm; + struct mutex pool_lock; +}; +struct mlx4_steer { + struct list_head promisc_qps[MLX4_NUM_STEERS]; + struct list_head steer_entries[MLX4_NUM_STEERS]; +}; + +enum { + MLX4_PCI_DEV_IS_VF = 1 << 0, + MLX4_PCI_DEV_FORCE_SENSE_PORT = 1 << 1, +}; + +struct mlx4_roce_gid_entry { + u8 raw[16]; +}; + +struct counter_index { + struct list_head list; + u32 index; +}; + +struct mlx4_counters { + struct mlx4_bitmap bitmap; + struct list_head global_port_list[MLX4_MAX_PORTS]; + struct list_head vf_list[MLX4_MAX_NUM_VF][MLX4_MAX_PORTS]; + struct mutex mutex; +}; + +enum { + MLX4_NO_RR = 0, + MLX4_USE_RR = 1, +}; + struct mlx4_priv { struct mlx4_dev dev; struct list_head dev_list; struct list_head ctx_list; spinlock_t ctx_lock; + int pci_dev_data; + struct list_head pgdir_list; struct mutex pgdir_mutex; struct mlx4_fw fw; struct mlx4_cmd cmd; + struct mlx4_mfunc mfunc; struct mlx4_bitmap pd_bitmap; struct mlx4_bitmap xrcd_bitmap; struct mlx4_uar_table uar_table; struct mlx4_mr_table mr_table; struct mlx4_cq_table cq_table; struct mlx4_eq_table eq_table; struct mlx4_srq_table srq_table; struct mlx4_qp_table qp_table; struct mlx4_mcg_table mcg_table; - struct mlx4_bitmap counters_bitmap; - struct list_head bf_list; - struct mutex bf_mutex; + struct mlx4_counters counters_table; struct mlx4_catas_err catas_err; void __iomem *clr_base; struct mlx4_uar driver_uar; void __iomem *kar; struct mlx4_port_info port[MLX4_MAX_PORTS + 1]; - struct device_attribute trigger_attr; - int trig; - int changed_ports; struct mlx4_sense sense; struct mutex port_mutex; - int iboe_counter_index[MLX4_MAX_PORTS]; - struct io_mapping *bf_mapping; + struct mlx4_msix_ctl msix_ctl; + struct mlx4_steer *steer; + struct list_head bf_list; + struct mutex bf_mutex; + struct io_mapping *bf_mapping; + void __iomem *clock_mapping; + int reserved_mtts; + int fs_hash_mode; + u8 virt2phys_pkey[MLX4_MFUNC_MAX][MLX4_MAX_PORTS][MLX4_MAX_PORT_PKEYS]; + __be64 slave_node_guids[MLX4_MFUNC_MAX]; + struct mlx4_roce_gid_entry roce_gids[MLX4_MAX_PORTS][128]; + atomic_t opreq_count; + struct work_struct opreq_task; }; static inline struct mlx4_priv *mlx4_priv(struct mlx4_dev *dev) { return container_of(dev, struct mlx4_priv, dev); } #define MLX4_SENSE_RANGE (HZ * 3) extern struct workqueue_struct *mlx4_wq; u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap); -void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj); -u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align); -void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt); +void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr); +u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, + int align, u32 skip_mask); +void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, + int use_rr); u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap); int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, u32 reserved_bot, u32 resetrved_top); void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap); int mlx4_reset(struct mlx4_dev *dev); int mlx4_alloc_eq_table(struct mlx4_dev *dev); void mlx4_free_eq_table(struct mlx4_dev *dev); int mlx4_init_pd_table(struct mlx4_dev *dev); int mlx4_init_xrcd_table(struct mlx4_dev *dev); int mlx4_init_uar_table(struct mlx4_dev *dev); int mlx4_init_mr_table(struct mlx4_dev *dev); int mlx4_init_eq_table(struct mlx4_dev *dev); int mlx4_init_cq_table(struct mlx4_dev *dev); int mlx4_init_qp_table(struct mlx4_dev *dev); int mlx4_init_srq_table(struct mlx4_dev *dev); int mlx4_init_mcg_table(struct mlx4_dev *dev); void mlx4_cleanup_pd_table(struct mlx4_dev *dev); +void mlx4_cleanup_xrcd_table(struct mlx4_dev *dev); void mlx4_cleanup_uar_table(struct mlx4_dev *dev); void mlx4_cleanup_mr_table(struct mlx4_dev *dev); void mlx4_cleanup_eq_table(struct mlx4_dev *dev); void mlx4_cleanup_cq_table(struct mlx4_dev *dev); void mlx4_cleanup_qp_table(struct mlx4_dev *dev); void mlx4_cleanup_srq_table(struct mlx4_dev *dev); void mlx4_cleanup_mcg_table(struct mlx4_dev *dev); -void mlx4_cleanup_xrcd_table(struct mlx4_dev *dev); +int __mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn); +void __mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn); +int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn); +void __mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn); +int __mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn); +void __mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn); +int __mlx4_mpt_reserve(struct mlx4_dev *dev); +void __mlx4_mpt_release(struct mlx4_dev *dev, u32 index); +int __mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index); +void __mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index); +u32 __mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order); +void __mlx4_free_mtt_range(struct mlx4_dev *dev, u32 first_seg, int order); +int mlx4_WRITE_MTT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SYNC_TPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SW2HW_MPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_HW2SW_MPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_MPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SW2HW_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_DMA_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int __mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, + int *base, u8 flags); +void __mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt); +int __mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac); +void __mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, u64 mac); +int __mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + int start_index, int npages, u64 *page_list); +int __mlx4_counter_alloc(struct mlx4_dev *dev, int slave, int port, u32 *idx); +void __mlx4_counter_free(struct mlx4_dev *dev, int slave, int port, u32 idx); + +int __mlx4_slave_counters_free(struct mlx4_dev *dev, int slave); +int __mlx4_clear_if_stat(struct mlx4_dev *dev, + u8 counter_index); +u8 mlx4_get_default_counter_index(struct mlx4_dev *dev, int slave, int port); + +int __mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn); +void __mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn); + void mlx4_start_catas_poll(struct mlx4_dev *dev); void mlx4_stop_catas_poll(struct mlx4_dev *dev); void mlx4_catas_init(void); int mlx4_restart_one(struct pci_dev *pdev); int mlx4_register_device(struct mlx4_dev *dev); void mlx4_unregister_device(struct mlx4_dev *dev); -void mlx4_dispatch_event(struct mlx4_dev *dev, enum mlx4_dev_event type, int port); -void *mlx4_find_get_prot_dev(struct mlx4_dev *dev, enum mlx4_prot proto, int port); +void mlx4_dispatch_event(struct mlx4_dev *dev, enum mlx4_dev_event type, + unsigned long param); struct mlx4_dev_cap; struct mlx4_init_hca_param; u64 mlx4_make_profile(struct mlx4_dev *dev, struct mlx4_profile *request, struct mlx4_dev_cap *dev_cap, struct mlx4_init_hca_param *init_hca); +void mlx4_master_comm_channel(struct work_struct *work); +void mlx4_master_arm_comm_channel(struct work_struct *work); +void mlx4_gen_slave_eqe(struct work_struct *work); +void mlx4_master_handle_slave_flr(struct work_struct *work); +int mlx4_ALLOC_RES_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_FREE_RES_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_MAP_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_COMM_INT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_HW2SW_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SW2HW_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_HW2SW_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_MODIFY_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SW2HW_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_HW2SW_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_ARM_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_GEN_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_RST2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_INIT2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_INIT2RTR_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_RTR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_RTS2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SQERR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_2ERR_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_RTS2SQD_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SQD2SQD_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SQD2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_2RST_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); + +int mlx4_GEN_EQE(struct mlx4_dev *dev, int slave, struct mlx4_eqe *eqe); + int mlx4_cmd_init(struct mlx4_dev *dev); void mlx4_cmd_cleanup(struct mlx4_dev *dev); +int mlx4_multi_func_init(struct mlx4_dev *dev); +void mlx4_multi_func_cleanup(struct mlx4_dev *dev); void mlx4_cmd_event(struct mlx4_dev *dev, u16 token, u8 status, u64 out_param); int mlx4_cmd_use_events(struct mlx4_dev *dev); void mlx4_cmd_use_polling(struct mlx4_dev *dev); +int mlx4_comm_cmd(struct mlx4_dev *dev, u8 cmd, u16 param, + unsigned long timeout); + void mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn); void mlx4_cq_event(struct mlx4_dev *dev, u32 cqn, int event_type); void mlx4_qp_event(struct mlx4_dev *dev, u32 qpn, int event_type); void mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type); void mlx4_handle_catas_err(struct mlx4_dev *dev); +int mlx4_SENSE_PORT(struct mlx4_dev *dev, int port, + enum mlx4_port_type *type); void mlx4_do_sense_ports(struct mlx4_dev *dev, enum mlx4_port_type *stype, enum mlx4_port_type *defaults); void mlx4_start_sense(struct mlx4_dev *dev); void mlx4_stop_sense(struct mlx4_dev *dev); -int mlx4_sense_init(struct mlx4_dev *dev); -void mlx4_sense_cleanup(struct mlx4_dev *dev); +void mlx4_sense_init(struct mlx4_dev *dev); int mlx4_check_port_params(struct mlx4_dev *dev, enum mlx4_port_type *port_type); int mlx4_change_port_types(struct mlx4_dev *dev, enum mlx4_port_type *port_types); void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table); void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table); +void __mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, u16 vlan); +int __mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index); -int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port); +int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port, int pkey_tbl_sz); +/* resource tracker functions*/ +int mlx4_get_slave_from_resource_id(struct mlx4_dev *dev, + enum mlx4_resource resource_type, + u64 resource_id, int *slave); +void mlx4_delete_all_resources_for_slave(struct mlx4_dev *dev, int slave_id); +int mlx4_init_resource_tracker(struct mlx4_dev *dev); + +void mlx4_free_resource_tracker(struct mlx4_dev *dev, + enum mlx4_res_tracker_free_type type); + +int mlx4_QUERY_FW_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SET_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_INIT_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_CLOSE_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_DEV_CAP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); int mlx4_get_port_ib_caps(struct mlx4_dev *dev, u8 port, __be32 *caps); + +int mlx4_get_slave_pkey_gid_tbl_len(struct mlx4_dev *dev, u8 port, + int *gid_tbl_len, int *pkey_tbl_len); + +int mlx4_QP_ATTACH_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); + +int mlx4_PROMISC_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_qp_detach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + enum mlx4_protocol prot, enum mlx4_steer_type steer); +int mlx4_qp_attach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + int block_mcast_loopback, enum mlx4_protocol prot, + enum mlx4_steer_type steer); +int mlx4_trans_to_dmfs_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, + u8 gid[16], u8 port, + int block_mcast_loopback, + enum mlx4_protocol prot, u64 *reg_id); +int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode); +int mlx4_SET_MCAST_FLTR_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_SET_VLAN_FLTR_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_common_set_vlan_fltr(struct mlx4_dev *dev, int function, + int port, void *buf); +int mlx4_DUMP_ETH_STATS_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_PKEY_TABLE_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QUERY_IF_STAT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QP_FLOW_STEERING_ATTACH_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_QP_FLOW_STEERING_DETACH_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); +int mlx4_MOD_STAT_CFG_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd); + +int mlx4_get_mgm_entry_size(struct mlx4_dev *dev); +int mlx4_get_qp_per_mgm(struct mlx4_dev *dev); + +static inline void set_param_l(u64 *arg, u32 val) +{ + *arg = (*arg & 0xffffffff00000000ULL) | (u64) val; +} + +static inline void set_param_h(u64 *arg, u32 val) +{ + *arg = (*arg & 0xffffffff) | ((u64) val << 32); +} + +static inline u32 get_param_l(u64 *arg) +{ + return (u32) (*arg & 0xffffffff); +} + +static inline u32 get_param_h(u64 *arg) +{ + return (u32)(*arg >> 32); +} + +static inline spinlock_t *mlx4_tlock(struct mlx4_dev *dev) +{ + return &mlx4_priv(dev)->mfunc.master.res_tracker.lock; +} + +#define NOT_MASKED_PD_BITS 17 + +void sys_tune_init(void); +void sys_tune_fini(void); + +void mlx4_init_quotas(struct mlx4_dev *dev); + +int mlx4_get_slave_num_gids(struct mlx4_dev *dev, int slave); +int mlx4_get_base_gid_ix(struct mlx4_dev *dev, int slave); +void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work); #endif /* MLX4_H */ Index: stable/9/sys/ofed/drivers/net/mlx4/mlx4_en.h =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/mlx4_en.h (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/mlx4_en.h (revision 273864) @@ -1,656 +1,950 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ #ifndef _MLX4_EN_H_ #define _MLX4_EN_H_ -#include - -#include +#include #include #include #include +#include #include +#include +#include +#ifdef CONFIG_MLX4_EN_DCB +#include +#endif #include #include #include #include #include #include -#include #include #include "en_port.h" +#include "mlx4_stats.h" #define DRV_NAME "mlx4_en" -#define DRV_VERSION "1.5.2" -#define DRV_RELDATE "July 2010" +#define DRV_VERSION "2.1" +#define DRV_RELDATE __DATE__ -/* XXX */ -#define NETIF_MSG_LINK 0x1 -#define NETIF_MSG_IFDOWN 0x2 -#define NETIF_MSG_HW 0x4 -#define NETIF_MSG_DRV 0x8 -#define NETIF_MSG_INTR 0x10 -#define NETIF_MSG_RX_ERR 0x20 - #define MLX4_EN_MSG_LEVEL (NETIF_MSG_LINK | NETIF_MSG_IFDOWN) -#define en_print(level, priv, format, arg...) \ - { \ - if ((priv)->registered) \ - printk(level "%s: %s: " format, DRV_NAME, \ - (priv->dev)->if_xname, ## arg); \ - else \ - printk(level "%s: %s: Port %d: " format, \ - DRV_NAME, dev_name(&priv->mdev->pdev->dev), \ - (priv)->port, ## arg); \ - } - -#define en_dbg(mlevel, priv, format, arg...) \ - if (NETIF_MSG_##mlevel & priv->msg_enable) \ - en_print(KERN_DEBUG, priv, format, ## arg) -#define en_warn(priv, format, arg...) \ - en_print(KERN_WARNING, priv, format, ## arg) -#define en_err(priv, format, arg...) \ - en_print(KERN_ERR, priv, format, ## arg) -#define en_info(priv, format, arg...) \ - en_print(KERN_INFO, priv, format, ## arg) - -#define mlx4_err(mdev, format, arg...) \ - printk(KERN_ERR "%s %s: " format , DRV_NAME ,\ - dev_name(&mdev->pdev->dev) , ## arg) -#define mlx4_info(mdev, format, arg...) \ - printk(KERN_INFO "%s %s: " format , DRV_NAME ,\ - dev_name(&mdev->pdev->dev) , ## arg) -#define mlx4_warn(mdev, format, arg...) \ - printk(KERN_WARNING "%s %s: " format , DRV_NAME ,\ - dev_name(&mdev->pdev->dev) , ## arg) - /* * Device constants */ #define MLX4_EN_PAGE_SHIFT 12 #define MLX4_EN_PAGE_SIZE (1 << MLX4_EN_PAGE_SHIFT) -#define MAX_TX_RINGS (MLX4_EN_NUM_HASH_RINGS + 1 + MLX4_EN_NUM_PPP_RINGS) -#define MAX_RX_RINGS 16 +#define DEF_RX_RINGS 16 +#define MAX_RX_RINGS 128 +#define MIN_RX_RINGS 4 #define TXBB_SIZE 64 #define HEADROOM (2048 / TXBB_SIZE + 1) #define STAMP_STRIDE 64 #define STAMP_DWORDS (STAMP_STRIDE / 4) #define STAMP_SHIFT 31 #define STAMP_VAL 0x7fffffff #define STATS_DELAY (HZ / 4) +#define SERVICE_TASK_DELAY (HZ / 4) +#define MAX_NUM_OF_FS_RULES 256 +#define MLX4_EN_FILTER_HASH_SHIFT 4 +#define MLX4_EN_FILTER_EXPIRY_QUOTA 60 + +#ifdef CONFIG_NET_RX_BUSY_POLL +#define LL_EXTENDED_STATS +#endif + +/* vlan valid range */ +#define VLAN_MIN_VALUE 1 +#define VLAN_MAX_VALUE 4094 + /* Typical TSO descriptor with 16 gather entries is 352 bytes... */ #define MAX_DESC_SIZE 512 #define MAX_DESC_TXBBS (MAX_DESC_SIZE / TXBB_SIZE) /* * OS related constants and tunables */ #define MLX4_EN_WATCHDOG_TIMEOUT (15 * HZ) -#define MLX4_EN_MAX_LRO_DESCRIPTORS 32 -#define MLX4_EN_NUM_IPFRAG_SESSIONS 16 +#define MLX4_EN_ALLOC_SIZE PAGE_ALIGN(PAGE_SIZE) +#define MLX4_EN_ALLOC_ORDER get_order(MLX4_EN_ALLOC_SIZE) +enum mlx4_en_alloc_type { + MLX4_EN_ALLOC_NEW = 0, + MLX4_EN_ALLOC_REPLACEMENT = 1, +}; + /* Receive fragment sizes; we use at most 3 fragments (for 9600 byte MTU * and 4K allocations) */ #if MJUMPAGESIZE == 4096 enum { FRAG_SZ0 = MCLBYTES, FRAG_SZ1 = MJUMPAGESIZE, FRAG_SZ2 = MJUMPAGESIZE, }; #define MLX4_EN_MAX_RX_FRAGS 3 #elif MJUMPAGESIZE == 8192 enum { FRAG_SZ0 = MCLBYTES, FRAG_SZ1 = MJUMPAGESIZE, }; #define MLX4_EN_MAX_RX_FRAGS 2 #elif MJUMPAGESIZE == 8192 #else #error "Unknown PAGE_SIZE" #endif /* Maximum ring sizes */ +#define MLX4_EN_DEF_TX_QUEUE_SIZE 4096 + +/* Minimum packet number till arming the CQ */ +#define MLX4_EN_MIN_RX_ARM 2048 +#define MLX4_EN_MIN_TX_ARM 2048 + +/* Maximum ring sizes */ #define MLX4_EN_MAX_TX_SIZE 8192 #define MLX4_EN_MAX_RX_SIZE 8192 -#define MLX4_EN_MIN_RX_SIZE (128) +/* Minimum ring sizes */ +#define MLX4_EN_MIN_RX_SIZE (4096 / TXBB_SIZE) #define MLX4_EN_MIN_TX_SIZE (4096 / TXBB_SIZE) #define MLX4_EN_SMALL_PKT_SIZE 64 -#define MLX4_EN_TX_HASH_SIZE 256 -#define MLX4_EN_TX_HASH_MASK (MLX4_EN_TX_HASH_SIZE - 1) -#define MLX4_EN_NUM_HASH_RINGS 4 -#define MLX4_EN_NUM_PPP_RINGS 8 -#define MLX4_EN_DEF_TX_RING_SIZE 512 -#define MLX4_EN_DEF_TX_QUEUE_SIZE 4096 + +#define MLX4_EN_MAX_TX_RING_P_UP 32 +#define MLX4_EN_NUM_UP 1 + +#define MAX_TX_RINGS (MLX4_EN_MAX_TX_RING_P_UP * \ + (MLX4_EN_NUM_UP + 1)) + +#define MLX4_EN_DEF_TX_RING_SIZE 1024 #define MLX4_EN_DEF_RX_RING_SIZE 1024 -#define MLX4_EN_MAX_RX_POLL 1024 /* Target number of bytes to coalesce with interrupt moderation */ #define MLX4_EN_RX_COAL_TARGET 0x20000 #define MLX4_EN_RX_COAL_TIME 0x10 -#define MLX4_EN_TX_COAL_PKTS 5 -#define MLX4_EN_TX_COAL_TIME 0x80 +#define MLX4_EN_TX_COAL_PKTS 64 +#define MLX4_EN_TX_COAL_TIME 64 #define MLX4_EN_RX_RATE_LOW 400000 #define MLX4_EN_RX_COAL_TIME_LOW 0 #define MLX4_EN_RX_RATE_HIGH 450000 #define MLX4_EN_RX_COAL_TIME_HIGH 128 #define MLX4_EN_RX_SIZE_THRESH 1024 #define MLX4_EN_RX_RATE_THRESH (1000000 / MLX4_EN_RX_COAL_TIME_HIGH) #define MLX4_EN_SAMPLE_INTERVAL 0 #define MLX4_EN_AVG_PKT_SMALL 256 #define MLX4_EN_AUTO_CONF 0xffff #define MLX4_EN_DEF_RX_PAUSE 1 #define MLX4_EN_DEF_TX_PAUSE 1 -/* Interval between sucessive polls in the Tx routine when polling is used +/* Interval between successive polls in the Tx routine when polling is used instead of interrupts (in per-core Tx rings) - should be power of 2 */ #define MLX4_EN_TX_POLL_MODER 16 #define MLX4_EN_TX_POLL_TIMEOUT (HZ / 4) -#define ETH_LLC_SNAP_SIZE 8 - -#define SMALL_PACKET_SIZE (MHLEN) +#define MLX4_EN_64_ALIGN (64 - NET_SKB_PAD) +#define SMALL_PACKET_SIZE (256 - NET_IP_ALIGN) #define HEADER_COPY_SIZE (128) #define MLX4_LOOPBACK_TEST_PAYLOAD (HEADER_COPY_SIZE - ETHER_HDR_LEN) #define MLX4_EN_MIN_MTU 46 #define ETH_BCAST 0xffffffffffffULL #define MLX4_EN_LOOPBACK_RETRIES 5 #define MLX4_EN_LOOPBACK_TIMEOUT 100 #ifdef MLX4_EN_PERF_STAT /* Number of samples to 'average' */ #define AVG_SIZE 128 #define AVG_FACTOR 1024 -#define NUM_PERF_STATS NUM_PERF_COUNTERS #define INC_PERF_COUNTER(cnt) (++(cnt)) #define ADD_PERF_COUNTER(cnt, add) ((cnt) += (add)) #define AVG_PERF_COUNTER(cnt, sample) \ ((cnt) = ((cnt) * (AVG_SIZE - 1) + (sample) * AVG_FACTOR) / AVG_SIZE) #define GET_PERF_COUNTER(cnt) (cnt) #define GET_AVG_PERF_COUNTER(cnt) ((cnt) / AVG_FACTOR) #else -#define NUM_PERF_STATS 0 #define INC_PERF_COUNTER(cnt) do {} while (0) #define ADD_PERF_COUNTER(cnt, add) do {} while (0) #define AVG_PERF_COUNTER(cnt, sample) do {} while (0) #define GET_PERF_COUNTER(cnt) (0) #define GET_AVG_PERF_COUNTER(cnt) (0) #endif /* MLX4_EN_PERF_STAT */ /* * Configurables */ enum cq_type { RX = 0, TX = 1, }; /* * Useful macros */ #define ROUNDUP_LOG2(x) ilog2(roundup_pow_of_two(x)) #define XNOR(x, y) (!(x) == !(y)) #define ILLEGAL_MAC(addr) (addr == 0xffffffffffffULL || addr == 0x0) - struct mlx4_en_tx_info { - struct mbuf *mb; - u32 nr_txbb; - u8 nr_segs; - u8 data_offset; - u8 inl; + struct mbuf *mb; + u32 nr_txbb; + u32 nr_bytes; + u8 linear; + u8 nr_segs; + u8 data_offset; + u8 inl; +#if 0 + u8 ts_requested; +#endif }; #define MLX4_EN_BIT_DESC_OWN 0x80000000 #define CTRL_SIZE sizeof(struct mlx4_wqe_ctrl_seg) #define MLX4_EN_MEMTYPE_PAD 0x100 #define DS_SIZE sizeof(struct mlx4_wqe_data_seg) struct mlx4_en_tx_desc { struct mlx4_wqe_ctrl_seg ctrl; union { struct mlx4_wqe_data_seg data; /* at least one data segment */ struct mlx4_wqe_lso_seg lso; struct mlx4_wqe_inline_seg inl; }; }; #define MLX4_EN_USE_SRQ 0x01000000 +#define MLX4_EN_TX_BUDGET 64*4 //Compensate for no NAPI in freeBSD - might need some fine tunning in the future. +#define MLX4_EN_RX_BUDGET 64 + +#define MLX4_EN_CX3_LOW_ID 0x1000 +#define MLX4_EN_CX3_HIGH_ID 0x1005 + struct mlx4_en_tx_ring { - spinlock_t tx_lock; + spinlock_t tx_lock; struct mlx4_hwq_resources wqres; u32 size ; /* number of TXBBs */ u32 size_mask; u16 stride; u16 cqn; /* index of port CQ associated with this ring */ u32 prod; u32 cons; u32 buf_size; u32 doorbell_qpn; void *buf; u16 poll_cnt; int blocked; - struct buf_ring *br; struct mlx4_en_tx_info *tx_info; u8 *bounce_buf; + u8 queue_index; + cpuset_t affinity_mask; + struct buf_ring *br; u32 last_nr_txbb; struct mlx4_qp qp; struct mlx4_qp_context context; int qpn; enum mlx4_qp_state qp_state; struct mlx4_srq dummy; unsigned long bytes; unsigned long packets; - unsigned long errors; - spinlock_t comp_lock; + unsigned long tx_csum; + unsigned long queue_stopped; + unsigned long wake_queue; struct mlx4_bf bf; bool bf_enabled; + struct netdev_queue *tx_queue; + int hwtstamp_tx_type; + spinlock_t comp_lock; + int full_size; + int inline_thold; u64 watchdog_time; }; -struct mlx4_en_ipfrag { - struct mbuf *fragments; - struct mbuf *last; - __be32 saddr; - __be32 daddr; - __be16 id; - u8 protocol; - int total_len; - u16 offset; -}; - struct mlx4_en_rx_desc { /* actual number of entries depends on rx ring stride */ struct mlx4_wqe_data_seg data[0]; }; +struct mlx4_en_rx_buf { + dma_addr_t dma; + struct page *page; + unsigned int page_offset; +}; + struct mlx4_en_rx_ring { struct mlx4_hwq_resources wqres; u32 size ; /* number of Rx descs*/ u32 actual_size; u32 size_mask; u16 stride; u16 log_stride; u16 cqn; /* index of port CQ associated with this ring */ u32 prod; u32 cons; u32 buf_size; + u8 fcs_del; + u16 rx_alloc_order; + u32 rx_alloc_size; + u32 rx_buf_size; + u32 rx_mb_size; + int qpn; void *buf; void *rx_info; + unsigned long errors; unsigned long bytes; unsigned long packets; - unsigned long errors; +#ifdef LL_EXTENDED_STATS + unsigned long yields; + unsigned long misses; + unsigned long cleaned; +#endif + unsigned long csum_ok; + unsigned long csum_none; + int hwtstamp_rx_filter; + int numa_node; struct lro_ctrl lro; - struct mlx4_en_ipfrag ipfrag[MLX4_EN_NUM_IPFRAG_SESSIONS]; }; - static inline int mlx4_en_can_lro(__be16 status) { - return (status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 | - MLX4_CQE_STATUS_IPV4F | - MLX4_CQE_STATUS_IPV6 | - MLX4_CQE_STATUS_IPV4OPT | - MLX4_CQE_STATUS_TCP | - MLX4_CQE_STATUS_UDP | - MLX4_CQE_STATUS_IPOK)) == - cpu_to_be16(MLX4_CQE_STATUS_IPV4 | - MLX4_CQE_STATUS_IPOK | - MLX4_CQE_STATUS_TCP); + static __be16 status_all; + static __be16 status_ipv4_ipok_tcp; + static __be16 status_ipv6_ipok_tcp; + + status_all = cpu_to_be16( + MLX4_CQE_STATUS_IPV4 | + MLX4_CQE_STATUS_IPV4F | + MLX4_CQE_STATUS_IPV6 | + MLX4_CQE_STATUS_IPV4OPT | + MLX4_CQE_STATUS_TCP | + MLX4_CQE_STATUS_UDP | + MLX4_CQE_STATUS_IPOK); + status_ipv4_ipok_tcp = cpu_to_be16( + MLX4_CQE_STATUS_IPV4 | + MLX4_CQE_STATUS_IPOK | + MLX4_CQE_STATUS_TCP); + status_ipv6_ipok_tcp = cpu_to_be16( + MLX4_CQE_STATUS_IPV6 | + MLX4_CQE_STATUS_IPOK | + MLX4_CQE_STATUS_TCP); + + status &= status_all; + return (status == status_ipv4_ipok_tcp || + status == status_ipv6_ipok_tcp); } + struct mlx4_en_cq { struct mlx4_cq mcq; struct mlx4_hwq_resources wqres; int ring; spinlock_t lock; struct net_device *dev; - /* Per-core Tx cq processing support */ - struct timer_list timer; + /* Per-core Tx cq processing support */ + struct timer_list timer; int size; int buf_size; unsigned vector; enum cq_type is_tx; u16 moder_time; u16 moder_cnt; struct mlx4_cqe *buf; struct task cq_task; struct taskqueue *tq; #define MLX4_EN_OPCODE_ERROR 0x1e u32 tot_rx; + u32 tot_tx; + +#ifdef CONFIG_NET_RX_BUSY_POLL + unsigned int state; +#define MLX4_EN_CQ_STATEIDLE 0 +#define MLX4_EN_CQ_STATENAPI 1 /* NAPI owns this CQ */ +#define MLX4_EN_CQ_STATEPOLL 2 /* poll owns this CQ */ +#define MLX4_CQ_LOCKED (MLX4_EN_CQ_STATENAPI | MLX4_EN_CQ_STATEPOLL) +#define MLX4_EN_CQ_STATENAPI_YIELD 4 /* NAPI yielded this CQ */ +#define MLX4_EN_CQ_STATEPOLL_YIELD 8 /* poll yielded this CQ */ +#define CQ_YIELD (MLX4_EN_CQ_STATENAPI_YIELD | MLX4_EN_CQ_STATEPOLL_YIELD) +#define CQ_USER_PEND (MLX4_EN_CQ_STATEPOLL | MLX4_EN_CQ_STATEPOLL_YIELD) + spinlock_t poll_lock; /* protects from LLS/napi conflicts */ +#endif /* CONFIG_NET_RX_BUSY_POLL */ }; struct mlx4_en_port_profile { u32 flags; u32 tx_ring_num; u32 rx_ring_num; u32 tx_ring_size; u32 rx_ring_size; u8 rx_pause; + u8 rx_ppp; u8 tx_pause; - u32 rx_ppp; - u32 tx_ppp; + u8 tx_ppp; + int rss_rings; }; struct mlx4_en_profile { int rss_xor; - int num_lro; - int ip_reasm; - int tcp_rss; int udp_rss; u8 rss_mask; u32 active_ports; u32 small_pkt_int; u8 no_reset; + u8 num_tx_rings_p_up; struct mlx4_en_port_profile prof[MLX4_MAX_PORTS + 1]; }; struct mlx4_en_dev { - struct mlx4_dev *dev; + struct mlx4_dev *dev; struct pci_dev *pdev; struct mutex state_lock; - struct net_device *pndev[MLX4_MAX_PORTS + 1]; - u32 port_cnt; + struct net_device *pndev[MLX4_MAX_PORTS + 1]; + u32 port_cnt; bool device_up; - struct mlx4_en_profile profile; + struct mlx4_en_profile profile; u32 LSO_support; struct workqueue_struct *workqueue; - struct device *dma_device; - void __iomem *uar_map; - struct mlx4_uar priv_uar; + struct device *dma_device; + void __iomem *uar_map; + struct mlx4_uar priv_uar; struct mlx4_mr mr; - u32 priv_pdn; - spinlock_t uar_lock; + u32 priv_pdn; + spinlock_t uar_lock; u8 mac_removed[MLX4_MAX_PORTS + 1]; + unsigned long last_overflow_check; + unsigned long overflow_period; }; struct mlx4_en_rss_map { int base_qpn; struct mlx4_qp qps[MAX_RX_RINGS]; enum mlx4_qp_state state[MAX_RX_RINGS]; struct mlx4_qp indir_qp; enum mlx4_qp_state indir_state; }; -struct mlx4_en_rss_context { - __be32 base_qpn; - __be32 default_qpn; - u16 reserved; - u8 hash_fn; - u8 flags; - __be32 rss_key[10]; - __be32 base_qpn_udp; -}; - struct mlx4_en_port_state { int link_state; int link_speed; int transciver; + int autoneg; }; -struct mlx4_en_pkt_stats { - unsigned long broadcast; - unsigned long rx_prio[8]; - unsigned long tx_prio[8]; -#define NUM_PKT_STATS 17 +enum mlx4_en_mclist_act { + MCLIST_NONE, + MCLIST_REM, + MCLIST_ADD, }; -struct mlx4_en_port_stats { - unsigned long tso_packets; - unsigned long queue_stopped; - unsigned long wake_queue; - unsigned long tx_timeout; - unsigned long rx_alloc_failed; - unsigned long rx_chksum_good; - unsigned long rx_chksum_none; - unsigned long tx_chksum_offload; +struct mlx4_en_mc_list { + struct list_head list; + enum mlx4_en_mclist_act action; + u8 addr[ETH_ALEN]; + u64 reg_id; }; -struct mlx4_en_perf_stats { - u32 tx_poll; - u64 tx_pktsz_avg; - u32 inflight_avg; - u32 tx_coal_avg; - u32 rx_coal_avg; +#ifdef CONFIG_MLX4_EN_DCB +/* Minimal TC BW - setting to 0 will block traffic */ +#define MLX4_EN_BW_MIN 1 +#define MLX4_EN_BW_MAX 100 /* Utilize 100% of the line */ + +#define MLX4_EN_TC_ETS 7 + +#endif + + +enum { + MLX4_EN_FLAG_PROMISC = (1 << 0), + MLX4_EN_FLAG_MC_PROMISC = (1 << 1), + /* whether we need to enable hardware loopback by putting dmac + * in Tx WQE + */ + MLX4_EN_FLAG_ENABLE_HW_LOOPBACK = (1 << 2), + /* whether we need to drop packets that hardware loopback-ed */ + MLX4_EN_FLAG_RX_FILTER_NEEDED = (1 << 3), + MLX4_EN_FLAG_FORCE_PROMISC = (1 << 4), +#ifdef CONFIG_MLX4_EN_DCB + MLX4_EN_FLAG_DCB_ENABLED = (1 << 5) +#endif }; -struct mlx4_en_frag_info { - u16 frag_size; - u16 frag_prefix_size; +#define MLX4_EN_MAC_HASH_SIZE (1 << BITS_PER_BYTE) +#define MLX4_EN_MAC_HASH_IDX 5 + +struct en_port { + struct kobject kobj; + struct mlx4_dev *dev; + u8 port_num; + u8 vport_num; }; -struct mlx4_en_tx_hash_entry { - u8 cnt; - unsigned int small_pkts; - unsigned int big_pkts; - unsigned int ring; +struct mlx4_en_frag_info { + u16 frag_size; + u16 frag_prefix_size; }; + struct mlx4_en_priv { struct mlx4_en_dev *mdev; struct mlx4_en_port_profile *prof; struct net_device *dev; - bool vlgrp_modified; - u32 vlan_register[VLAN_FLTR_SIZE]; - u32 vlan_unregister[VLAN_FLTR_SIZE]; - u32 vlans[VLAN_FLTR_SIZE]; - spinlock_t vlan_lock; + unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; struct mlx4_en_port_state port_state; spinlock_t stats_lock; + /* To allow rules removal while port is going down */ + struct list_head ethtool_list; - unsigned long last_moder_packets; + unsigned long last_moder_packets[MAX_RX_RINGS]; unsigned long last_moder_tx_packets; - unsigned long last_moder_bytes; + unsigned long last_moder_bytes[MAX_RX_RINGS]; unsigned long last_moder_jiffies; - int last_moder_time; + int last_moder_time[MAX_RX_RINGS]; u16 rx_usecs; u16 rx_frames; u16 tx_usecs; u16 tx_frames; u32 pkt_rate_low; u16 rx_usecs_low; u32 pkt_rate_high; u16 rx_usecs_high; u16 sample_interval; u16 adaptive_rx_coal; u32 msg_enable; u32 loopback_ok; u32 validate_loopback; struct mlx4_hwq_resources res; int link_state; int last_link_state; bool port_up; int port; int registered; int allocated; - int rx_csum; - u64 mac; + int stride; + unsigned char current_mac[ETH_ALEN + 2]; + u64 mac; int mac_index; unsigned max_mtu; int base_qpn; + int cqe_factor; struct mlx4_en_rss_map rss_map; - u16 tx_prio_map[8]; + __be32 ctrl_flags; u32 flags; -#define MLX4_EN_FLAG_PROMISC 0x1 + u8 num_tx_rings_p_up; u32 tx_ring_num; u32 rx_ring_num; u32 rx_mb_size; - struct mlx4_en_frag_info frag_info[MLX4_EN_MAX_RX_FRAGS]; - u16 num_frags; + struct mlx4_en_frag_info frag_info[MLX4_EN_MAX_RX_FRAGS]; + u16 rx_alloc_order; + u32 rx_alloc_size; + u32 rx_buf_size; + u16 num_frags; u16 log_rx_info; - int ip_reasm; - bool wol; - struct mlx4_en_tx_ring tx_ring[MAX_TX_RINGS]; - struct mlx4_en_rx_ring rx_ring[MAX_RX_RINGS]; - struct mlx4_en_cq tx_cq[MAX_TX_RINGS]; - struct mlx4_en_cq rx_cq[MAX_RX_RINGS]; - struct mlx4_en_tx_hash_entry tx_hash[MLX4_EN_TX_HASH_SIZE]; - struct work_struct mcast_task; + struct mlx4_en_tx_ring **tx_ring; + struct mlx4_en_rx_ring *rx_ring[MAX_RX_RINGS]; + struct mlx4_en_cq **tx_cq; + struct mlx4_en_cq *rx_cq[MAX_RX_RINGS]; + struct mlx4_qp drop_qp; + struct work_struct rx_mode_task; struct work_struct watchdog_task; struct work_struct linkstate_task; struct delayed_work stats_task; + struct delayed_work service_task; struct mlx4_en_perf_stats pstats; struct mlx4_en_pkt_stats pkstats; + struct mlx4_en_flow_stats flowstats[MLX4_NUM_PRIORITIES]; struct mlx4_en_port_stats port_stats; + struct mlx4_en_vport_stats vport_stats; + struct mlx4_en_vf_stats vf_stats; + DECLARE_BITMAP(stats_bitmap, NUM_ALL_STATS); + struct list_head mc_list; + struct list_head curr_list; + u64 broadcast_id; struct mlx4_en_stat_out_mbox hw_stats; - struct ifmedia media; + int vids[128]; + bool wol; + struct device *ddev; + struct dentry *dev_root; + u32 counter_index; eventhandler_tag vlan_attach; eventhandler_tag vlan_detach; struct callout watchdog_timer; + struct ifmedia media; volatile int blocked; struct sysctl_oid *sysctl; struct sysctl_ctx_list conf_ctx; struct sysctl_ctx_list stat_ctx; +#define MLX4_EN_MAC_HASH_IDX 5 + struct hlist_head mac_hash[MLX4_EN_MAC_HASH_SIZE]; + +#ifdef CONFIG_MLX4_EN_DCB + struct ieee_ets ets; + u16 maxrate[IEEE_8021QAZ_MAX_TCS]; + u8 dcbx_cap; +#endif +#ifdef CONFIG_RFS_ACCEL + spinlock_t filters_lock; + int last_filter_id; + struct list_head filters; + struct hlist_head filter_hash[1 << MLX4_EN_FILTER_HASH_SHIFT]; +#endif + struct en_port *vf_ports[MLX4_MAX_NUM_VF]; + unsigned long last_ifq_jiffies; + u64 if_counters_rx_errors; + u64 if_counters_rx_no_buffer; + }; enum mlx4_en_wol { MLX4_EN_WOL_MAGIC = (1ULL << 61), MLX4_EN_WOL_ENABLED = (1ULL << 62), - MLX4_EN_WOL_DO_MODIFY = (1ULL << 63), }; -int mlx4_en_transmit(struct net_device *dev, struct mbuf *mb); -void mlx4_en_qflush(struct net_device *dev); +struct mlx4_mac_entry { + struct hlist_node hlist; + unsigned char mac[ETH_ALEN + 2]; + u64 reg_id; +}; -int mlx4_en_rx_frags(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, - struct mbuf *mb, struct mlx4_cqe *cqe); -void mlx4_en_flush_frags(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring); +#ifdef CONFIG_NET_RX_BUSY_POLL +static inline void mlx4_en_cq_init_lock(struct mlx4_en_cq *cq) +{ + spin_lock_init(&cq->poll_lock); + cq->state = MLX4_EN_CQ_STATEIDLE; +} + +/* called from the device poll rutine to get ownership of a cq */ +static inline bool mlx4_en_cq_lock_napi(struct mlx4_en_cq *cq) +{ + int rc = true; + spin_lock(&cq->poll_lock); + if (cq->state & MLX4_CQ_LOCKED) { + WARN_ON(cq->state & MLX4_EN_CQ_STATENAPI); + cq->state |= MLX4_EN_CQ_STATENAPI_YIELD; + rc = false; + } else + /* we don't care if someone yielded */ + cq->state = MLX4_EN_CQ_STATENAPI; + spin_unlock(&cq->poll_lock); + return rc; +} + +/* returns true is someone tried to get the cq while napi had it */ +static inline bool mlx4_en_cq_unlock_napi(struct mlx4_en_cq *cq) +{ + int rc = false; + spin_lock(&cq->poll_lock); + WARN_ON(cq->state & (MLX4_EN_CQ_STATEPOLL | + MLX4_EN_CQ_STATENAPI_YIELD)); + + if (cq->state & MLX4_EN_CQ_STATEPOLL_YIELD) + rc = true; + cq->state = MLX4_EN_CQ_STATEIDLE; + spin_unlock(&cq->poll_lock); + return rc; +} + +/* called from mlx4_en_low_latency_poll() */ +static inline bool mlx4_en_cq_lock_poll(struct mlx4_en_cq *cq) +{ + int rc = true; + spin_lock_bh(&cq->poll_lock); + if ((cq->state & MLX4_CQ_LOCKED)) { + struct net_device *dev = cq->dev; + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_rx_ring *rx_ring = priv->rx_ring[cq->ring]; + + cq->state |= MLX4_EN_CQ_STATEPOLL_YIELD; + rc = false; +#ifdef LL_EXTENDED_STATS + rx_ring->yields++; +#endif + } else + /* preserve yield marks */ + cq->state |= MLX4_EN_CQ_STATEPOLL; + spin_unlock_bh(&cq->poll_lock); + return rc; +} + +/* returns true if someone tried to get the cq while it was locked */ +static inline bool mlx4_en_cq_unlock_poll(struct mlx4_en_cq *cq) +{ + int rc = false; + spin_lock_bh(&cq->poll_lock); + WARN_ON(cq->state & (MLX4_EN_CQ_STATENAPI)); + + if (cq->state & MLX4_EN_CQ_STATEPOLL_YIELD) + rc = true; + cq->state = MLX4_EN_CQ_STATEIDLE; + spin_unlock_bh(&cq->poll_lock); + return rc; +} + +/* true if a socket is polling, even if it did not get the lock */ +static inline bool mlx4_en_cq_ll_polling(struct mlx4_en_cq *cq) +{ + WARN_ON(!(cq->state & MLX4_CQ_LOCKED)); + return cq->state & CQ_USER_PEND; +} +#else +static inline void mlx4_en_cq_init_lock(struct mlx4_en_cq *cq) +{ +} + +static inline bool mlx4_en_cq_lock_napi(struct mlx4_en_cq *cq) +{ + return true; +} + +static inline bool mlx4_en_cq_unlock_napi(struct mlx4_en_cq *cq) +{ + return false; +} + +static inline bool mlx4_en_cq_lock_poll(struct mlx4_en_cq *cq) +{ + return false; +} + +static inline bool mlx4_en_cq_unlock_poll(struct mlx4_en_cq *cq) +{ + return false; +} + +static inline bool mlx4_en_cq_ll_polling(struct mlx4_en_cq *cq) +{ + return false; +} +#endif /* CONFIG_NET_RX_BUSY_POLL */ + +#define MLX4_EN_WOL_DO_MODIFY (1ULL << 63) + void mlx4_en_destroy_netdev(struct net_device *dev); int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, struct mlx4_en_port_profile *prof); int mlx4_en_start_port(struct net_device *dev); void mlx4_en_stop_port(struct net_device *dev); void mlx4_en_free_resources(struct mlx4_en_priv *priv); int mlx4_en_alloc_resources(struct mlx4_en_priv *priv); -int mlx4_en_create_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, - int entries, int ring, enum cq_type mode); -void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); -int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); +int mlx4_en_pre_config(struct mlx4_en_priv *priv); +int mlx4_en_create_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq, + int entries, int ring, enum cq_type mode, int node); +void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq); +int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, + int cq_idx); void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); int mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); -void mlx4_en_poll_tx_cq(unsigned long data); void mlx4_en_tx_irq(struct mlx4_cq *mcq); u16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb); -int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring, - u32 size, u16 stride); -void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring); +int mlx4_en_transmit(struct ifnet *dev, struct mbuf *m); +int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring **pring, + u32 size, u16 stride, int node, int queue_idx); +void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring **pring); int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring, - int cq); + int cq, int user_prio); void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring); +void mlx4_en_qflush(struct ifnet *dev); int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring, u32 size); + struct mlx4_en_rx_ring **pring, + u32 size, int node); void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring); + struct mlx4_en_rx_ring **pring, + u32 size, u16 stride); +void mlx4_en_tx_que(void *context, int pending); +void mlx4_en_rx_que(void *context, int pending); int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv); void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring); int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget); -int mlx4_en_process_rx_cq_mb(struct net_device *dev, - struct mlx4_en_cq *cq, - int budget); -void mlx4_en_tx_que(void *context, int pending); -void mlx4_en_rx_que(void *context, int pending); +void mlx4_en_poll_tx_cq(unsigned long data); void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride, - int is_tx, int rss, int qpn, int cqn, - struct mlx4_qp_context *context); + int is_tx, int rss, int qpn, int cqn, int user_prio, + struct mlx4_qp_context *context); void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event); int mlx4_en_map_buffer(struct mlx4_buf *buf); void mlx4_en_unmap_buffer(struct mlx4_buf *buf); - void mlx4_en_calc_rx_buf(struct net_device *dev); -void mlx4_en_set_prio_map(struct mlx4_en_priv *priv, u16 *prio_map, u32 ring_num); + int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv); void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv); +int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv); +void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv); int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring); void mlx4_en_rx_irq(struct mlx4_cq *mcq); int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode); -int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, u8 port, u32 *vlans); -int mlx4_SET_PORT_general(struct mlx4_dev *dev, u8 port, int mtu, - u8 pptx, u8 pfctx, u8 pprx, u8 pfcrx); -int mlx4_SET_PORT_qpn_calc(struct mlx4_dev *dev, u8 port, u32 base_qpn, - u8 promisc); +int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, struct mlx4_en_priv *priv); int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset); int mlx4_en_QUERY_PORT(struct mlx4_en_dev *mdev, u8 port); +int mlx4_en_get_vport_stats(struct mlx4_en_dev *mdev, u8 port); +void mlx4_en_create_debug_files(struct mlx4_en_priv *priv); +void mlx4_en_delete_debug_files(struct mlx4_en_priv *priv); +int mlx4_en_register_debugfs(void); +void mlx4_en_unregister_debugfs(void); +#ifdef CONFIG_MLX4_EN_DCB +extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_ops; +extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_pfc_ops; +#endif + +int mlx4_en_setup_tc(struct net_device *dev, u8 up); + +#ifdef CONFIG_RFS_ACCEL +void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring *rx_ring); +#endif + #define MLX4_EN_NUM_SELF_TEST 5 void mlx4_en_ex_selftest(struct net_device *dev, u32 *flags, u64 *buf); -u64 mlx4_en_mac_to_u64(u8 *addr); +void mlx4_en_ptp_overflow_check(struct mlx4_en_dev *mdev); /* + * Functions for time stamping + */ +#define SKBTX_HW_TSTAMP (1 << 0) +#define SKBTX_IN_PROGRESS (1 << 2) + +u64 mlx4_en_get_cqe_ts(struct mlx4_cqe *cqe); + +/* Functions for caching and restoring statistics */ +int mlx4_en_get_sset_count(struct net_device *dev, int sset); +void mlx4_en_restore_ethtool_stats(struct mlx4_en_priv *priv, + u64 *data); + +/* * Globals */ extern const struct ethtool_ops mlx4_en_ethtool_ops; + +/* + * Defines for link speed - needed by selftest + */ +#define MLX4_EN_LINK_SPEED_1G 1000 +#define MLX4_EN_LINK_SPEED_10G 10000 +#define MLX4_EN_LINK_SPEED_40G 40000 + +enum { + NETIF_MSG_DRV = 0x0001, + NETIF_MSG_PROBE = 0x0002, + NETIF_MSG_LINK = 0x0004, + NETIF_MSG_TIMER = 0x0008, + NETIF_MSG_IFDOWN = 0x0010, + NETIF_MSG_IFUP = 0x0020, + NETIF_MSG_RX_ERR = 0x0040, + NETIF_MSG_TX_ERR = 0x0080, + NETIF_MSG_TX_QUEUED = 0x0100, + NETIF_MSG_INTR = 0x0200, + NETIF_MSG_TX_DONE = 0x0400, + NETIF_MSG_RX_STATUS = 0x0800, + NETIF_MSG_PKTDATA = 0x1000, + NETIF_MSG_HW = 0x2000, + NETIF_MSG_WOL = 0x4000, +}; + + +/* + * printk / logging functions + */ + +#define en_print(level, priv, format, arg...) \ + { \ + if ((priv)->registered) \ + printk(level "%s: %s: " format, DRV_NAME, \ + (priv->dev)->if_xname, ## arg); \ + else \ + printk(level "%s: %s: Port %d: " format, \ + DRV_NAME, dev_name(&priv->mdev->pdev->dev), \ + (priv)->port, ## arg); \ + } + + +#define en_dbg(mlevel, priv, format, arg...) \ +do { \ + if (NETIF_MSG_##mlevel & priv->msg_enable) \ + en_print(KERN_DEBUG, priv, format, ##arg); \ +} while (0) +#define en_warn(priv, format, arg...) \ + en_print(KERN_WARNING, priv, format, ##arg) +#define en_err(priv, format, arg...) \ + en_print(KERN_ERR, priv, format, ##arg) +#define en_info(priv, format, arg...) \ + en_print(KERN_INFO, priv, format, ## arg) + +#define mlx4_err(mdev, format, arg...) \ + pr_err("%s %s: " format, DRV_NAME, \ + dev_name(&mdev->pdev->dev), ##arg) +#define mlx4_info(mdev, format, arg...) \ + pr_info("%s %s: " format, DRV_NAME, \ + dev_name(&mdev->pdev->dev), ##arg) +#define mlx4_warn(mdev, format, arg...) \ + pr_warning("%s %s: " format, DRV_NAME, \ + dev_name(&mdev->pdev->dev), ##arg) + #endif Index: stable/9/sys/ofed/drivers/net/mlx4/mlx4_stats.h =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/mlx4_stats.h (nonexistent) +++ stable/9/sys/ofed/drivers/net/mlx4/mlx4_stats.h (revision 273864) @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2014 Mellanox Technologies Ltd. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _MLX4_STATS_ +#define _MLX4_STATS_ + + +#ifdef MLX4_EN_PERF_STAT +#define NUM_PERF_STATS NUM_PERF_COUNTERS +#else +#define NUM_PERF_STATS 0 +#endif + +#define NUM_PRIORITIES 9 +#define NUM_PRIORITY_STATS 2 + +struct mlx4_en_pkt_stats { + unsigned long rx_packets; + unsigned long rx_bytes; + unsigned long rx_multicast_packets; + unsigned long rx_broadcast_packets; + unsigned long rx_errors; + unsigned long rx_dropped; + unsigned long rx_length_errors; + unsigned long rx_over_errors; + unsigned long rx_crc_errors; + unsigned long rx_jabbers; + unsigned long rx_in_range_length_error; + unsigned long rx_out_range_length_error; + unsigned long rx_lt_64_bytes_packets; + unsigned long rx_127_bytes_packets; + unsigned long rx_255_bytes_packets; + unsigned long rx_511_bytes_packets; + unsigned long rx_1023_bytes_packets; + unsigned long rx_1518_bytes_packets; + unsigned long rx_1522_bytes_packets; + unsigned long rx_1548_bytes_packets; + unsigned long rx_gt_1548_bytes_packets; + unsigned long tx_packets; + unsigned long tx_bytes; + unsigned long tx_multicast_packets; + unsigned long tx_broadcast_packets; + unsigned long tx_errors; + unsigned long tx_dropped; + unsigned long tx_lt_64_bytes_packets; + unsigned long tx_127_bytes_packets; + unsigned long tx_255_bytes_packets; + unsigned long tx_511_bytes_packets; + unsigned long tx_1023_bytes_packets; + unsigned long tx_1518_bytes_packets; + unsigned long tx_1522_bytes_packets; + unsigned long tx_1548_bytes_packets; + unsigned long tx_gt_1548_bytes_packets; + unsigned long rx_prio[NUM_PRIORITIES][NUM_PRIORITY_STATS]; + unsigned long tx_prio[NUM_PRIORITIES][NUM_PRIORITY_STATS]; +#define NUM_PKT_STATS 72 +}; + +struct mlx4_en_vf_stats { + unsigned long rx_packets; + unsigned long rx_bytes; + unsigned long rx_multicast_packets; + unsigned long rx_broadcast_packets; + unsigned long rx_errors; + unsigned long rx_dropped; + unsigned long tx_packets; + unsigned long tx_bytes; + unsigned long tx_multicast_packets; + unsigned long tx_broadcast_packets; + unsigned long tx_errors; +#define NUM_VF_STATS 11 +}; + +struct mlx4_en_vport_stats { + unsigned long rx_unicast_packets; + unsigned long rx_unicast_bytes; + unsigned long rx_multicast_packets; + unsigned long rx_multicast_bytes; + unsigned long rx_broadcast_packets; + unsigned long rx_broadcast_bytes; + unsigned long rx_dropped; + unsigned long rx_errors; + unsigned long tx_unicast_packets; + unsigned long tx_unicast_bytes; + unsigned long tx_multicast_packets; + unsigned long tx_multicast_bytes; + unsigned long tx_broadcast_packets; + unsigned long tx_broadcast_bytes; + unsigned long tx_errors; +#define NUM_VPORT_STATS 15 +}; + +struct mlx4_en_port_stats { + unsigned long tso_packets; + unsigned long queue_stopped; + unsigned long wake_queue; + unsigned long tx_timeout; + unsigned long rx_alloc_failed; + unsigned long rx_chksum_good; + unsigned long rx_chksum_none; + unsigned long tx_chksum_offload; +#define NUM_PORT_STATS 8 +}; + +struct mlx4_en_perf_stats { + u32 tx_poll; + u64 tx_pktsz_avg; + u32 inflight_avg; + u16 tx_coal_avg; + u16 rx_coal_avg; + u32 napi_quota; +#define NUM_PERF_COUNTERS 6 +}; + +struct mlx4_en_flow_stats { + u64 rx_pause; + u64 rx_pause_duration; + u64 rx_pause_transition; + u64 tx_pause; + u64 tx_pause_duration; + u64 tx_pause_transition; +}; +#define MLX4_NUM_PRIORITIES 8 +#define NUM_FLOW_PRIORITY_STATS 6 +#define NUM_FLOW_STATS (NUM_FLOW_PRIORITY_STATS*MLX4_NUM_PRIORITIES) + + +struct mlx4_en_stat_out_flow_control_mbox { + /* Total number of PAUSE frames received from the far-end port */ + __be64 rx_pause; + /* Total number of microseconds that far-end port requested to pause + * transmission of packets + */ + __be64 rx_pause_duration; + /* Number of received transmission from XOFF state to XON state */ + __be64 rx_pause_transition; + /* Total number of PAUSE frames sent from the far-end port */ + __be64 tx_pause; + /* Total time in microseconds that transmission of packets has been + * paused + */ + __be64 tx_pause_duration; + /* Number of transmitter transitions from XOFF state to XON state */ + __be64 tx_pause_transition; + /* Reserverd */ + __be64 reserved[2]; +}; + +int mlx4_get_vport_ethtool_stats(struct mlx4_dev *dev, int port, + struct mlx4_en_vport_stats *vport_stats, + int reset); + +#define NUM_ALL_STATS (NUM_PKT_STATS + NUM_FLOW_STATS + NUM_VPORT_STATS + \ + NUM_VF_STATS + NUM_PORT_STATS + NUM_PERF_STATS) +#endif Property changes on: stable/9/sys/ofed/drivers/net/mlx4/mlx4_stats.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/drivers/net/mlx4/mr.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/mr.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/mr.c (revision 273864) @@ -1,773 +1,996 @@ /* * Copyright (c) 2004 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include +#include #include +#include +#include +#include +#include #include +#include + #include "mlx4.h" #include "icm.h" -/* - * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits. - */ -struct mlx4_mpt_entry { - __be32 flags; - __be32 qpn; - __be32 key; - __be32 pd_flags; - __be64 start; - __be64 length; - __be32 lkey; - __be32 win_cnt; - u8 reserved1; - u8 flags2; - u8 reserved2; - u8 mtt_rep; - __be64 mtt_seg; - __be32 mtt_sz; - __be32 entity_size; - __be32 first_byte_offset; -} __attribute__((packed)); - -#define MLX4_MPT_FLAG_SW_OWNS (0xfUL << 28) -#define MLX4_MPT_FLAG_FREE (0x3UL << 28) -#define MLX4_MPT_FLAG_MIO (1 << 17) -#define MLX4_MPT_FLAG_BIND_ENABLE (1 << 15) -#define MLX4_MPT_FLAG_PHYSICAL (1 << 9) -#define MLX4_MPT_FLAG_REGION (1 << 8) - -#define MLX4_MPT_PD_FLAG_FAST_REG (1 << 27) -#define MLX4_MPT_PD_FLAG_RAE (1 << 28) -#define MLX4_MPT_PD_FLAG_EN_INV (3 << 24) - -#define MLX4_MPT_FLAG2_FBO_EN (1 << 7) - -#define MLX4_MPT_STATUS_SW 0xF0 -#define MLX4_MPT_STATUS_HW 0x00 - static u32 mlx4_buddy_alloc(struct mlx4_buddy *buddy, int order) { int o; int m; u32 seg; spin_lock(&buddy->lock); for (o = order; o <= buddy->max_order; ++o) if (buddy->num_free[o]) { m = 1 << (buddy->max_order - o); seg = find_first_bit(buddy->bits[o], m); if (seg < m) goto found; } spin_unlock(&buddy->lock); return -1; found: clear_bit(seg, buddy->bits[o]); --buddy->num_free[o]; while (o > order) { --o; seg <<= 1; set_bit(seg ^ 1, buddy->bits[o]); ++buddy->num_free[o]; } spin_unlock(&buddy->lock); seg <<= order; return seg; } static void mlx4_buddy_free(struct mlx4_buddy *buddy, u32 seg, int order) { seg >>= order; spin_lock(&buddy->lock); while (test_bit(seg ^ 1, buddy->bits[order])) { clear_bit(seg ^ 1, buddy->bits[order]); --buddy->num_free[order]; seg >>= 1; ++order; } set_bit(seg, buddy->bits[order]); ++buddy->num_free[order]; spin_unlock(&buddy->lock); } static int mlx4_buddy_init(struct mlx4_buddy *buddy, int max_order) { int i, s; buddy->max_order = max_order; spin_lock_init(&buddy->lock); - buddy->bits = kzalloc((buddy->max_order + 1) * sizeof (long *), + buddy->bits = kcalloc(buddy->max_order + 1, sizeof (long *), GFP_KERNEL); - buddy->num_free = kzalloc((buddy->max_order + 1) * sizeof (int *), + buddy->num_free = kcalloc((buddy->max_order + 1), sizeof *buddy->num_free, GFP_KERNEL); if (!buddy->bits || !buddy->num_free) goto err_out; for (i = 0; i <= buddy->max_order; ++i) { s = BITS_TO_LONGS(1 << (buddy->max_order - i)); - buddy->bits[i] = kmalloc(s * sizeof (long), GFP_KERNEL); - if (!buddy->bits[i]) + buddy->bits[i] = kcalloc(s, sizeof (long), GFP_KERNEL | __GFP_NOWARN); + if (!buddy->bits[i]) goto err_out_free; - bitmap_zero(buddy->bits[i], 1 << (buddy->max_order - i)); } set_bit(0, buddy->bits[buddy->max_order]); buddy->num_free[buddy->max_order] = 1; return 0; err_out_free: for (i = 0; i <= buddy->max_order; ++i) kfree(buddy->bits[i]); err_out: kfree(buddy->bits); kfree(buddy->num_free); return -ENOMEM; } static void mlx4_buddy_cleanup(struct mlx4_buddy *buddy) { int i; for (i = 0; i <= buddy->max_order; ++i) kfree(buddy->bits[i]); kfree(buddy->bits); kfree(buddy->num_free); } -static u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) +u32 __mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) { struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; u32 seg; + int seg_order; + u32 offset; - seg = mlx4_buddy_alloc(&mr_table->mtt_buddy, order); + seg_order = max_t(int, order - log_mtts_per_seg, 0); + + seg = mlx4_buddy_alloc(&mr_table->mtt_buddy, seg_order); if (seg == -1) return -1; - if (mlx4_table_get_range(dev, &mr_table->mtt_table, seg, - seg + (1 << order) - 1)) { - mlx4_buddy_free(&mr_table->mtt_buddy, seg, order); + offset = seg * (1 << log_mtts_per_seg); + + if (mlx4_table_get_range(dev, &mr_table->mtt_table, offset, + offset + (1 << order) - 1)) { + mlx4_buddy_free(&mr_table->mtt_buddy, seg, seg_order); return -1; } - return seg; + return offset; } +static u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) +{ + u64 in_param = 0; + u64 out_param; + int err; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, order); + err = mlx4_cmd_imm(dev, in_param, &out_param, RES_MTT, + RES_OP_RESERVE_AND_MAP, + MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + if (err) + return -1; + return get_param_l(&out_param); + } + return __mlx4_alloc_mtt_range(dev, order); +} + int mlx4_mtt_init(struct mlx4_dev *dev, int npages, int page_shift, struct mlx4_mtt *mtt) { int i; if (!npages) { mtt->order = -1; mtt->page_shift = MLX4_ICM_PAGE_SHIFT; return 0; } else mtt->page_shift = page_shift; - for (mtt->order = 0, i = dev->caps.mtts_per_seg; i < npages; i <<= 1) + for (mtt->order = 0, i = 1; i < npages; i <<= 1) ++mtt->order; - mtt->first_seg = mlx4_alloc_mtt_range(dev, mtt->order); - if (mtt->first_seg == -1) + mtt->offset = mlx4_alloc_mtt_range(dev, mtt->order); + if (mtt->offset == -1) { + mlx4_err(dev, "Failed to allocate mtts for %d pages(order %d)\n", + npages, mtt->order); return -ENOMEM; + } return 0; } EXPORT_SYMBOL_GPL(mlx4_mtt_init); -void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt) +void __mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) { + u32 first_seg; + int seg_order; struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; + seg_order = max_t(int, order - log_mtts_per_seg, 0); + first_seg = offset / (1 << log_mtts_per_seg); + + mlx4_buddy_free(&mr_table->mtt_buddy, first_seg, seg_order); + mlx4_table_put_range(dev, &mr_table->mtt_table, offset, + offset + (1 << order) - 1); +} + +static void mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) +{ + u64 in_param = 0; + int err; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, offset); + set_param_h(&in_param, order); + err = mlx4_cmd(dev, in_param, RES_MTT, RES_OP_RESERVE_AND_MAP, + MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + if (err) + mlx4_warn(dev, "Failed to free mtt range at:" + "%d order:%d\n", offset, order); + return; + } + __mlx4_free_mtt_range(dev, offset, order); +} + +void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt) +{ if (mtt->order < 0) return; - mlx4_buddy_free(&mr_table->mtt_buddy, mtt->first_seg, mtt->order); - mlx4_table_put_range(dev, &mr_table->mtt_table, mtt->first_seg, - mtt->first_seg + (1 << mtt->order) - 1); + mlx4_free_mtt_range(dev, mtt->offset, mtt->order); } EXPORT_SYMBOL_GPL(mlx4_mtt_cleanup); u64 mlx4_mtt_addr(struct mlx4_dev *dev, struct mlx4_mtt *mtt) { - return (u64) mtt->first_seg * dev->caps.mtt_entry_sz; + return (u64) mtt->offset * dev->caps.mtt_entry_sz; } EXPORT_SYMBOL_GPL(mlx4_mtt_addr); static u32 hw_index_to_key(u32 ind) { return (ind >> 24) | (ind << 8); } static u32 key_to_hw_index(u32 key) { return (key << 24) | (key >> 8); } static int mlx4_SW2HW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, int mpt_index) { - return mlx4_cmd(dev, mailbox->dma, mpt_index, 0, MLX4_CMD_SW2HW_MPT, - MLX4_CMD_TIME_CLASS_B); + return mlx4_cmd(dev, mailbox->dma, mpt_index, + 0, MLX4_CMD_SW2HW_MPT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); } static int mlx4_HW2SW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, int mpt_index) { return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, mpt_index, - !mailbox, MLX4_CMD_HW2SW_MPT, MLX4_CMD_TIME_CLASS_B); + !mailbox, MLX4_CMD_HW2SW_MPT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); } -int mlx4_mr_reserve_range(struct mlx4_dev *dev, int cnt, int align, u32 *base_mridx) +static int mlx4_mr_alloc_reserved(struct mlx4_dev *dev, u32 mridx, u32 pd, + u64 iova, u64 size, u32 access, int npages, + int page_shift, struct mlx4_mr *mr) { + mr->iova = iova; + mr->size = size; + mr->pd = pd; + mr->access = access; + mr->enabled = MLX4_MPT_DISABLED; + mr->key = hw_index_to_key(mridx); + + return mlx4_mtt_init(dev, npages, page_shift, &mr->mtt); +} + +static int mlx4_WRITE_MTT(struct mlx4_dev *dev, + struct mlx4_cmd_mailbox *mailbox, + int num_entries) +{ + return mlx4_cmd(dev, mailbox->dma, num_entries, 0, MLX4_CMD_WRITE_MTT, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); +} + +int __mlx4_mpt_reserve(struct mlx4_dev *dev) +{ struct mlx4_priv *priv = mlx4_priv(dev); - u32 mridx; - mridx = mlx4_bitmap_alloc_range(&priv->mr_table.mpt_bitmap, cnt, align); - if (mridx == -1) - return -ENOMEM; + return mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap); +} - *base_mridx = mridx; - return 0; +static int mlx4_mpt_reserve(struct mlx4_dev *dev) +{ + u64 out_param; + if (mlx4_is_mfunc(dev)) { + if (mlx4_cmd_imm(dev, 0, &out_param, RES_MPT, RES_OP_RESERVE, + MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) + return -1; + return get_param_l(&out_param); + } + return __mlx4_mpt_reserve(dev); } -EXPORT_SYMBOL_GPL(mlx4_mr_reserve_range); -void mlx4_mr_release_range(struct mlx4_dev *dev, u32 base_mridx, int cnt) +void __mlx4_mpt_release(struct mlx4_dev *dev, u32 index) { struct mlx4_priv *priv = mlx4_priv(dev); - mlx4_bitmap_free_range(&priv->mr_table.mpt_bitmap, base_mridx, cnt); + + mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, index, MLX4_NO_RR); } -EXPORT_SYMBOL_GPL(mlx4_mr_release_range); -int mlx4_mr_alloc_reserved(struct mlx4_dev *dev, u32 mridx, u32 pd, - u64 iova, u64 size, u32 access, int npages, - int page_shift, struct mlx4_mr *mr) +static void mlx4_mpt_release(struct mlx4_dev *dev, u32 index) { - mr->iova = iova; - mr->size = size; - mr->pd = pd; - mr->access = access; - mr->enabled = 0; - mr->key = hw_index_to_key(mridx); + u64 in_param = 0; - return mlx4_mtt_init(dev, npages, page_shift, &mr->mtt); + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, index); + if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_RESERVE, + MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) + mlx4_warn(dev, "Failed to release mr index:%d\n", + index); + return; + } + __mlx4_mpt_release(dev, index); } -EXPORT_SYMBOL_GPL(mlx4_mr_alloc_reserved); +int __mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index) +{ + struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; + + return mlx4_table_get(dev, &mr_table->dmpt_table, index); +} + +static int mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index) +{ + u64 param = 0; + + if (mlx4_is_mfunc(dev)) { + set_param_l(¶m, index); + return mlx4_cmd_imm(dev, param, ¶m, RES_MPT, RES_OP_MAP_ICM, + MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + } + return __mlx4_mpt_alloc_icm(dev, index); +} + +void __mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index) +{ + struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; + + mlx4_table_put(dev, &mr_table->dmpt_table, index); +} + +static void mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index) +{ + u64 in_param = 0; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, index); + if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_MAP_ICM, + MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED)) + mlx4_warn(dev, "Failed to free icm of mr index:%d\n", + index); + return; + } + return __mlx4_mpt_free_icm(dev, index); +} + int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access, int npages, int page_shift, struct mlx4_mr *mr) { - struct mlx4_priv *priv = mlx4_priv(dev); u32 index; int err; - index = mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap); + index = mlx4_mpt_reserve(dev); if (index == -1) return -ENOMEM; err = mlx4_mr_alloc_reserved(dev, index, pd, iova, size, access, npages, page_shift, mr); if (err) - mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, index); + mlx4_mpt_release(dev, index); return err; } EXPORT_SYMBOL_GPL(mlx4_mr_alloc); -void mlx4_mr_free_reserved(struct mlx4_dev *dev, struct mlx4_mr *mr) +static int mlx4_mr_free_reserved(struct mlx4_dev *dev, struct mlx4_mr *mr) { int err; - if (mr->enabled) { + if (mr->enabled == MLX4_MPT_EN_HW) { err = mlx4_HW2SW_MPT(dev, NULL, key_to_hw_index(mr->key) & (dev->caps.num_mpts - 1)); - if (err) - mlx4_warn(dev, "HW2SW_MPT failed (%d)\n", err); - } + if (err) { + mlx4_warn(dev, "HW2SW_MPT failed (%d).", err); + mlx4_warn(dev, "Most likely the MR has MWs bound to it.\n"); + return err; + } + mr->enabled = MLX4_MPT_EN_SW; + } mlx4_mtt_cleanup(dev, &mr->mtt); + + return 0; } -EXPORT_SYMBOL_GPL(mlx4_mr_free_reserved); -void mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr) +int mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr) { - struct mlx4_priv *priv = mlx4_priv(dev); - mlx4_mr_free_reserved(dev, mr); - mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, key_to_hw_index(mr->key)); + int ret; + + ret = mlx4_mr_free_reserved(dev, mr); + if (ret) + return ret; + if (mr->enabled) + mlx4_mpt_free_icm(dev, key_to_hw_index(mr->key)); + mlx4_mpt_release(dev, key_to_hw_index(mr->key)); + + return 0; } EXPORT_SYMBOL_GPL(mlx4_mr_free); int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr) { - struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; struct mlx4_cmd_mailbox *mailbox; struct mlx4_mpt_entry *mpt_entry; int err; - err = mlx4_table_get(dev, &mr_table->dmpt_table, key_to_hw_index(mr->key)); + err = mlx4_mpt_alloc_icm(dev, key_to_hw_index(mr->key)); if (err) return err; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) { err = PTR_ERR(mailbox); goto err_table; } mpt_entry = mailbox->buf; memset(mpt_entry, 0, sizeof *mpt_entry); mpt_entry->flags = cpu_to_be32(MLX4_MPT_FLAG_MIO | MLX4_MPT_FLAG_REGION | mr->access); mpt_entry->key = cpu_to_be32(key_to_hw_index(mr->key)); mpt_entry->pd_flags = cpu_to_be32(mr->pd | MLX4_MPT_PD_FLAG_EN_INV); mpt_entry->start = cpu_to_be64(mr->iova); mpt_entry->length = cpu_to_be64(mr->size); mpt_entry->entity_size = cpu_to_be32(mr->mtt.page_shift); if (mr->mtt.order < 0) { mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_PHYSICAL); - mpt_entry->mtt_seg = 0; + mpt_entry->mtt_addr = 0; } else { - mpt_entry->mtt_seg = cpu_to_be64(mlx4_mtt_addr(dev, &mr->mtt)); + mpt_entry->mtt_addr = cpu_to_be64(mlx4_mtt_addr(dev, + &mr->mtt)); } if (mr->mtt.order >= 0 && mr->mtt.page_shift == 0) { /* fast register MR in free state */ mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_FAST_REG | MLX4_MPT_PD_FLAG_RAE); - mpt_entry->mtt_sz = cpu_to_be32((1 << mr->mtt.order) * - dev->caps.mtts_per_seg); + mpt_entry->mtt_sz = cpu_to_be32(1 << mr->mtt.order); } else { mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_SW_OWNS); } err = mlx4_SW2HW_MPT(dev, mailbox, key_to_hw_index(mr->key) & (dev->caps.num_mpts - 1)); if (err) { mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); goto err_cmd; } + mr->enabled = MLX4_MPT_EN_HW; - mr->enabled = 1; - mlx4_free_cmd_mailbox(dev, mailbox); return 0; err_cmd: mlx4_free_cmd_mailbox(dev, mailbox); err_table: - mlx4_table_put(dev, &mr_table->dmpt_table, key_to_hw_index(mr->key)); + mlx4_mpt_free_icm(dev, key_to_hw_index(mr->key)); return err; } EXPORT_SYMBOL_GPL(mlx4_mr_enable); static int mlx4_write_mtt_chunk(struct mlx4_dev *dev, struct mlx4_mtt *mtt, int start_index, int npages, u64 *page_list) { struct mlx4_priv *priv = mlx4_priv(dev); __be64 *mtts; dma_addr_t dma_handle; int i; - int s = start_index * sizeof (u64); - /* All MTTs must fit in the same page */ - if (start_index / (PAGE_SIZE / sizeof (u64)) != - (start_index + npages - 1) / (PAGE_SIZE / sizeof (u64))) - return -EINVAL; + mtts = mlx4_table_find(&priv->mr_table.mtt_table, mtt->offset + + start_index, &dma_handle); - if (start_index & (dev->caps.mtts_per_seg - 1)) - return -EINVAL; - - mtts = mlx4_table_find(&priv->mr_table.mtt_table, mtt->first_seg + - s / dev->caps.mtt_entry_sz, &dma_handle); if (!mtts) return -ENOMEM; + dma_sync_single_for_cpu(&dev->pdev->dev, dma_handle, + npages * sizeof (u64), DMA_TO_DEVICE); + for (i = 0; i < npages; ++i) mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); - dma_sync_single(&dev->pdev->dev, dma_handle, npages * sizeof (u64), DMA_TO_DEVICE); + dma_sync_single_for_device(&dev->pdev->dev, dma_handle, + npages * sizeof (u64), DMA_TO_DEVICE); return 0; } -int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - int start_index, int npages, u64 *page_list) +int __mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + int start_index, int npages, u64 *page_list) { + int err = 0; int chunk; - int err; + int mtts_per_page; + int max_mtts_first_page; - if (mtt->order < 0) - return -EINVAL; + /* compute how may mtts fit in the first page */ + mtts_per_page = PAGE_SIZE / sizeof(u64); + max_mtts_first_page = mtts_per_page - (mtt->offset + start_index) + % mtts_per_page; + chunk = min_t(int, max_mtts_first_page, npages); + while (npages > 0) { - chunk = min_t(int, PAGE_SIZE / sizeof(u64), npages); err = mlx4_write_mtt_chunk(dev, mtt, start_index, chunk, page_list); if (err) return err; - npages -= chunk; start_index += chunk; page_list += chunk; + + chunk = min_t(int, mtts_per_page, npages); } + return err; +} - return 0; +int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + int start_index, int npages, u64 *page_list) +{ + struct mlx4_cmd_mailbox *mailbox = NULL; + __be64 *inbox = NULL; + int chunk; + int err = 0; + int i; + + if (mtt->order < 0) + return -EINVAL; + + if (mlx4_is_mfunc(dev)) { + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + inbox = mailbox->buf; + + while (npages > 0) { + chunk = min_t(int, MLX4_MAILBOX_SIZE / sizeof(u64) - 2, + npages); + inbox[0] = cpu_to_be64(mtt->offset + start_index); + inbox[1] = 0; + for (i = 0; i < chunk; ++i) + inbox[i + 2] = cpu_to_be64(page_list[i] | + MLX4_MTT_FLAG_PRESENT); + err = mlx4_WRITE_MTT(dev, mailbox, chunk); + if (err) { + mlx4_free_cmd_mailbox(dev, mailbox); + return err; + } + + npages -= chunk; + start_index += chunk; + page_list += chunk; + } + mlx4_free_cmd_mailbox(dev, mailbox); + return err; + } + + return __mlx4_write_mtt(dev, mtt, start_index, npages, page_list); } EXPORT_SYMBOL_GPL(mlx4_write_mtt); int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, struct mlx4_buf *buf) { u64 *page_list; int err; int i; page_list = kmalloc(buf->npages * sizeof *page_list, GFP_KERNEL); if (!page_list) return -ENOMEM; for (i = 0; i < buf->npages; ++i) - if (buf->direct.map) + if (buf->nbufs == 1) page_list[i] = buf->direct.map + (i << buf->page_shift); else page_list[i] = buf->page_list[i].map; err = mlx4_write_mtt(dev, mtt, 0, buf->npages, page_list); kfree(page_list); return err; } EXPORT_SYMBOL_GPL(mlx4_buf_write_mtt); +int mlx4_mw_alloc(struct mlx4_dev *dev, u32 pd, enum mlx4_mw_type type, + struct mlx4_mw *mw) +{ + u32 index; + + index = mlx4_mpt_reserve(dev); + if (index == -1) + return -ENOMEM; + + mw->key = hw_index_to_key(index); + mw->pd = pd; + mw->type = type; + mw->enabled = MLX4_MPT_DISABLED; + + return 0; +} +EXPORT_SYMBOL_GPL(mlx4_mw_alloc); + +int mlx4_mw_enable(struct mlx4_dev *dev, struct mlx4_mw *mw) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_mpt_entry *mpt_entry; + int err; + + err = mlx4_mpt_alloc_icm(dev, key_to_hw_index(mw->key)); + if (err) + return err; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = PTR_ERR(mailbox); + goto err_table; + } + mpt_entry = mailbox->buf; + + memset(mpt_entry, 0, sizeof(*mpt_entry)); + + /* Note that the MLX4_MPT_FLAG_REGION bit in mpt_entry->flags is turned + * off, thus creating a memory window and not a memory region. + */ + mpt_entry->key = cpu_to_be32(key_to_hw_index(mw->key)); + mpt_entry->pd_flags = cpu_to_be32(mw->pd); + if (mw->type == MLX4_MW_TYPE_2) { + mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); + mpt_entry->qpn = cpu_to_be32(MLX4_MPT_QP_FLAG_BOUND_QP); + mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_EN_INV); + } + + err = mlx4_SW2HW_MPT(dev, mailbox, + key_to_hw_index(mw->key) & + (dev->caps.num_mpts - 1)); + if (err) { + mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); + goto err_cmd; + } + mw->enabled = MLX4_MPT_EN_HW; + + mlx4_free_cmd_mailbox(dev, mailbox); + + return 0; + +err_cmd: + mlx4_free_cmd_mailbox(dev, mailbox); + +err_table: + mlx4_mpt_free_icm(dev, key_to_hw_index(mw->key)); + return err; +} +EXPORT_SYMBOL_GPL(mlx4_mw_enable); + +void mlx4_mw_free(struct mlx4_dev *dev, struct mlx4_mw *mw) +{ + int err; + + if (mw->enabled == MLX4_MPT_EN_HW) { + err = mlx4_HW2SW_MPT(dev, NULL, + key_to_hw_index(mw->key) & + (dev->caps.num_mpts - 1)); + if (err) + mlx4_warn(dev, "xxx HW2SW_MPT failed (%d)\n", err); + + mw->enabled = MLX4_MPT_EN_SW; + } + if (mw->enabled) + mlx4_mpt_free_icm(dev, key_to_hw_index(mw->key)); + mlx4_mpt_release(dev, key_to_hw_index(mw->key)); +} +EXPORT_SYMBOL_GPL(mlx4_mw_free); + int mlx4_init_mr_table(struct mlx4_dev *dev) { - struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_mr_table *mr_table = &priv->mr_table; int err; + /* Nothing to do for slaves - all MR handling is forwarded + * to the master */ + if (mlx4_is_slave(dev)) + return 0; + if (!is_power_of_2(dev->caps.num_mpts)) return -EINVAL; err = mlx4_bitmap_init(&mr_table->mpt_bitmap, dev->caps.num_mpts, ~0, dev->caps.reserved_mrws, 0); if (err) return err; err = mlx4_buddy_init(&mr_table->mtt_buddy, - ilog2(dev->caps.num_mtt_segs)); + ilog2(div_u64(dev->caps.num_mtts, + (1 << log_mtts_per_seg)))); if (err) goto err_buddy; if (dev->caps.reserved_mtts) { - if (mlx4_alloc_mtt_range(dev, fls(dev->caps.reserved_mtts - 1)) == -1) { - mlx4_warn(dev, "MTT table of order %d is too small.\n", + priv->reserved_mtts = + mlx4_alloc_mtt_range(dev, + fls(dev->caps.reserved_mtts - 1)); + if (priv->reserved_mtts < 0) { + mlx4_warn(dev, "MTT table of order %u is too small.\n", mr_table->mtt_buddy.max_order); err = -ENOMEM; goto err_reserve_mtts; } } return 0; err_reserve_mtts: mlx4_buddy_cleanup(&mr_table->mtt_buddy); err_buddy: mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); return err; } void mlx4_cleanup_mr_table(struct mlx4_dev *dev) { - struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_mr_table *mr_table = &priv->mr_table; + if (mlx4_is_slave(dev)) + return; + if (priv->reserved_mtts >= 0) + mlx4_free_mtt_range(dev, priv->reserved_mtts, + fls(dev->caps.reserved_mtts - 1)); mlx4_buddy_cleanup(&mr_table->mtt_buddy); mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); } static inline int mlx4_check_fmr(struct mlx4_fmr *fmr, u64 *page_list, int npages, u64 iova) { int i, page_mask; if (npages > fmr->max_pages) return -EINVAL; page_mask = (1 << fmr->page_shift) - 1; /* We are getting page lists, so va must be page aligned. */ if (iova & page_mask) return -EINVAL; /* Trust the user not to pass misaligned data in page_list */ if (0) for (i = 0; i < npages; ++i) { if (page_list[i] & ~page_mask) return -EINVAL; } if (fmr->maps >= fmr->max_maps) return -EINVAL; return 0; } -int mlx4_map_phys_fmr_fbo(struct mlx4_dev *dev, struct mlx4_fmr *fmr, - u64 *page_list, int npages, u64 iova, u32 fbo, - u32 len, u32 *lkey, u32 *rkey, int same_key) +int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list, + int npages, u64 iova, u32 *lkey, u32 *rkey) { u32 key; int i, err; err = mlx4_check_fmr(fmr, page_list, npages, iova); if (err) return err; ++fmr->maps; key = key_to_hw_index(fmr->mr.key); - if (!same_key) - key += dev->caps.num_mpts; + key += dev->caps.num_mpts; *lkey = *rkey = fmr->mr.key = hw_index_to_key(key); *(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW; /* Make sure MPT status is visible before writing MTT entries */ wmb(); + dma_sync_single_for_cpu(&dev->pdev->dev, fmr->dma_handle, + npages * sizeof(u64), DMA_TO_DEVICE); + for (i = 0; i < npages; ++i) fmr->mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); - dma_sync_single(&dev->pdev->dev, fmr->dma_handle, - npages * sizeof(u64), DMA_TO_DEVICE); + dma_sync_single_for_device(&dev->pdev->dev, fmr->dma_handle, + npages * sizeof(u64), DMA_TO_DEVICE); fmr->mpt->key = cpu_to_be32(key); fmr->mpt->lkey = cpu_to_be32(key); - fmr->mpt->length = cpu_to_be64(len); + fmr->mpt->length = cpu_to_be64(npages * (1ull << fmr->page_shift)); fmr->mpt->start = cpu_to_be64(iova); - fmr->mpt->first_byte_offset = cpu_to_be32(fbo & 0x001fffff); - fmr->mpt->flags2 = (fbo ? MLX4_MPT_FLAG2_FBO_EN : 0); /* Make MTT entries are visible before setting MPT status */ wmb(); *(u8 *) fmr->mpt = MLX4_MPT_STATUS_HW; /* Make sure MPT status is visible before consumer can use FMR */ wmb(); return 0; } -EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr_fbo); - -int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list, - int npages, u64 iova, u32 *lkey, u32 *rkey) -{ - u32 len = npages * (1ull << fmr->page_shift); - - return mlx4_map_phys_fmr_fbo(dev, fmr, page_list, npages, iova, 0, - len, lkey, rkey, 0); -} EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr); int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages, int max_maps, u8 page_shift, struct mlx4_fmr *fmr) { struct mlx4_priv *priv = mlx4_priv(dev); - u64 mtt_seg; - int err = -ENOMEM; + int err = -ENOMEM, ret; + if (max_maps > dev->caps.max_fmr_maps) + return -EINVAL; + if (page_shift < (ffs(dev->caps.page_size_cap) - 1) || page_shift >= 32) return -EINVAL; /* All MTTs must fit in the same page */ if (max_pages * sizeof *fmr->mtts > PAGE_SIZE) return -EINVAL; fmr->page_shift = page_shift; fmr->max_pages = max_pages; fmr->max_maps = max_maps; fmr->maps = 0; err = mlx4_mr_alloc(dev, pd, 0, 0, access, max_pages, page_shift, &fmr->mr); if (err) return err; - mtt_seg = fmr->mr.mtt.first_seg * dev->caps.mtt_entry_sz; - fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table, - fmr->mr.mtt.first_seg, + fmr->mr.mtt.offset, &fmr->dma_handle); + if (!fmr->mtts) { err = -ENOMEM; goto err_free; } return 0; err_free: - mlx4_mr_free(dev, &fmr->mr); + ret = mlx4_mr_free(dev, &fmr->mr); + if (ret) + mlx4_err(dev, "Error deregistering MR. The system may have become unstable."); return err; } EXPORT_SYMBOL_GPL(mlx4_fmr_alloc); -int mlx4_fmr_alloc_reserved(struct mlx4_dev *dev, u32 mridx, - u32 pd, u32 access, int max_pages, - int max_maps, u8 page_shift, struct mlx4_fmr *fmr) -{ - struct mlx4_priv *priv = mlx4_priv(dev); - u64 mtt_seg; - int err = -ENOMEM; - - if (page_shift < (ffs(dev->caps.page_size_cap) - 1) || page_shift >= 32) - return -EINVAL; - - /* All MTTs must fit in the same page */ - if (max_pages * sizeof *fmr->mtts > PAGE_SIZE) - return -EINVAL; - - fmr->page_shift = page_shift; - fmr->max_pages = max_pages; - fmr->max_maps = max_maps; - fmr->maps = 0; - - err = mlx4_mr_alloc_reserved(dev, mridx, pd, 0, 0, access, max_pages, - page_shift, &fmr->mr); - if (err) - return err; - - mtt_seg = fmr->mr.mtt.first_seg * dev->caps.mtt_entry_sz; - - fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table, - fmr->mr.mtt.first_seg, - &fmr->dma_handle); - if (!fmr->mtts) { - err = -ENOMEM; - goto err_free; - } - - return 0; - -err_free: - mlx4_mr_free_reserved(dev, &fmr->mr); - return err; -} -EXPORT_SYMBOL_GPL(mlx4_fmr_alloc_reserved); - int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr) { struct mlx4_priv *priv = mlx4_priv(dev); int err; err = mlx4_mr_enable(dev, &fmr->mr); if (err) return err; fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table, key_to_hw_index(fmr->mr.key), NULL); if (!fmr->mpt) return -ENOMEM; return 0; } EXPORT_SYMBOL_GPL(mlx4_fmr_enable); void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u32 *lkey, u32 *rkey) { + u32 key; + if (!fmr->maps) return; - fmr->maps = 0; + key = key_to_hw_index(fmr->mr.key) & (dev->caps.num_mpts - 1); - *(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW; + *(u8 *)fmr->mpt = MLX4_MPT_STATUS_SW; + + /* Make sure MPT status is visible before changing MPT fields */ + wmb(); + + fmr->mr.key = hw_index_to_key(key); + + fmr->mpt->key = cpu_to_be32(key); + fmr->mpt->lkey = cpu_to_be32(key); + fmr->mpt->length = 0; + fmr->mpt->start = 0; + + /* Make sure MPT data is visible before changing MPT status */ + wmb(); + + *(u8 *)fmr->mpt = MLX4_MPT_STATUS_HW; + + /* Make sure MPT satus is visible */ + wmb(); + + fmr->maps = 0; } EXPORT_SYMBOL_GPL(mlx4_fmr_unmap); int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr) { + int ret; + if (fmr->maps) return -EBUSY; - fmr->mr.enabled = 0; - mlx4_mr_free(dev, &fmr->mr); + ret = mlx4_mr_free(dev, &fmr->mr); + if (ret) + return ret; + fmr->mr.enabled = MLX4_MPT_DISABLED; return 0; } EXPORT_SYMBOL_GPL(mlx4_fmr_free); -int mlx4_fmr_free_reserved(struct mlx4_dev *dev, struct mlx4_fmr *fmr) -{ - if (fmr->maps) - return -EBUSY; - - fmr->mr.enabled = 0; - mlx4_mr_free_reserved(dev, &fmr->mr); - - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_fmr_free_reserved); - int mlx4_SYNC_TPT(struct mlx4_dev *dev) { - return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT, 1000); + return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT, 1000, + MLX4_CMD_NATIVE); } EXPORT_SYMBOL_GPL(mlx4_SYNC_TPT); Index: stable/9/sys/ofed/drivers/net/mlx4/pd.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/pd.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/pd.c (revision 273864) @@ -1,211 +1,302 @@ /* * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include #include +#include #include #include #include "mlx4.h" #include "icm.h" enum { MLX4_NUM_RESERVED_UARS = 8 }; int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn) { struct mlx4_priv *priv = mlx4_priv(dev); *pdn = mlx4_bitmap_alloc(&priv->pd_bitmap); if (*pdn == -1) return -ENOMEM; return 0; } EXPORT_SYMBOL_GPL(mlx4_pd_alloc); void mlx4_pd_free(struct mlx4_dev *dev, u32 pdn) { - mlx4_bitmap_free(&mlx4_priv(dev)->pd_bitmap, pdn); + mlx4_bitmap_free(&mlx4_priv(dev)->pd_bitmap, pdn, MLX4_USE_RR); } EXPORT_SYMBOL_GPL(mlx4_pd_free); +int __mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + *xrcdn = mlx4_bitmap_alloc(&priv->xrcd_bitmap); + if (*xrcdn == -1) + return -ENOMEM; + + return 0; +} + +int mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn) +{ + u64 out_param; + int err; + + if (mlx4_is_mfunc(dev)) { + err = mlx4_cmd_imm(dev, 0, &out_param, + RES_XRCD, RES_OP_RESERVE, + MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + return err; + + *xrcdn = get_param_l(&out_param); + return 0; + } + return __mlx4_xrcd_alloc(dev, xrcdn); +} +EXPORT_SYMBOL_GPL(mlx4_xrcd_alloc); + +void __mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn) +{ + mlx4_bitmap_free(&mlx4_priv(dev)->xrcd_bitmap, xrcdn, MLX4_USE_RR); +} + +void mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn) +{ + u64 in_param = 0; + int err; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, xrcdn); + err = mlx4_cmd(dev, in_param, RES_XRCD, + RES_OP_RESERVE, MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + mlx4_warn(dev, "Failed to release xrcdn %d\n", xrcdn); + } else + __mlx4_xrcd_free(dev, xrcdn); +} +EXPORT_SYMBOL_GPL(mlx4_xrcd_free); + int mlx4_init_pd_table(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); return mlx4_bitmap_init(&priv->pd_bitmap, dev->caps.num_pds, - (1 << 24) - 1, dev->caps.reserved_pds, 0); + (1 << NOT_MASKED_PD_BITS) - 1, + dev->caps.reserved_pds, 0); } void mlx4_cleanup_pd_table(struct mlx4_dev *dev) { mlx4_bitmap_cleanup(&mlx4_priv(dev)->pd_bitmap); } +int mlx4_init_xrcd_table(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + return mlx4_bitmap_init(&priv->xrcd_bitmap, (1 << 16), + (1 << 16) - 1, dev->caps.reserved_xrcds + 1, 0); +} + +void mlx4_cleanup_xrcd_table(struct mlx4_dev *dev) +{ + mlx4_bitmap_cleanup(&mlx4_priv(dev)->xrcd_bitmap); +} + int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar) { + int offset; + uar->index = mlx4_bitmap_alloc(&mlx4_priv(dev)->uar_table.bitmap); if (uar->index == -1) return -ENOMEM; - uar->pfn = (pci_resource_start(dev->pdev, 2) >> PAGE_SHIFT) + uar->index; + if (mlx4_is_slave(dev)) + offset = uar->index % ((int) pci_resource_len(dev->pdev, 2) / + dev->caps.uar_page_size); + else + offset = uar->index; + uar->pfn = (pci_resource_start(dev->pdev, 2) >> PAGE_SHIFT) + offset; uar->map = NULL; - return 0; } EXPORT_SYMBOL_GPL(mlx4_uar_alloc); void mlx4_uar_free(struct mlx4_dev *dev, struct mlx4_uar *uar) { - mlx4_bitmap_free(&mlx4_priv(dev)->uar_table.bitmap, uar->index); + mlx4_bitmap_free(&mlx4_priv(dev)->uar_table.bitmap, uar->index, MLX4_USE_RR); } EXPORT_SYMBOL_GPL(mlx4_uar_free); -int mlx4_bf_alloc(struct mlx4_dev *dev, struct mlx4_bf *bf) +#ifndef CONFIG_PPC +int mlx4_bf_alloc(struct mlx4_dev *dev, struct mlx4_bf *bf, int node) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_uar *uar; int err = 0; int idx; if (!priv->bf_mapping) return -ENOMEM; mutex_lock(&priv->bf_mutex); if (!list_empty(&priv->bf_list)) uar = list_entry(priv->bf_list.next, struct mlx4_uar, bf_list); else { if (mlx4_bitmap_avail(&priv->uar_table.bitmap) < MLX4_NUM_RESERVED_UARS) { err = -ENOMEM; goto out; } - uar = kmalloc(sizeof *uar, GFP_KERNEL); + uar = kmalloc_node(sizeof *uar, GFP_KERNEL, node); if (!uar) { - err = -ENOMEM; - goto out; + uar = kmalloc(sizeof *uar, GFP_KERNEL); + if (!uar) { + err = -ENOMEM; + goto out; + } } err = mlx4_uar_alloc(dev, uar); if (err) goto free_kmalloc; uar->map = ioremap(uar->pfn << PAGE_SHIFT, PAGE_SIZE); if (!uar->map) { err = -ENOMEM; goto free_uar; } uar->bf_map = io_mapping_map_wc(priv->bf_mapping, uar->index << PAGE_SHIFT); if (!uar->bf_map) { err = -ENOMEM; goto unamp_uar; } uar->free_bf_bmap = 0; list_add(&uar->bf_list, &priv->bf_list); } bf->uar = uar; idx = ffz(uar->free_bf_bmap); uar->free_bf_bmap |= 1 << idx; bf->uar = uar; bf->offset = 0; bf->buf_size = dev->caps.bf_reg_size / 2; bf->reg = uar->bf_map + idx * dev->caps.bf_reg_size; if (uar->free_bf_bmap == (1 << dev->caps.bf_regs_per_page) - 1) list_del_init(&uar->bf_list); goto out; unamp_uar: bf->uar = NULL; iounmap(uar->map); free_uar: mlx4_uar_free(dev, uar); free_kmalloc: kfree(uar); out: mutex_unlock(&priv->bf_mutex); return err; } EXPORT_SYMBOL_GPL(mlx4_bf_alloc); void mlx4_bf_free(struct mlx4_dev *dev, struct mlx4_bf *bf) { struct mlx4_priv *priv = mlx4_priv(dev); int idx; if (!bf->uar || !bf->uar->bf_map) return; mutex_lock(&priv->bf_mutex); idx = (bf->reg - bf->uar->bf_map) / dev->caps.bf_reg_size; bf->uar->free_bf_bmap &= ~(1 << idx); if (!bf->uar->free_bf_bmap) { if (!list_empty(&bf->uar->bf_list)) list_del(&bf->uar->bf_list); io_mapping_unmap(bf->uar->bf_map); iounmap(bf->uar->map); mlx4_uar_free(dev, bf->uar); kfree(bf->uar); } else if (list_empty(&bf->uar->bf_list)) list_add(&bf->uar->bf_list, &priv->bf_list); mutex_unlock(&priv->bf_mutex); } EXPORT_SYMBOL_GPL(mlx4_bf_free); +#else +int mlx4_bf_alloc(struct mlx4_dev *dev, struct mlx4_bf *bf, int node) +{ + memset(bf, 0, sizeof *bf); + return -ENOSYS; +} +EXPORT_SYMBOL_GPL(mlx4_bf_alloc); + +void mlx4_bf_free(struct mlx4_dev *dev, struct mlx4_bf *bf) +{ + return; +} +EXPORT_SYMBOL_GPL(mlx4_bf_free); +#endif + int mlx4_init_uar_table(struct mlx4_dev *dev) { if (dev->caps.num_uars <= 128) { mlx4_err(dev, "Only %d UAR pages (need more than 128)\n", dev->caps.num_uars); mlx4_err(dev, "Increase firmware log2_uar_bar_megabytes?\n"); return -ENODEV; } return mlx4_bitmap_init(&mlx4_priv(dev)->uar_table.bitmap, dev->caps.num_uars, dev->caps.num_uars - 1, - max(128, dev->caps.reserved_uars), 0); + dev->caps.reserved_uars, 0); } void mlx4_cleanup_uar_table(struct mlx4_dev *dev) { mlx4_bitmap_cleanup(&mlx4_priv(dev)->uar_table.bitmap); } Index: stable/9/sys/ofed/drivers/net/mlx4/port.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/port.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/port.c (revision 273864) @@ -1,354 +1,1060 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include +#include +#include #include - +#include #include "mlx4.h" +#include "mlx4_stats.h" -int mlx4_ib_set_4k_mtu = 0; -module_param_named(set_4k_mtu, mlx4_ib_set_4k_mtu, int, 0444); -MODULE_PARM_DESC(set_4k_mtu, "attempt to set 4K MTU to all ConnectX ports"); +int mlx4_set_4k_mtu = -1; +module_param_named(set_4k_mtu, mlx4_set_4k_mtu, int, 0444); +MODULE_PARM_DESC(set_4k_mtu, + "(Obsolete) attempt to set 4K MTU to all ConnectX ports"); + + #define MLX4_MAC_VALID (1ull << 63) -#define MLX4_MAC_MASK 0xffffffffffffULL #define MLX4_VLAN_VALID (1u << 31) #define MLX4_VLAN_MASK 0xfff void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table) { int i; mutex_init(&table->mutex); for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { table->entries[i] = 0; table->refs[i] = 0; } table->max = 1 << dev->caps.log_num_macs; table->total = 0; } void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table) { int i; mutex_init(&table->mutex); for (i = 0; i < MLX4_MAX_VLAN_NUM; i++) { table->entries[i] = 0; table->refs[i] = 0; } - table->max = 1 << dev->caps.log_num_vlans; + table->max = (1 << dev->caps.log_num_vlans) - MLX4_VLAN_REGULAR; table->total = 0; } +static int validate_index(struct mlx4_dev *dev, + struct mlx4_mac_table *table, int index) +{ + int err = 0; + + if (index < 0 || index >= table->max || !table->refs[index]) { + mlx4_warn(dev, "No valid Mac entry for the given index\n"); + err = -EINVAL; + } + return err; +} + +static int find_index(struct mlx4_dev *dev, + struct mlx4_mac_table *table, u64 mac) +{ + int i; + + for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { + if ((mac & MLX4_MAC_MASK) == + (MLX4_MAC_MASK & be64_to_cpu(table->entries[i]))) + return i; + } + /* Mac not found */ + return -EINVAL; +} + static int mlx4_set_port_mac_table(struct mlx4_dev *dev, u8 port, __be64 *entries) { struct mlx4_cmd_mailbox *mailbox; u32 in_mod; int err; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); memcpy(mailbox->buf, entries, MLX4_MAC_TABLE_SIZE); in_mod = MLX4_SET_PORT_MAC_TABLE << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B); + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); mlx4_free_cmd_mailbox(dev, mailbox); return err; } -int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *index) +int __mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac) { - struct mlx4_mac_table *table = &mlx4_priv(dev)->port[port].mac_table; + struct mlx4_port_info *info = &mlx4_priv(dev)->port[port]; + struct mlx4_mac_table *table = &info->mac_table; int i, err = 0; int free = -1; - mlx4_dbg(dev, "Registering MAC: 0x%llx\n", (unsigned long long) mac); + mlx4_dbg(dev, "Registering MAC: 0x%llx for port %d\n", + (unsigned long long) mac, port); + mutex_lock(&table->mutex); - for (i = 0; i < MLX4_MAX_MAC_NUM - 1; i++) { + for (i = 0; i < MLX4_MAX_MAC_NUM; i++) { if (free < 0 && !table->refs[i]) { free = i; continue; } - if (mac == (MLX4_MAC_MASK & be64_to_cpu(table->entries[i]))) { - /* MAC already registered, increase refernce count */ - *index = i; + if ((mac == (MLX4_MAC_MASK & be64_to_cpu(table->entries[i]))) && + table->refs[i]) { + /* MAC already registered, Must not have duplicates */ + err = i; ++table->refs[i]; goto out; } } - if (free < 0) { - err = -ENOMEM; - goto out; - } - mlx4_dbg(dev, "Free MAC index is %d\n", free); if (table->total == table->max) { /* No free mac entries */ err = -ENOSPC; goto out; } /* Register new MAC */ - table->refs[free] = 1; table->entries[free] = cpu_to_be64(mac | MLX4_MAC_VALID); err = mlx4_set_port_mac_table(dev, port, table->entries); if (unlikely(err)) { - mlx4_err(dev, "Failed adding MAC: 0x%llx\n", (unsigned long long) mac); - table->refs[free] = 0; + mlx4_err(dev, "Failed adding MAC: 0x%llx\n", + (unsigned long long) mac); table->entries[free] = 0; goto out; } + table->refs[free] = 1; - *index = free; + err = free; ++table->total; out: mutex_unlock(&table->mutex); return err; } +EXPORT_SYMBOL_GPL(__mlx4_register_mac); + +int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac) +{ + u64 out_param = 0; + int err = -EINVAL; + + if (mlx4_is_mfunc(dev)) { + if (!(dev->flags & MLX4_FLAG_OLD_REG_MAC)) { + err = mlx4_cmd_imm(dev, mac, &out_param, + ((u32) port) << 8 | (u32) RES_MAC, + RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + } + if (err && err == -EINVAL && mlx4_is_slave(dev)) { + /* retry using old REG_MAC format */ + set_param_l(&out_param, port); + err = mlx4_cmd_imm(dev, mac, &out_param, RES_MAC, + RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (!err) + dev->flags |= MLX4_FLAG_OLD_REG_MAC; + } + if (err) + return err; + + return get_param_l(&out_param); + } + return __mlx4_register_mac(dev, port, mac); +} EXPORT_SYMBOL_GPL(mlx4_register_mac); -void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, int index) +int mlx4_get_base_qpn(struct mlx4_dev *dev, u8 port) { - struct mlx4_mac_table *table = &mlx4_priv(dev)->port[port].mac_table; + return dev->caps.reserved_qps_base[MLX4_QP_REGION_ETH_ADDR] + + (port - 1) * (1 << dev->caps.log_num_macs); +} +EXPORT_SYMBOL_GPL(mlx4_get_base_qpn); +void __mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, u64 mac) +{ + struct mlx4_port_info *info; + struct mlx4_mac_table *table; + int index; + + if (port < 1 || port > dev->caps.num_ports) { + mlx4_warn(dev, "invalid port number (%d), aborting...\n", port); + return; + } + info = &mlx4_priv(dev)->port[port]; + table = &info->mac_table; mutex_lock(&table->mutex); - if (!table->refs[index]) { - mlx4_warn(dev, "No MAC entry for index %d\n", index); + + index = find_index(dev, table, mac); + + if (validate_index(dev, table, index)) goto out; - } + if (--table->refs[index]) { - mlx4_warn(dev, "Have more references for index %d," - "no need to modify MAC table\n", index); + mlx4_dbg(dev, "Have more references for index %d," + "no need to modify mac table\n", index); goto out; } + table->entries[index] = 0; mlx4_set_port_mac_table(dev, port, table->entries); --table->total; out: mutex_unlock(&table->mutex); } +EXPORT_SYMBOL_GPL(__mlx4_unregister_mac); + +void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, u64 mac) +{ + u64 out_param = 0; + + if (mlx4_is_mfunc(dev)) { + if (!(dev->flags & MLX4_FLAG_OLD_REG_MAC)) { + (void) mlx4_cmd_imm(dev, mac, &out_param, + ((u32) port) << 8 | (u32) RES_MAC, + RES_OP_RESERVE_AND_MAP, MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + } else { + /* use old unregister mac format */ + set_param_l(&out_param, port); + (void) mlx4_cmd_imm(dev, mac, &out_param, RES_MAC, + RES_OP_RESERVE_AND_MAP, MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + } + return; + } + __mlx4_unregister_mac(dev, port, mac); + return; +} EXPORT_SYMBOL_GPL(mlx4_unregister_mac); +int __mlx4_replace_mac(struct mlx4_dev *dev, u8 port, int qpn, u64 new_mac) +{ + struct mlx4_port_info *info = &mlx4_priv(dev)->port[port]; + struct mlx4_mac_table *table = &info->mac_table; + int index = qpn - info->base_qpn; + int err = 0; + + /* CX1 doesn't support multi-functions */ + mutex_lock(&table->mutex); + + err = validate_index(dev, table, index); + if (err) + goto out; + + table->entries[index] = cpu_to_be64(new_mac | MLX4_MAC_VALID); + + err = mlx4_set_port_mac_table(dev, port, table->entries); + if (unlikely(err)) { + mlx4_err(dev, "Failed adding MAC: 0x%llx\n", + (unsigned long long) new_mac); + table->entries[index] = 0; + } +out: + mutex_unlock(&table->mutex); + return err; +} +EXPORT_SYMBOL_GPL(__mlx4_replace_mac); + static int mlx4_set_port_vlan_table(struct mlx4_dev *dev, u8 port, __be32 *entries) { struct mlx4_cmd_mailbox *mailbox; u32 in_mod; int err; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); memcpy(mailbox->buf, entries, MLX4_VLAN_TABLE_SIZE); in_mod = MLX4_SET_PORT_VLAN_TABLE << 8 | port; err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B); + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); mlx4_free_cmd_mailbox(dev, mailbox); return err; } int mlx4_find_cached_vlan(struct mlx4_dev *dev, u8 port, u16 vid, int *idx) { struct mlx4_vlan_table *table = &mlx4_priv(dev)->port[port].vlan_table; int i; for (i = 0; i < MLX4_MAX_VLAN_NUM; ++i) { if (table->refs[i] && (vid == (MLX4_VLAN_MASK & be32_to_cpu(table->entries[i])))) { - /* Vlan already registered, increase refernce count */ + /* VLAN already registered, increase reference count */ *idx = i; return 0; } } return -ENOENT; } EXPORT_SYMBOL_GPL(mlx4_find_cached_vlan); -int mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index) +int __mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, + int *index) { struct mlx4_vlan_table *table = &mlx4_priv(dev)->port[port].vlan_table; int i, err = 0; int free = -1; mutex_lock(&table->mutex); + + if (table->total == table->max) { + /* No free vlan entries */ + err = -ENOSPC; + goto out; + } + for (i = MLX4_VLAN_REGULAR; i < MLX4_MAX_VLAN_NUM; i++) { if (free < 0 && (table->refs[i] == 0)) { free = i; continue; } if (table->refs[i] && (vlan == (MLX4_VLAN_MASK & be32_to_cpu(table->entries[i])))) { - /* Vlan already registered, increase refernce count */ + /* Vlan already registered, increase references count */ *index = i; ++table->refs[i]; goto out; } } if (free < 0) { err = -ENOMEM; goto out; } - if (table->total == table->max) { - /* No free vlan entries */ - err = -ENOSPC; - goto out; - } - - /* Register new MAC */ + /* Register new VLAN */ table->refs[free] = 1; table->entries[free] = cpu_to_be32(vlan | MLX4_VLAN_VALID); err = mlx4_set_port_vlan_table(dev, port, table->entries); if (unlikely(err)) { mlx4_warn(dev, "Failed adding vlan: %u\n", vlan); table->refs[free] = 0; table->entries[free] = 0; goto out; } *index = free; ++table->total; out: mutex_unlock(&table->mutex); return err; } + +int mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index) +{ + u64 out_param = 0; + int err; + + if (vlan > 4095) + return -EINVAL; + + if (mlx4_is_mfunc(dev)) { + err = mlx4_cmd_imm(dev, vlan, &out_param, + ((u32) port) << 8 | (u32) RES_VLAN, + RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (!err) + *index = get_param_l(&out_param); + + return err; + } + return __mlx4_register_vlan(dev, port, vlan, index); +} EXPORT_SYMBOL_GPL(mlx4_register_vlan); -void mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, int index) +void __mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, u16 vlan) { struct mlx4_vlan_table *table = &mlx4_priv(dev)->port[port].vlan_table; + int index; - if (index < MLX4_VLAN_REGULAR) { - mlx4_warn(dev, "Trying to free special vlan index %d\n", index); - return; + mutex_lock(&table->mutex); + if (mlx4_find_cached_vlan(dev, port, vlan, &index)) { + mlx4_warn(dev, "vlan 0x%x is not in the vlan table\n", vlan); + goto out; } - mutex_lock(&table->mutex); - if (!table->refs[index]) { - mlx4_warn(dev, "No vlan entry for index %d\n", index); + if (index < MLX4_VLAN_REGULAR) { + mlx4_warn(dev, "Trying to free special vlan index %d\n", index); goto out; } + if (--table->refs[index]) { - mlx4_dbg(dev, "Have more references for index %d," - "no need to modify vlan table\n", index); + mlx4_dbg(dev, "Have %d more references for index %d, " + "no need to modify vlan table\n", table->refs[index], + index); goto out; } table->entries[index] = 0; mlx4_set_port_vlan_table(dev, port, table->entries); --table->total; out: mutex_unlock(&table->mutex); } + +void mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, u16 vlan) +{ + u64 out_param = 0; + + if (mlx4_is_mfunc(dev)) { + (void) mlx4_cmd_imm(dev, vlan, &out_param, + ((u32) port) << 8 | (u32) RES_VLAN, + RES_OP_RESERVE_AND_MAP, + MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + return; + } + __mlx4_unregister_vlan(dev, port, vlan); +} EXPORT_SYMBOL_GPL(mlx4_unregister_vlan); int mlx4_get_port_ib_caps(struct mlx4_dev *dev, u8 port, __be32 *caps) { struct mlx4_cmd_mailbox *inmailbox, *outmailbox; u8 *inbuf, *outbuf; int err; inmailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(inmailbox)) return PTR_ERR(inmailbox); outmailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(outmailbox)) { mlx4_free_cmd_mailbox(dev, inmailbox); return PTR_ERR(outmailbox); } inbuf = inmailbox->buf; outbuf = outmailbox->buf; memset(inbuf, 0, 256); memset(outbuf, 0, 256); inbuf[0] = 1; inbuf[1] = 1; inbuf[2] = 1; inbuf[3] = 1; *(__be16 *) (&inbuf[16]) = cpu_to_be16(0x0015); *(__be32 *) (&inbuf[20]) = cpu_to_be32(port); err = mlx4_cmd_box(dev, inmailbox->dma, outmailbox->dma, port, 3, - MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C); + MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_NATIVE); if (!err) *caps = *(__be32 *) (outbuf + 84); mlx4_free_cmd_mailbox(dev, inmailbox); mlx4_free_cmd_mailbox(dev, outmailbox); return err; } +static struct mlx4_roce_gid_entry zgid_entry; -int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port) +int mlx4_get_slave_num_gids(struct mlx4_dev *dev, int slave) { - struct mlx4_cmd_mailbox *mailbox; + if (slave == 0) + return MLX4_ROCE_PF_GIDS; + if (slave <= ((MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS) % dev->num_vfs)) + return ((MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS) / dev->num_vfs) + 1; + return (MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS) / dev->num_vfs; +} + +int mlx4_get_base_gid_ix(struct mlx4_dev *dev, int slave) +{ + int gids; + int vfs; + + gids = MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS; + vfs = dev->num_vfs; + + if (slave == 0) + return 0; + if (slave <= gids % vfs) + return MLX4_ROCE_PF_GIDS + ((gids / vfs) + 1) * (slave - 1); + + return MLX4_ROCE_PF_GIDS + (gids % vfs) + ((gids / vfs) * (slave - 1)); +} + +static int mlx4_common_set_port(struct mlx4_dev *dev, int slave, u32 in_mod, + u8 op_mod, struct mlx4_cmd_mailbox *inbox) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_port_info *port_info; + struct mlx4_mfunc_master_ctx *master = &priv->mfunc.master; + struct mlx4_slave_state *slave_st = &master->slave_state[slave]; + struct mlx4_set_port_rqp_calc_context *qpn_context; + struct mlx4_set_port_general_context *gen_context; + struct mlx4_roce_gid_entry *gid_entry_tbl, *gid_entry_mbox, *gid_entry_mb1; + int reset_qkey_viols; + int port; + int is_eth; + int num_gids; + int base; + u32 in_modifier; + u32 promisc; + u16 mtu, prev_mtu; int err; + int i, j; + int offset; + __be32 agg_cap_mask; + __be32 slave_cap_mask; + __be32 new_cap_mask; - if (dev->caps.port_type[port] != MLX4_PORT_TYPE_IB) + port = in_mod & 0xff; + in_modifier = (in_mod >> 8) & 0xff; + is_eth = op_mod; + port_info = &priv->port[port]; + + if (op_mod > 1) + return -EINVAL; + + /* Slaves cannot perform SET_PORT operations except changing MTU */ + if (is_eth) { + if (slave != dev->caps.function && + in_modifier != MLX4_SET_PORT_GENERAL && + in_modifier != MLX4_SET_PORT_GID_TABLE) { + mlx4_warn(dev, "denying SET_PORT for slave:%d," + "port %d, config_select 0x%x\n", + slave, port, in_modifier); + return -EINVAL; + } + switch (in_modifier) { + case MLX4_SET_PORT_RQP_CALC: + qpn_context = inbox->buf; + qpn_context->base_qpn = + cpu_to_be32(port_info->base_qpn); + qpn_context->n_mac = 0x7; + promisc = be32_to_cpu(qpn_context->promisc) >> + SET_PORT_PROMISC_SHIFT; + qpn_context->promisc = cpu_to_be32( + promisc << SET_PORT_PROMISC_SHIFT | + port_info->base_qpn); + promisc = be32_to_cpu(qpn_context->mcast) >> + SET_PORT_MC_PROMISC_SHIFT; + qpn_context->mcast = cpu_to_be32( + promisc << SET_PORT_MC_PROMISC_SHIFT | + port_info->base_qpn); + break; + case MLX4_SET_PORT_GENERAL: + gen_context = inbox->buf; + /* Mtu is configured as the max MTU among all the + * the functions on the port. */ + mtu = be16_to_cpu(gen_context->mtu); + mtu = min_t(int, mtu, dev->caps.eth_mtu_cap[port] + + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN); + prev_mtu = slave_st->mtu[port]; + slave_st->mtu[port] = mtu; + if (mtu > master->max_mtu[port]) + master->max_mtu[port] = mtu; + if (mtu < prev_mtu && prev_mtu == + master->max_mtu[port]) { + slave_st->mtu[port] = mtu; + master->max_mtu[port] = mtu; + for (i = 0; i < dev->num_slaves; i++) { + master->max_mtu[port] = + max(master->max_mtu[port], + master->slave_state[i].mtu[port]); + } + } + + gen_context->mtu = cpu_to_be16(master->max_mtu[port]); + break; + case MLX4_SET_PORT_GID_TABLE: + /* change to MULTIPLE entries: number of guest's gids + * need a FOR-loop here over number of gids the guest has. + * 1. Check no duplicates in gids passed by slave + */ + num_gids = mlx4_get_slave_num_gids(dev, slave); + base = mlx4_get_base_gid_ix(dev, slave); + gid_entry_mbox = (struct mlx4_roce_gid_entry *) (inbox->buf); + for (i = 0; i < num_gids; gid_entry_mbox++, i++) { + if (!memcmp(gid_entry_mbox->raw, zgid_entry.raw, + sizeof(zgid_entry))) + continue; + gid_entry_mb1 = gid_entry_mbox + 1; + for (j = i + 1; j < num_gids; gid_entry_mb1++, j++) { + if (!memcmp(gid_entry_mb1->raw, + zgid_entry.raw, sizeof(zgid_entry))) + continue; + if (!memcmp(gid_entry_mb1->raw, gid_entry_mbox->raw, + sizeof(gid_entry_mbox->raw))) { + /* found duplicate */ + return -EINVAL; + } + } + } + + /* 2. Check that do not have duplicates in OTHER + * entries in the port GID table + */ + for (i = 0; i < MLX4_ROCE_MAX_GIDS; i++) { + if (i >= base && i < base + num_gids) + continue; /* don't compare to slave's current gids */ + gid_entry_tbl = &priv->roce_gids[port - 1][i]; + if (!memcmp(gid_entry_tbl->raw, zgid_entry.raw, sizeof(zgid_entry))) + continue; + gid_entry_mbox = (struct mlx4_roce_gid_entry *) (inbox->buf); + for (j = 0; j < num_gids; gid_entry_mbox++, j++) { + if (!memcmp(gid_entry_mbox->raw, zgid_entry.raw, + sizeof(zgid_entry))) + continue; + if (!memcmp(gid_entry_mbox->raw, gid_entry_tbl->raw, + sizeof(gid_entry_tbl->raw))) { + /* found duplicate */ + mlx4_warn(dev, "requested gid entry for slave:%d " + "is a duplicate of gid at index %d\n", + slave, i); + return -EINVAL; + } + } + } + + /* insert slave GIDs with memcpy, starting at slave's base index */ + gid_entry_mbox = (struct mlx4_roce_gid_entry *) (inbox->buf); + for (i = 0, offset = base; i < num_gids; gid_entry_mbox++, offset++, i++) + memcpy(priv->roce_gids[port - 1][offset].raw, gid_entry_mbox->raw, 16); + + /* Now, copy roce port gids table to current mailbox for passing to FW */ + gid_entry_mbox = (struct mlx4_roce_gid_entry *) (inbox->buf); + for (i = 0; i < MLX4_ROCE_MAX_GIDS; gid_entry_mbox++, i++) + memcpy(gid_entry_mbox->raw, priv->roce_gids[port - 1][i].raw, 16); + + break; + } + return mlx4_cmd(dev, inbox->dma, in_mod & 0xffff, op_mod, + MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + } + + /* For IB, we only consider: + * - The capability mask, which is set to the aggregate of all + * slave function capabilities + * - The QKey violatin counter - reset according to each request. + */ + + if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { + reset_qkey_viols = (*(u8 *) inbox->buf) & 0x40; + new_cap_mask = ((__be32 *) inbox->buf)[2]; + } else { + reset_qkey_viols = ((u8 *) inbox->buf)[3] & 0x1; + new_cap_mask = ((__be32 *) inbox->buf)[1]; + } + + /* slave may not set the IS_SM capability for the port */ + if (slave != mlx4_master_func_num(dev) && + (be32_to_cpu(new_cap_mask) & MLX4_PORT_CAP_IS_SM)) + return -EINVAL; + + /* No DEV_MGMT in multifunc mode */ + if (mlx4_is_mfunc(dev) && + (be32_to_cpu(new_cap_mask) & MLX4_PORT_CAP_DEV_MGMT_SUP)) + return -EINVAL; + + agg_cap_mask = 0; + slave_cap_mask = + priv->mfunc.master.slave_state[slave].ib_cap_mask[port]; + priv->mfunc.master.slave_state[slave].ib_cap_mask[port] = new_cap_mask; + for (i = 0; i < dev->num_slaves; i++) + agg_cap_mask |= + priv->mfunc.master.slave_state[i].ib_cap_mask[port]; + + /* only clear mailbox for guests. Master may be setting + * MTU or PKEY table size + */ + if (slave != dev->caps.function) + memset(inbox->buf, 0, 256); + if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) { + *(u8 *) inbox->buf |= !!reset_qkey_viols << 6; + ((__be32 *) inbox->buf)[2] = agg_cap_mask; + } else { + ((u8 *) inbox->buf)[3] |= !!reset_qkey_viols; + ((__be32 *) inbox->buf)[1] = agg_cap_mask; + } + + err = mlx4_cmd(dev, inbox->dma, port, is_eth, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + if (err) + priv->mfunc.master.slave_state[slave].ib_cap_mask[port] = + slave_cap_mask; + return err; +} + +int mlx4_SET_PORT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + return mlx4_common_set_port(dev, slave, vhcr->in_modifier, + vhcr->op_modifier, inbox); +} + +/* bit locations for set port command with zero op modifier */ +enum { + MLX4_SET_PORT_VL_CAP = 4, /* bits 7:4 */ + MLX4_SET_PORT_MTU_CAP = 12, /* bits 15:12 */ + MLX4_CHANGE_PORT_PKEY_TBL_SZ = 20, + MLX4_CHANGE_PORT_VL_CAP = 21, + MLX4_CHANGE_PORT_MTU_CAP = 22, +}; + +int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port, int pkey_tbl_sz) +{ + struct mlx4_cmd_mailbox *mailbox; + int err = -EINVAL, vl_cap, pkey_tbl_flag = 0; + u32 in_mod; + + if (dev->caps.port_type[port] == MLX4_PORT_TYPE_NONE) return 0; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); memset(mailbox->buf, 0, 256); - if (mlx4_ib_set_4k_mtu) - ((__be32 *) mailbox->buf)[0] |= cpu_to_be32((1 << 22) | (1 << 21) | (5 << 12) | (2 << 4)); + if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) { + in_mod = MLX4_SET_PORT_GENERAL << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, + MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); + } else { + ((__be32 *) mailbox->buf)[1] = dev->caps.ib_port_def_cap[port]; - ((__be32 *) mailbox->buf)[1] = dev->caps.ib_port_def_cap[port]; - err = mlx4_cmd(dev, mailbox->dma, port, 0, MLX4_CMD_SET_PORT, - MLX4_CMD_TIME_CLASS_B); + if (pkey_tbl_sz >= 0 && mlx4_is_master(dev)) { + pkey_tbl_flag = 1; + ((__be16 *) mailbox->buf)[20] = cpu_to_be16(pkey_tbl_sz); + } + /* IB VL CAP enum isn't used by the firmware, just numerical values */ + for (vl_cap = dev->caps.vl_cap[port]; + vl_cap >= 1; vl_cap >>= 1) { + ((__be32 *) mailbox->buf)[0] = cpu_to_be32( + (1 << MLX4_CHANGE_PORT_MTU_CAP) | + (1 << MLX4_CHANGE_PORT_VL_CAP) | + (pkey_tbl_flag << MLX4_CHANGE_PORT_PKEY_TBL_SZ) | + (dev->caps.port_ib_mtu[port] << MLX4_SET_PORT_MTU_CAP) | + (vl_cap << MLX4_SET_PORT_VL_CAP)); + err = mlx4_cmd(dev, mailbox->dma, port, 0, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); + if (err != -ENOMEM) + break; + } + } + mlx4_free_cmd_mailbox(dev, mailbox); return err; } + +int mlx4_SET_PORT_general(struct mlx4_dev *dev, u8 port, int mtu, + u8 pptx, u8 pfctx, u8 pprx, u8 pfcrx) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_port_general_context *context; + int err; + u32 in_mod; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + context = mailbox->buf; + memset(context, 0, sizeof *context); + + context->flags = SET_PORT_GEN_ALL_VALID; + context->mtu = cpu_to_be16(mtu); + context->pptx = (pptx * (!pfctx)) << 7; + context->pfctx = pfctx; + context->pprx = (pprx * (!pfcrx)) << 7; + context->pfcrx = pfcrx; + + in_mod = MLX4_SET_PORT_GENERAL << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_SET_PORT_general); + +int mlx4_SET_PORT_qpn_calc(struct mlx4_dev *dev, u8 port, u32 base_qpn, + u8 promisc) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_port_rqp_calc_context *context; + int err; + u32 in_mod; + u32 m_promisc = (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) ? + MCAST_DIRECT : MCAST_DEFAULT; + + if (dev->caps.steering_mode != MLX4_STEERING_MODE_A0) + return 0; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + context = mailbox->buf; + memset(context, 0, sizeof *context); + + context->base_qpn = cpu_to_be32(base_qpn); + context->n_mac = dev->caps.log_num_macs; + context->promisc = cpu_to_be32(promisc << SET_PORT_PROMISC_SHIFT | + base_qpn); + context->mcast = cpu_to_be32(m_promisc << SET_PORT_MC_PROMISC_SHIFT | + base_qpn); + context->intra_no_vlan = 0; + context->no_vlan = MLX4_NO_VLAN_IDX; + context->intra_vlan_miss = 0; + context->vlan_miss = MLX4_VLAN_MISS_IDX; + + in_mod = MLX4_SET_PORT_RQP_CALC << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_SET_PORT_qpn_calc); + +int mlx4_SET_PORT_PRIO2TC(struct mlx4_dev *dev, u8 port, u8 *prio2tc) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_port_prio2tc_context *context; + int err; + u32 in_mod; + int i; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + context = mailbox->buf; + memset(context, 0, sizeof *context); + + for (i = 0; i < MLX4_NUM_UP; i += 2) + context->prio2tc[i >> 1] = prio2tc[i] << 4 | prio2tc[i + 1]; + + in_mod = MLX4_SET_PORT_PRIO2TC << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_SET_PORT_PRIO2TC); + +int mlx4_SET_PORT_SCHEDULER(struct mlx4_dev *dev, u8 port, u8 *tc_tx_bw, + u8 *pg, u16 *ratelimit) +{ + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_set_port_scheduler_context *context; + int err; + u32 in_mod; + int i; + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + context = mailbox->buf; + memset(context, 0, sizeof *context); + + for (i = 0; i < MLX4_NUM_TC; i++) { + struct mlx4_port_scheduler_tc_cfg_be *tc = &context->tc[i]; + u16 r; + if (ratelimit && ratelimit[i]) { + if (ratelimit[i] <= MLX4_MAX_100M_UNITS_VAL) { + r = ratelimit[i]; + tc->max_bw_units = + htons(MLX4_RATELIMIT_100M_UNITS); + } else { + r = ratelimit[i]/10; + tc->max_bw_units = + htons(MLX4_RATELIMIT_1G_UNITS); + } + tc->max_bw_value = htons(r); + } else { + tc->max_bw_value = htons(MLX4_RATELIMIT_DEFAULT); + tc->max_bw_units = htons(MLX4_RATELIMIT_1G_UNITS); + } + + tc->pg = htons(pg[i]); + tc->bw_precentage = htons(tc_tx_bw[i]); + } + + in_mod = MLX4_SET_PORT_SCHEDULER << 8 | port; + err = mlx4_cmd(dev, mailbox->dma, in_mod, 1, MLX4_CMD_SET_PORT, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); + + mlx4_free_cmd_mailbox(dev, mailbox); + return err; +} +EXPORT_SYMBOL(mlx4_SET_PORT_SCHEDULER); + +int mlx4_SET_MCAST_FLTR_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err = 0; + + return err; +} + +int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, + u64 mac, u64 clear, u8 mode) +{ + return mlx4_cmd(dev, (mac | (clear << 63)), port, mode, + MLX4_CMD_SET_MCAST_FLTR, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); +} +EXPORT_SYMBOL(mlx4_SET_MCAST_FLTR); + +int mlx4_SET_VLAN_FLTR_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err = 0; + + return err; +} + +int mlx4_DUMP_ETH_STATS_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + return 0; +} + +void mlx4_set_stats_bitmap(struct mlx4_dev *dev, unsigned long *stats_bitmap) +{ + int last_i = 0; + + bitmap_zero(stats_bitmap, NUM_ALL_STATS); + + if (mlx4_is_slave(dev)) { + last_i = dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN ? + NUM_PKT_STATS + NUM_FLOW_STATS : NUM_PKT_STATS; + } else { + bitmap_set(stats_bitmap, last_i, NUM_PKT_STATS); + last_i = NUM_PKT_STATS; + + if (dev->caps.flags2 & + MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) { + bitmap_set(stats_bitmap, last_i, NUM_FLOW_STATS); + last_i += NUM_FLOW_STATS; + } + } + + if (mlx4_is_slave(dev)) + bitmap_set(stats_bitmap, last_i, NUM_VF_STATS); + last_i += NUM_VF_STATS; + + if (mlx4_is_master(dev)) + bitmap_set(stats_bitmap, last_i, NUM_VPORT_STATS); + last_i += NUM_VPORT_STATS; + + bitmap_set(stats_bitmap, last_i, NUM_PORT_STATS); +} +EXPORT_SYMBOL(mlx4_set_stats_bitmap); + +int mlx4_get_slave_from_roce_gid(struct mlx4_dev *dev, int port, u8 *gid, int *slave_id) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int i, found_ix = -1; + int vf_gids = MLX4_ROCE_MAX_GIDS - MLX4_ROCE_PF_GIDS; + + if (!mlx4_is_mfunc(dev)) + return -EINVAL; + + for (i = 0; i < MLX4_ROCE_MAX_GIDS; i++) { + if (!memcmp(priv->roce_gids[port - 1][i].raw, gid, 16)) { + found_ix = i; + break; + } + } + + if (found_ix >= 0) { + if (found_ix < MLX4_ROCE_PF_GIDS) + *slave_id = 0; + else if (found_ix < MLX4_ROCE_PF_GIDS + (vf_gids % dev->num_vfs) * + (vf_gids / dev->num_vfs + 1)) + *slave_id = ((found_ix - MLX4_ROCE_PF_GIDS) / + (vf_gids / dev->num_vfs + 1)) + 1; + else + *slave_id = + ((found_ix - MLX4_ROCE_PF_GIDS - + ((vf_gids % dev->num_vfs) * ((vf_gids / dev->num_vfs + 1)))) / + (vf_gids / dev->num_vfs)) + vf_gids % dev->num_vfs + 1; + } + + return (found_ix >= 0) ? 0 : -EINVAL; +} +EXPORT_SYMBOL(mlx4_get_slave_from_roce_gid); + +int mlx4_get_roce_gid_from_slave(struct mlx4_dev *dev, int port, int slave_id, u8 *gid) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + if (!mlx4_is_master(dev)) + return -EINVAL; + + memcpy(gid, priv->roce_gids[port - 1][slave_id].raw, 16); + return 0; +} +EXPORT_SYMBOL(mlx4_get_roce_gid_from_slave); + Index: stable/9/sys/ofed/drivers/net/mlx4/profile.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/profile.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/profile.c (revision 273864) @@ -1,240 +1,249 @@ /* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include +#include #include "mlx4.h" #include "fw.h" enum { MLX4_RES_QP, MLX4_RES_RDMARC, MLX4_RES_ALTC, MLX4_RES_AUXC, MLX4_RES_SRQ, MLX4_RES_CQ, MLX4_RES_EQ, MLX4_RES_DMPT, MLX4_RES_CMPT, MLX4_RES_MTT, MLX4_RES_MCG, MLX4_RES_NUM }; static const char *res_name[] = { [MLX4_RES_QP] = "QP", [MLX4_RES_RDMARC] = "RDMARC", [MLX4_RES_ALTC] = "ALTC", [MLX4_RES_AUXC] = "AUXC", [MLX4_RES_SRQ] = "SRQ", [MLX4_RES_CQ] = "CQ", [MLX4_RES_EQ] = "EQ", [MLX4_RES_DMPT] = "DMPT", [MLX4_RES_CMPT] = "CMPT", [MLX4_RES_MTT] = "MTT", [MLX4_RES_MCG] = "MCG", }; u64 mlx4_make_profile(struct mlx4_dev *dev, struct mlx4_profile *request, struct mlx4_dev_cap *dev_cap, struct mlx4_init_hca_param *init_hca) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_resource { u64 size; u64 start; int type; - int num; + u64 num; int log_num; }; u64 total_size = 0; struct mlx4_resource *profile; struct mlx4_resource tmp; int i, j; - profile = kzalloc(MLX4_RES_NUM * sizeof *profile, GFP_KERNEL); + profile = kcalloc(MLX4_RES_NUM, sizeof(*profile), GFP_KERNEL); if (!profile) return -ENOMEM; profile[MLX4_RES_QP].size = dev_cap->qpc_entry_sz; profile[MLX4_RES_RDMARC].size = dev_cap->rdmarc_entry_sz; profile[MLX4_RES_ALTC].size = dev_cap->altc_entry_sz; profile[MLX4_RES_AUXC].size = dev_cap->aux_entry_sz; profile[MLX4_RES_SRQ].size = dev_cap->srq_entry_sz; profile[MLX4_RES_CQ].size = dev_cap->cqc_entry_sz; profile[MLX4_RES_EQ].size = dev_cap->eqc_entry_sz; profile[MLX4_RES_DMPT].size = dev_cap->dmpt_entry_sz; profile[MLX4_RES_CMPT].size = dev_cap->cmpt_entry_sz; - profile[MLX4_RES_MTT].size = dev->caps.mtts_per_seg * dev_cap->mtt_entry_sz; - profile[MLX4_RES_MCG].size = MLX4_MGM_ENTRY_SIZE; + profile[MLX4_RES_MTT].size = dev_cap->mtt_entry_sz; + profile[MLX4_RES_MCG].size = mlx4_get_mgm_entry_size(dev); profile[MLX4_RES_QP].num = request->num_qp; profile[MLX4_RES_RDMARC].num = request->num_qp * request->rdmarc_per_qp; profile[MLX4_RES_ALTC].num = request->num_qp; profile[MLX4_RES_AUXC].num = request->num_qp; profile[MLX4_RES_SRQ].num = request->num_srq; profile[MLX4_RES_CQ].num = request->num_cq; - profile[MLX4_RES_EQ].num = min_t(unsigned, dev_cap->max_eqs, - dev_cap->reserved_eqs + - num_possible_cpus() + 1); + profile[MLX4_RES_EQ].num = mlx4_is_mfunc(dev) ? + dev->phys_caps.num_phys_eqs : + min_t(unsigned, dev_cap->max_eqs, MAX_MSIX); profile[MLX4_RES_DMPT].num = request->num_mpt; profile[MLX4_RES_CMPT].num = MLX4_NUM_CMPTS; - profile[MLX4_RES_MTT].num = request->num_mtt; + profile[MLX4_RES_MTT].num = ((u64)request->num_mtt_segs) * + (1 << log_mtts_per_seg); profile[MLX4_RES_MCG].num = request->num_mcg; for (i = 0; i < MLX4_RES_NUM; ++i) { profile[i].type = i; profile[i].num = roundup_pow_of_two(profile[i].num); profile[i].log_num = ilog2(profile[i].num); profile[i].size *= profile[i].num; profile[i].size = max(profile[i].size, (u64) PAGE_SIZE); } /* * Sort the resources in decreasing order of size. Since they * all have sizes that are powers of 2, we'll be able to keep * resources aligned to their size and pack them without gaps * using the sorted order. */ for (i = MLX4_RES_NUM; i > 0; --i) for (j = 1; j < i; ++j) { if (profile[j].size > profile[j - 1].size) { tmp = profile[j]; profile[j] = profile[j - 1]; profile[j - 1] = tmp; } } for (i = 0; i < MLX4_RES_NUM; ++i) { if (profile[i].size) { profile[i].start = total_size; total_size += profile[i].size; } if (total_size > dev_cap->max_icm_sz) { mlx4_err(dev, "Profile requires 0x%llx bytes; " "won't fit in 0x%llx bytes of context memory.\n", (unsigned long long) total_size, (unsigned long long) dev_cap->max_icm_sz); kfree(profile); return -ENOMEM; } if (profile[i].size) mlx4_dbg(dev, " profile[%2d] (%6s): 2^%02d entries @ 0x%10llx, " "size 0x%10llx\n", i, res_name[profile[i].type], profile[i].log_num, (unsigned long long) profile[i].start, (unsigned long long) profile[i].size); } mlx4_dbg(dev, "HCA context memory: reserving %d KB\n", (int) (total_size >> 10)); for (i = 0; i < MLX4_RES_NUM; ++i) { switch (profile[i].type) { case MLX4_RES_QP: dev->caps.num_qps = profile[i].num; init_hca->qpc_base = profile[i].start; init_hca->log_num_qps = profile[i].log_num; break; case MLX4_RES_RDMARC: for (priv->qp_table.rdmarc_shift = 0; request->num_qp << priv->qp_table.rdmarc_shift < profile[i].num; ++priv->qp_table.rdmarc_shift) ; /* nothing */ dev->caps.max_qp_dest_rdma = 1 << priv->qp_table.rdmarc_shift; priv->qp_table.rdmarc_base = (u32) profile[i].start; init_hca->rdmarc_base = profile[i].start; init_hca->log_rd_per_qp = priv->qp_table.rdmarc_shift; break; case MLX4_RES_ALTC: init_hca->altc_base = profile[i].start; break; case MLX4_RES_AUXC: init_hca->auxc_base = profile[i].start; break; case MLX4_RES_SRQ: dev->caps.num_srqs = profile[i].num; init_hca->srqc_base = profile[i].start; init_hca->log_num_srqs = profile[i].log_num; break; case MLX4_RES_CQ: dev->caps.num_cqs = profile[i].num; init_hca->cqc_base = profile[i].start; init_hca->log_num_cqs = profile[i].log_num; break; case MLX4_RES_EQ: - dev->caps.num_eqs = profile[i].num; + dev->caps.num_eqs = roundup_pow_of_two(min_t(unsigned, dev_cap->max_eqs, + MAX_MSIX)); init_hca->eqc_base = profile[i].start; - init_hca->log_num_eqs = profile[i].log_num; + init_hca->log_num_eqs = ilog2(dev->caps.num_eqs); break; case MLX4_RES_DMPT: dev->caps.num_mpts = profile[i].num; priv->mr_table.mpt_base = profile[i].start; init_hca->dmpt_base = profile[i].start; init_hca->log_mpt_sz = profile[i].log_num; break; case MLX4_RES_CMPT: init_hca->cmpt_base = profile[i].start; break; case MLX4_RES_MTT: - dev->caps.num_mtt_segs = profile[i].num; + dev->caps.num_mtts = profile[i].num; priv->mr_table.mtt_base = profile[i].start; init_hca->mtt_base = profile[i].start; break; case MLX4_RES_MCG: - dev->caps.num_mgms = profile[i].num >> 1; - dev->caps.num_amgms = profile[i].num >> 1; init_hca->mc_base = profile[i].start; - init_hca->log_mc_entry_sz = ilog2(MLX4_MGM_ENTRY_SIZE); + init_hca->log_mc_entry_sz = + ilog2(mlx4_get_mgm_entry_size(dev)); init_hca->log_mc_table_sz = profile[i].log_num; - init_hca->log_mc_hash_sz = profile[i].log_num - 1; + if (dev->caps.steering_mode == + MLX4_STEERING_MODE_DEVICE_MANAGED) { + dev->caps.num_mgms = profile[i].num; + } else { + init_hca->log_mc_hash_sz = + profile[i].log_num - 1; + dev->caps.num_mgms = profile[i].num >> 1; + dev->caps.num_amgms = profile[i].num >> 1; + } break; default: break; } } /* * PDs don't take any HCA memory, but we assign them as part * of the HCA profile anyway. */ dev->caps.num_pds = MLX4_NUM_PDS; kfree(profile); return total_size; } Index: stable/9/sys/ofed/drivers/net/mlx4/qp.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/qp.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/qp.c (revision 273864) @@ -1,410 +1,615 @@ /* * Copyright (c) 2004 Topspin Communications. All rights reserved. * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * Copyright (c) 2004 Voltaire, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include +#include +#include +#include #include #include #include "mlx4.h" #include "icm.h" +/* + * QP to support BF should have bits 6,7 cleared + */ +#define MLX4_BF_QP_SKIP_MASK 0xc0 +#define MLX4_MAX_BF_QP_RANGE 0x40 + void mlx4_qp_event(struct mlx4_dev *dev, u32 qpn, int event_type) { struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; struct mlx4_qp *qp; spin_lock(&qp_table->lock); qp = __mlx4_qp_lookup(dev, qpn); if (qp) atomic_inc(&qp->refcount); spin_unlock(&qp_table->lock); if (!qp) { - mlx4_warn(dev, "Async event for bogus QP %08x\n", qpn); + mlx4_dbg(dev, "Async event for none existent QP %08x\n", qpn); return; } qp->event(qp, event_type); if (atomic_dec_and_test(&qp->refcount)) complete(&qp->free); } -int mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt, - enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state, - struct mlx4_qp_context *context, enum mlx4_qp_optpar optpar, - int sqd_event, struct mlx4_qp *qp) +/* used for INIT/CLOSE port logic */ +static int is_master_qp0(struct mlx4_dev *dev, struct mlx4_qp *qp, int *real_qp0, int *proxy_qp0) { + /* this procedure is called after we already know we are on the master */ + /* qp0 is either the proxy qp0, or the real qp0 */ + u32 pf_proxy_offset = dev->phys_caps.base_proxy_sqpn + 8 * mlx4_master_func_num(dev); + *proxy_qp0 = qp->qpn >= pf_proxy_offset && qp->qpn <= pf_proxy_offset + 1; + + *real_qp0 = qp->qpn >= dev->phys_caps.base_sqpn && + qp->qpn <= dev->phys_caps.base_sqpn + 1; + + return *real_qp0 || *proxy_qp0; +} + +static int __mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state, + struct mlx4_qp_context *context, + enum mlx4_qp_optpar optpar, + int sqd_event, struct mlx4_qp *qp, int native) +{ static const u16 op[MLX4_QP_NUM_STATE][MLX4_QP_NUM_STATE] = { [MLX4_QP_STATE_RST] = { [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, [MLX4_QP_STATE_INIT] = MLX4_CMD_RST2INIT_QP, }, [MLX4_QP_STATE_INIT] = { [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, [MLX4_QP_STATE_INIT] = MLX4_CMD_INIT2INIT_QP, [MLX4_QP_STATE_RTR] = MLX4_CMD_INIT2RTR_QP, }, [MLX4_QP_STATE_RTR] = { [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, [MLX4_QP_STATE_RTS] = MLX4_CMD_RTR2RTS_QP, }, [MLX4_QP_STATE_RTS] = { [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, [MLX4_QP_STATE_RTS] = MLX4_CMD_RTS2RTS_QP, [MLX4_QP_STATE_SQD] = MLX4_CMD_RTS2SQD_QP, }, [MLX4_QP_STATE_SQD] = { [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, [MLX4_QP_STATE_RTS] = MLX4_CMD_SQD2RTS_QP, [MLX4_QP_STATE_SQD] = MLX4_CMD_SQD2SQD_QP, }, [MLX4_QP_STATE_SQER] = { [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, [MLX4_QP_STATE_RTS] = MLX4_CMD_SQERR2RTS_QP, }, [MLX4_QP_STATE_ERR] = { [MLX4_QP_STATE_RST] = MLX4_CMD_2RST_QP, [MLX4_QP_STATE_ERR] = MLX4_CMD_2ERR_QP, } }; + struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_cmd_mailbox *mailbox; int ret = 0; + int real_qp0 = 0; + int proxy_qp0 = 0; + u8 port; if (cur_state >= MLX4_QP_NUM_STATE || new_state >= MLX4_QP_NUM_STATE || !op[cur_state][new_state]) return -EINVAL; - if (op[cur_state][new_state] == MLX4_CMD_2RST_QP) - return mlx4_cmd(dev, 0, qp->qpn, 2, - MLX4_CMD_2RST_QP, MLX4_CMD_TIME_CLASS_A); + if (op[cur_state][new_state] == MLX4_CMD_2RST_QP) { + ret = mlx4_cmd(dev, 0, qp->qpn, 2, + MLX4_CMD_2RST_QP, MLX4_CMD_TIME_CLASS_A, native); + if (mlx4_is_master(dev) && cur_state != MLX4_QP_STATE_ERR && + cur_state != MLX4_QP_STATE_RST && + is_master_qp0(dev, qp, &real_qp0, &proxy_qp0)) { + port = (qp->qpn & 1) + 1; + if (proxy_qp0) + priv->mfunc.master.qp0_state[port].proxy_qp0_active = 0; + else + priv->mfunc.master.qp0_state[port].qp0_active = 0; + } + return ret; + } mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); if (cur_state == MLX4_QP_STATE_RST && new_state == MLX4_QP_STATE_INIT) { u64 mtt_addr = mlx4_mtt_addr(dev, mtt); context->mtt_base_addr_h = mtt_addr >> 32; context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; } *(__be32 *) mailbox->buf = cpu_to_be32(optpar); memcpy(mailbox->buf + 8, context, sizeof *context); ((struct mlx4_qp_context *) (mailbox->buf + 8))->local_qpn = cpu_to_be32(qp->qpn); - ret = mlx4_cmd(dev, mailbox->dma, qp->qpn | (!!sqd_event << 31), + ret = mlx4_cmd(dev, mailbox->dma, + qp->qpn | (!!sqd_event << 31), new_state == MLX4_QP_STATE_RST ? 2 : 0, - op[cur_state][new_state], MLX4_CMD_TIME_CLASS_C); + op[cur_state][new_state], MLX4_CMD_TIME_CLASS_C, native); + if (mlx4_is_master(dev) && is_master_qp0(dev, qp, &real_qp0, &proxy_qp0)) { + port = (qp->qpn & 1) + 1; + if (cur_state != MLX4_QP_STATE_ERR && + cur_state != MLX4_QP_STATE_RST && + new_state == MLX4_QP_STATE_ERR) { + if (proxy_qp0) + priv->mfunc.master.qp0_state[port].proxy_qp0_active = 0; + else + priv->mfunc.master.qp0_state[port].qp0_active = 0; + } else if (new_state == MLX4_QP_STATE_RTR) { + if (proxy_qp0) + priv->mfunc.master.qp0_state[port].proxy_qp0_active = 1; + else + priv->mfunc.master.qp0_state[port].qp0_active = 1; + } + } + mlx4_free_cmd_mailbox(dev, mailbox); return ret; } + +int mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt, + enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state, + struct mlx4_qp_context *context, + enum mlx4_qp_optpar optpar, + int sqd_event, struct mlx4_qp *qp) +{ + return __mlx4_qp_modify(dev, mtt, cur_state, new_state, context, + optpar, sqd_event, qp, 0); +} EXPORT_SYMBOL_GPL(mlx4_qp_modify); -int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, int *base) +int __mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, + int *base, u8 flags) { + int bf_qp = !!(flags & (u8) MLX4_RESERVE_BF_QP); + struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_qp_table *qp_table = &priv->qp_table; - int qpn; - qpn = mlx4_bitmap_alloc_range(&qp_table->bitmap, cnt, align); - if (qpn == -1) + /* Only IPoIB uses a large cnt. In this case, just allocate + * as usual, ignoring bf skipping, since IPoIB does not run over RoCE + */ + if (cnt > MLX4_MAX_BF_QP_RANGE && bf_qp) + bf_qp = 0; + + *base = mlx4_bitmap_alloc_range(&qp_table->bitmap, cnt, align, + bf_qp ? MLX4_BF_QP_SKIP_MASK : 0); + if (*base == -1) return -ENOMEM; - *base = qpn; return 0; } + +int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, + int *base, u8 flags) +{ + u64 in_param = 0; + u64 out_param; + int err; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, (((u32) flags) << 24) | (u32) cnt); + set_param_h(&in_param, align); + err = mlx4_cmd_imm(dev, in_param, &out_param, + RES_QP, RES_OP_RESERVE, + MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) + return err; + + *base = get_param_l(&out_param); + return 0; + } + return __mlx4_qp_reserve_range(dev, cnt, align, base, flags); +} EXPORT_SYMBOL_GPL(mlx4_qp_reserve_range); -void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt) +void __mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_qp_table *qp_table = &priv->qp_table; - if (base_qpn < dev->caps.sqp_start + 8) + + if (mlx4_is_qp_reserved(dev, (u32) base_qpn)) return; + mlx4_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt, MLX4_USE_RR); +} - mlx4_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt); +void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt) +{ + u64 in_param = 0; + int err; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, base_qpn); + set_param_h(&in_param, cnt); + err = mlx4_cmd(dev, in_param, RES_QP, RES_OP_RESERVE, + MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (err) { + mlx4_warn(dev, "Failed to release qp range" + " base:%d cnt:%d\n", base_qpn, cnt); + } + } else + __mlx4_qp_release_range(dev, base_qpn, cnt); } EXPORT_SYMBOL_GPL(mlx4_qp_release_range); -int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp) +int __mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_qp_table *qp_table = &priv->qp_table; int err; - if (!qpn) - return -EINVAL; - - qp->qpn = qpn; - - err = mlx4_table_get(dev, &qp_table->qp_table, qp->qpn); + err = mlx4_table_get(dev, &qp_table->qp_table, qpn); if (err) goto err_out; - err = mlx4_table_get(dev, &qp_table->auxc_table, qp->qpn); + err = mlx4_table_get(dev, &qp_table->auxc_table, qpn); if (err) goto err_put_qp; - err = mlx4_table_get(dev, &qp_table->altc_table, qp->qpn); + err = mlx4_table_get(dev, &qp_table->altc_table, qpn); if (err) goto err_put_auxc; - err = mlx4_table_get(dev, &qp_table->rdmarc_table, qp->qpn); + err = mlx4_table_get(dev, &qp_table->rdmarc_table, qpn); if (err) goto err_put_altc; - err = mlx4_table_get(dev, &qp_table->cmpt_table, qp->qpn); + err = mlx4_table_get(dev, &qp_table->cmpt_table, qpn); if (err) goto err_put_rdmarc; - spin_lock_irq(&qp_table->lock); - err = radix_tree_insert(&dev->qp_table_tree, qp->qpn & (dev->caps.num_qps - 1), qp); - spin_unlock_irq(&qp_table->lock); - if (err) - goto err_put_cmpt; - - atomic_set(&qp->refcount, 1); - init_completion(&qp->free); - return 0; -err_put_cmpt: - mlx4_table_put(dev, &qp_table->cmpt_table, qp->qpn); - err_put_rdmarc: - mlx4_table_put(dev, &qp_table->rdmarc_table, qp->qpn); + mlx4_table_put(dev, &qp_table->rdmarc_table, qpn); err_put_altc: - mlx4_table_put(dev, &qp_table->altc_table, qp->qpn); + mlx4_table_put(dev, &qp_table->altc_table, qpn); err_put_auxc: - mlx4_table_put(dev, &qp_table->auxc_table, qp->qpn); + mlx4_table_put(dev, &qp_table->auxc_table, qpn); err_put_qp: - mlx4_table_put(dev, &qp_table->qp_table, qp->qpn); + mlx4_table_put(dev, &qp_table->qp_table, qpn); err_out: return err; } -EXPORT_SYMBOL_GPL(mlx4_qp_alloc); -struct mlx4_qp *mlx4_qp_lookup_lock(struct mlx4_dev *dev, u32 qpn) +static int mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn) { - struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; - unsigned long flags; - struct mlx4_qp *qp; + u64 param = 0; - spin_lock_irqsave(&qp_table->lock, flags); - qp = radix_tree_lookup(&dev->qp_table_tree, qpn & (dev->caps.num_qps - 1)); - spin_unlock_irqrestore(&qp_table->lock, flags); - return qp; + if (mlx4_is_mfunc(dev)) { + set_param_l(¶m, qpn); + return mlx4_cmd_imm(dev, param, ¶m, RES_QP, RES_OP_MAP_ICM, + MLX4_CMD_ALLOC_RES, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); + } + return __mlx4_qp_alloc_icm(dev, qpn); } -EXPORT_SYMBOL_GPL(mlx4_qp_lookup_lock); +void __mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_qp_table *qp_table = &priv->qp_table; + + mlx4_table_put(dev, &qp_table->cmpt_table, qpn); + mlx4_table_put(dev, &qp_table->rdmarc_table, qpn); + mlx4_table_put(dev, &qp_table->altc_table, qpn); + mlx4_table_put(dev, &qp_table->auxc_table, qpn); + mlx4_table_put(dev, &qp_table->qp_table, qpn); +} + +static void mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn) +{ + u64 in_param = 0; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, qpn); + if (mlx4_cmd(dev, in_param, RES_QP, RES_OP_MAP_ICM, + MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED)) + mlx4_warn(dev, "Failed to free icm of qp:%d\n", qpn); + } else + __mlx4_qp_free_icm(dev, qpn); +} + +int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_qp_table *qp_table = &priv->qp_table; + int err; + + if (!qpn) + return -EINVAL; + + qp->qpn = qpn; + + err = mlx4_qp_alloc_icm(dev, qpn); + if (err) + return err; + + spin_lock_irq(&qp_table->lock); + err = radix_tree_insert(&dev->qp_table_tree, qp->qpn & + (dev->caps.num_qps - 1), qp); + spin_unlock_irq(&qp_table->lock); + if (err) + goto err_icm; + + atomic_set(&qp->refcount, 1); + init_completion(&qp->free); + + return 0; + +err_icm: + mlx4_qp_free_icm(dev, qpn); + return err; +} + +EXPORT_SYMBOL_GPL(mlx4_qp_alloc); + void mlx4_qp_remove(struct mlx4_dev *dev, struct mlx4_qp *qp) { struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; unsigned long flags; spin_lock_irqsave(&qp_table->lock, flags); radix_tree_delete(&dev->qp_table_tree, qp->qpn & (dev->caps.num_qps - 1)); spin_unlock_irqrestore(&qp_table->lock, flags); } EXPORT_SYMBOL_GPL(mlx4_qp_remove); void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp) { - struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; - if (atomic_dec_and_test(&qp->refcount)) complete(&qp->free); wait_for_completion(&qp->free); - mlx4_table_put(dev, &qp_table->cmpt_table, qp->qpn); - mlx4_table_put(dev, &qp_table->rdmarc_table, qp->qpn); - mlx4_table_put(dev, &qp_table->altc_table, qp->qpn); - mlx4_table_put(dev, &qp_table->auxc_table, qp->qpn); - mlx4_table_put(dev, &qp_table->qp_table, qp->qpn); + mlx4_qp_free_icm(dev, qp->qpn); } EXPORT_SYMBOL_GPL(mlx4_qp_free); static int mlx4_CONF_SPECIAL_QP(struct mlx4_dev *dev, u32 base_qpn) { - return mlx4_cmd(dev, 0, base_qpn, - (dev->caps.flags & MLX4_DEV_CAP_FLAG_RAW_ETY) ? 4 : 0, - MLX4_CMD_CONF_SPECIAL_QP, MLX4_CMD_TIME_CLASS_B); + return mlx4_cmd(dev, 0, base_qpn, 0, MLX4_CMD_CONF_SPECIAL_QP, + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); } int mlx4_init_qp_table(struct mlx4_dev *dev) { struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table; int err; int reserved_from_top = 0; + int reserved_from_bot; + int k; spin_lock_init(&qp_table->lock); INIT_RADIX_TREE(&dev->qp_table_tree, GFP_ATOMIC); + if (mlx4_is_slave(dev)) + return 0; /* * We reserve 2 extra QPs per port for the special QPs. The * block of special QPs must be aligned to a multiple of 8, so * round up. + * * We also reserve the MSB of the 24-bit QP number to indicate - * an XRC qp. + * that a QP is an XRC QP. */ - dev->caps.sqp_start = + dev->phys_caps.base_sqpn = ALIGN(dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], 8); { int sort[MLX4_NUM_QP_REGION]; int i, j, tmp; int last_base = dev->caps.num_qps; for (i = 1; i < MLX4_NUM_QP_REGION; ++i) sort[i] = i; for (i = MLX4_NUM_QP_REGION; i > 0; --i) { for (j = 2; j < i; ++j) { if (dev->caps.reserved_qps_cnt[sort[j]] > dev->caps.reserved_qps_cnt[sort[j - 1]]) { tmp = sort[j]; sort[j] = sort[j - 1]; sort[j - 1] = tmp; } } } for (i = 1; i < MLX4_NUM_QP_REGION; ++i) { last_base -= dev->caps.reserved_qps_cnt[sort[i]]; dev->caps.reserved_qps_base[sort[i]] = last_base; reserved_from_top += dev->caps.reserved_qps_cnt[sort[i]]; } } + /* Reserve 8 real SQPs in both native and SRIOV modes. + * In addition, in SRIOV mode, reserve 8 proxy SQPs per function + * (for all PFs and VFs), and 8 corresponding tunnel QPs. + * Each proxy SQP works opposite its own tunnel QP. + * + * The QPs are arranged as follows: + * a. 8 real SQPs + * b. All the proxy SQPs (8 per function) + * c. All the tunnel QPs (8 per function) + */ + reserved_from_bot = mlx4_num_reserved_sqps(dev); + if (reserved_from_bot + reserved_from_top > dev->caps.num_qps) { + mlx4_err(dev, "Number of reserved QPs is higher than number " + "of QPs, increase the value of log_num_qp\n"); + return -EINVAL; + } + err = mlx4_bitmap_init(&qp_table->bitmap, dev->caps.num_qps, - (1 << 23) - 1, dev->caps.sqp_start + 8, + (1 << 23) - 1, reserved_from_bot, reserved_from_top); if (err) return err; - return mlx4_CONF_SPECIAL_QP(dev, dev->caps.sqp_start); + if (mlx4_is_mfunc(dev)) { + /* for PPF use */ + dev->phys_caps.base_proxy_sqpn = dev->phys_caps.base_sqpn + 8; + dev->phys_caps.base_tunnel_sqpn = dev->phys_caps.base_sqpn + 8 + 8 * MLX4_MFUNC_MAX; + + /* In mfunc, calculate proxy and tunnel qp offsets for the PF here, + * since the PF does not call mlx4_slave_caps */ + dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL); + + if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy || + !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy) { + err = -ENOMEM; + goto err_mem; + } + + for (k = 0; k < dev->caps.num_ports; k++) { + dev->caps.qp0_proxy[k] = dev->phys_caps.base_proxy_sqpn + + 8 * mlx4_master_func_num(dev) + k; + dev->caps.qp0_tunnel[k] = dev->caps.qp0_proxy[k] + 8 * MLX4_MFUNC_MAX; + dev->caps.qp1_proxy[k] = dev->phys_caps.base_proxy_sqpn + + 8 * mlx4_master_func_num(dev) + MLX4_MAX_PORTS + k; + dev->caps.qp1_tunnel[k] = dev->caps.qp1_proxy[k] + 8 * MLX4_MFUNC_MAX; + } + } + + + err = mlx4_CONF_SPECIAL_QP(dev, dev->phys_caps.base_sqpn); + if (err) + goto err_mem; + return 0; + +err_mem: + kfree(dev->caps.qp0_tunnel); + kfree(dev->caps.qp0_proxy); + kfree(dev->caps.qp1_tunnel); + kfree(dev->caps.qp1_proxy); + dev->caps.qp0_tunnel = dev->caps.qp0_proxy = + dev->caps.qp1_tunnel = dev->caps.qp1_proxy = NULL; + return err; } void mlx4_cleanup_qp_table(struct mlx4_dev *dev) { + if (mlx4_is_slave(dev)) + return; + mlx4_CONF_SPECIAL_QP(dev, 0); mlx4_bitmap_cleanup(&mlx4_priv(dev)->qp_table.bitmap); } -int mlx4_qp_get_region(struct mlx4_dev *dev, enum mlx4_qp_region region, - int *base_qpn, int *cnt) -{ - if ((region < 0) || (region >= MLX4_NUM_QP_REGION)) - return -EINVAL; - - *base_qpn = dev->caps.reserved_qps_base[region]; - *cnt = dev->caps.reserved_qps_cnt[region]; - - return 0; -} -EXPORT_SYMBOL_GPL(mlx4_qp_get_region); - int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp, struct mlx4_qp_context *context) { struct mlx4_cmd_mailbox *mailbox; int err; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); err = mlx4_cmd_box(dev, 0, mailbox->dma, qp->qpn, 0, - MLX4_CMD_QUERY_QP, MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_QUERY_QP, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); if (!err) memcpy(context, mailbox->buf + 8, sizeof *context); mlx4_free_cmd_mailbox(dev, mailbox); return err; } EXPORT_SYMBOL_GPL(mlx4_qp_query); int mlx4_qp_to_ready(struct mlx4_dev *dev, struct mlx4_mtt *mtt, struct mlx4_qp_context *context, struct mlx4_qp *qp, enum mlx4_qp_state *qp_state) { int err; int i; enum mlx4_qp_state states[] = { MLX4_QP_STATE_RST, MLX4_QP_STATE_INIT, MLX4_QP_STATE_RTR, MLX4_QP_STATE_RTS }; for (i = 0; i < ARRAY_SIZE(states) - 1; i++) { context->flags &= cpu_to_be32(~(0xf << 28)); context->flags |= cpu_to_be32(states[i + 1] << 28); err = mlx4_qp_modify(dev, mtt, states[i], states[i + 1], context, 0, 0, qp); if (err) { mlx4_err(dev, "Failed to bring QP to state: " "%d with error: %d\n", states[i + 1], err); return err; } *qp_state = states[i + 1]; } return 0; } EXPORT_SYMBOL_GPL(mlx4_qp_to_ready); Index: stable/9/sys/ofed/drivers/net/mlx4/reset.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/reset.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/reset.c (revision 273864) @@ -1,186 +1,185 @@ /* * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include #include #include #include #include #include #include "mlx4.h" int mlx4_reset(struct mlx4_dev *dev) { void __iomem *reset; u32 *hca_header = NULL; int pcie_cap; u16 devctl; u16 linkctl; u16 vendor; unsigned long end; u32 sem; int i; int err = 0; #define MLX4_RESET_BASE 0xf0000 #define MLX4_RESET_SIZE 0x400 #define MLX4_SEM_OFFSET 0x3fc #define MLX4_RESET_OFFSET 0x10 #define MLX4_RESET_VALUE swab32(1) #define MLX4_SEM_TIMEOUT_JIFFIES (10 * HZ) #define MLX4_RESET_TIMEOUT_JIFFIES (2 * HZ) /* * Reset the chip. This is somewhat ugly because we have to * save off the PCI header before reset and then restore it * after the chip reboots. We skip config space offsets 22 * and 23 since those have a special meaning. */ /* Do we need to save off the full 4K PCI Express header?? */ hca_header = kmalloc(256, GFP_KERNEL); if (!hca_header) { err = -ENOMEM; mlx4_err(dev, "Couldn't allocate memory to save HCA " "PCI header, aborting.\n"); goto out; } - pcie_cap = pci_find_capability(dev->pdev, PCI_CAP_ID_EXP); + pcie_cap = pci_pcie_cap(dev->pdev); for (i = 0; i < 64; ++i) { if (i == 22 || i == 23) continue; if (pci_read_config_dword(dev->pdev, i * 4, hca_header + i)) { err = -ENODEV; mlx4_err(dev, "Couldn't save HCA " "PCI header, aborting.\n"); goto out; } } reset = ioremap(pci_resource_start(dev->pdev, 0) + MLX4_RESET_BASE, MLX4_RESET_SIZE); if (!reset) { err = -ENOMEM; mlx4_err(dev, "Couldn't map HCA reset register, aborting.\n"); goto out; } /* grab HW semaphore to lock out flash updates */ end = jiffies + MLX4_SEM_TIMEOUT_JIFFIES; do { sem = readl(reset + MLX4_SEM_OFFSET); if (!sem) break; msleep(1); } while (time_before(jiffies, end)); if (sem) { mlx4_err(dev, "Failed to obtain HW semaphore, aborting\n"); err = -EAGAIN; iounmap(reset); goto out; } /* actually hit reset */ writel(MLX4_RESET_VALUE, reset + MLX4_RESET_OFFSET); iounmap(reset); - /* Docs say to wait one second before accessing device */ - msleep(1000); + /* wait half a second before accessing device */ + msleep(500); end = jiffies + MLX4_RESET_TIMEOUT_JIFFIES; do { if (!pci_read_config_word(dev->pdev, PCI_VENDOR_ID, &vendor) && vendor != 0xffff) break; msleep(1); } while (time_before(jiffies, end)); if (vendor == 0xffff) { err = -ENODEV; mlx4_err(dev, "PCI device did not come back after reset, " "aborting.\n"); goto out; } /* Now restore the PCI headers */ if (pcie_cap) { devctl = hca_header[(pcie_cap + PCI_EXP_DEVCTL) / 4]; - if (pci_write_config_word(dev->pdev, pcie_cap + PCI_EXP_DEVCTL, - devctl)) { + if (pcie_capability_write_word(dev->pdev, PCI_EXP_DEVCTL, + devctl)) { err = -ENODEV; mlx4_err(dev, "Couldn't restore HCA PCI Express " "Device Control register, aborting.\n"); goto out; } linkctl = hca_header[(pcie_cap + PCI_EXP_LNKCTL) / 4]; - if (pci_write_config_word(dev->pdev, pcie_cap + PCI_EXP_LNKCTL, - linkctl)) { + if (pcie_capability_write_word(dev->pdev, PCI_EXP_LNKCTL, + linkctl)) { err = -ENODEV; mlx4_err(dev, "Couldn't restore HCA PCI Express " "Link control register, aborting.\n"); goto out; } } for (i = 0; i < 16; ++i) { if (i * 4 == PCI_COMMAND) continue; if (pci_write_config_dword(dev->pdev, i * 4, hca_header[i])) { err = -ENODEV; mlx4_err(dev, "Couldn't restore HCA reg %x, " "aborting.\n", i); goto out; } } if (pci_write_config_dword(dev->pdev, PCI_COMMAND, hca_header[PCI_COMMAND / 4])) { err = -ENODEV; mlx4_err(dev, "Couldn't restore HCA COMMAND, " "aborting.\n"); goto out; } out: kfree(hca_header); return err; } Index: stable/9/sys/ofed/drivers/net/mlx4/resource_tracker.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/resource_tracker.c (nonexistent) +++ stable/9/sys/ofed/drivers/net/mlx4/resource_tracker.c (revision 273864) @@ -0,0 +1,4686 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006, 2007, 2008, 2014 Mellanox Technologies. + * All rights reserved. + * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mlx4.h" +#include "fw.h" + +#define MLX4_MAC_VALID (1ull << 63) + +struct mac_res { + struct list_head list; + u64 mac; + int ref_count; + u8 smac_index; + u8 port; +}; + +struct vlan_res { + struct list_head list; + u16 vlan; + int ref_count; + int vlan_index; + u8 port; +}; + +struct res_common { + struct list_head list; + struct rb_node node; + u64 res_id; + int owner; + int state; + int from_state; + int to_state; + int removing; +}; + +enum { + RES_ANY_BUSY = 1 +}; + +struct res_gid { + struct list_head list; + u8 gid[16]; + enum mlx4_protocol prot; + enum mlx4_steer_type steer; + u64 reg_id; +}; + +enum res_qp_states { + RES_QP_BUSY = RES_ANY_BUSY, + + /* QP number was allocated */ + RES_QP_RESERVED, + + /* ICM memory for QP context was mapped */ + RES_QP_MAPPED, + + /* QP is in hw ownership */ + RES_QP_HW +}; + +struct res_qp { + struct res_common com; + struct res_mtt *mtt; + struct res_cq *rcq; + struct res_cq *scq; + struct res_srq *srq; + struct list_head mcg_list; + spinlock_t mcg_spl; + int local_qpn; + atomic_t ref_count; + u32 qpc_flags; + /* saved qp params before VST enforcement in order to restore on VGT */ + u8 sched_queue; + __be32 param3; + u8 vlan_control; + u8 fvl_rx; + u8 pri_path_fl; + u8 vlan_index; + u8 feup; +}; + +enum res_mtt_states { + RES_MTT_BUSY = RES_ANY_BUSY, + RES_MTT_ALLOCATED, +}; + +static inline const char *mtt_states_str(enum res_mtt_states state) +{ + switch (state) { + case RES_MTT_BUSY: return "RES_MTT_BUSY"; + case RES_MTT_ALLOCATED: return "RES_MTT_ALLOCATED"; + default: return "Unknown"; + } +} + +struct res_mtt { + struct res_common com; + int order; + atomic_t ref_count; +}; + +enum res_mpt_states { + RES_MPT_BUSY = RES_ANY_BUSY, + RES_MPT_RESERVED, + RES_MPT_MAPPED, + RES_MPT_HW, +}; + +struct res_mpt { + struct res_common com; + struct res_mtt *mtt; + int key; +}; + +enum res_eq_states { + RES_EQ_BUSY = RES_ANY_BUSY, + RES_EQ_RESERVED, + RES_EQ_HW, +}; + +struct res_eq { + struct res_common com; + struct res_mtt *mtt; +}; + +enum res_cq_states { + RES_CQ_BUSY = RES_ANY_BUSY, + RES_CQ_ALLOCATED, + RES_CQ_HW, +}; + +struct res_cq { + struct res_common com; + struct res_mtt *mtt; + atomic_t ref_count; +}; + +enum res_srq_states { + RES_SRQ_BUSY = RES_ANY_BUSY, + RES_SRQ_ALLOCATED, + RES_SRQ_HW, +}; + +struct res_srq { + struct res_common com; + struct res_mtt *mtt; + struct res_cq *cq; + atomic_t ref_count; +}; + +enum res_counter_states { + RES_COUNTER_BUSY = RES_ANY_BUSY, + RES_COUNTER_ALLOCATED, +}; + +struct res_counter { + struct res_common com; + int port; +}; + +enum res_xrcdn_states { + RES_XRCD_BUSY = RES_ANY_BUSY, + RES_XRCD_ALLOCATED, +}; + +struct res_xrcdn { + struct res_common com; + int port; +}; + +enum res_fs_rule_states { + RES_FS_RULE_BUSY = RES_ANY_BUSY, + RES_FS_RULE_ALLOCATED, +}; + +struct res_fs_rule { + struct res_common com; + int qpn; +}; + +static int mlx4_is_eth(struct mlx4_dev *dev, int port) +{ + return dev->caps.port_mask[port] == MLX4_PORT_TYPE_IB ? 0 : 1; +} + +static void *res_tracker_lookup(struct rb_root *root, u64 res_id) +{ + struct rb_node *node = root->rb_node; + + while (node) { + struct res_common *res = container_of(node, struct res_common, + node); + + if (res_id < res->res_id) + node = node->rb_left; + else if (res_id > res->res_id) + node = node->rb_right; + else + return res; + } + return NULL; +} + +static int res_tracker_insert(struct rb_root *root, struct res_common *res) +{ + struct rb_node **new = &(root->rb_node), *parent = NULL; + + /* Figure out where to put new node */ + while (*new) { + struct res_common *this = container_of(*new, struct res_common, + node); + + parent = *new; + if (res->res_id < this->res_id) + new = &((*new)->rb_left); + else if (res->res_id > this->res_id) + new = &((*new)->rb_right); + else + return -EEXIST; + } + + /* Add new node and rebalance tree. */ + rb_link_node(&res->node, parent, new); + rb_insert_color(&res->node, root); + + return 0; +} + +enum qp_transition { + QP_TRANS_INIT2RTR, + QP_TRANS_RTR2RTS, + QP_TRANS_RTS2RTS, + QP_TRANS_SQERR2RTS, + QP_TRANS_SQD2SQD, + QP_TRANS_SQD2RTS +}; + +/* For Debug uses */ +static const char *ResourceType(enum mlx4_resource rt) +{ + switch (rt) { + case RES_QP: return "RES_QP"; + case RES_CQ: return "RES_CQ"; + case RES_SRQ: return "RES_SRQ"; + case RES_MPT: return "RES_MPT"; + case RES_MTT: return "RES_MTT"; + case RES_MAC: return "RES_MAC"; + case RES_VLAN: return "RES_VLAN"; + case RES_EQ: return "RES_EQ"; + case RES_COUNTER: return "RES_COUNTER"; + case RES_FS_RULE: return "RES_FS_RULE"; + case RES_XRCD: return "RES_XRCD"; + default: return "Unknown resource type !!!"; + }; +} + +static void rem_slave_vlans(struct mlx4_dev *dev, int slave); +static inline int mlx4_grant_resource(struct mlx4_dev *dev, int slave, + enum mlx4_resource res_type, int count, + int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct resource_allocator *res_alloc = + &priv->mfunc.master.res_tracker.res_alloc[res_type]; + int err = -EINVAL; + int allocated, free, reserved, guaranteed, from_free; + + spin_lock(&res_alloc->alloc_lock); + allocated = (port > 0) ? + res_alloc->allocated[(port - 1) * (dev->num_vfs + 1) + slave] : + res_alloc->allocated[slave]; + free = (port > 0) ? res_alloc->res_port_free[port - 1] : + res_alloc->res_free; + reserved = (port > 0) ? res_alloc->res_port_rsvd[port - 1] : + res_alloc->res_reserved; + guaranteed = res_alloc->guaranteed[slave]; + + if (allocated + count > res_alloc->quota[slave]) + goto out; + + if (allocated + count <= guaranteed) { + err = 0; + } else { + /* portion may need to be obtained from free area */ + if (guaranteed - allocated > 0) + from_free = count - (guaranteed - allocated); + else + from_free = count; + + if (free - from_free > reserved) + err = 0; + } + + if (!err) { + /* grant the request */ + if (port > 0) { + res_alloc->allocated[(port - 1) * (dev->num_vfs + 1) + slave] += count; + res_alloc->res_port_free[port - 1] -= count; + } else { + res_alloc->allocated[slave] += count; + res_alloc->res_free -= count; + } + } + +out: + spin_unlock(&res_alloc->alloc_lock); + return err; + +} + +static inline void mlx4_release_resource(struct mlx4_dev *dev, int slave, + enum mlx4_resource res_type, int count, + int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct resource_allocator *res_alloc = + &priv->mfunc.master.res_tracker.res_alloc[res_type]; + + spin_lock(&res_alloc->alloc_lock); + if (port > 0) { + res_alloc->allocated[(port - 1) * (dev->num_vfs + 1) + slave] -= count; + res_alloc->res_port_free[port - 1] += count; + } else { + res_alloc->allocated[slave] -= count; + res_alloc->res_free += count; + } + + spin_unlock(&res_alloc->alloc_lock); + return; +} + +static inline void initialize_res_quotas(struct mlx4_dev *dev, + struct resource_allocator *res_alloc, + enum mlx4_resource res_type, + int vf, int num_instances) +{ + res_alloc->guaranteed[vf] = num_instances / (2 * (dev->num_vfs + 1)); + res_alloc->quota[vf] = (num_instances / 2) + res_alloc->guaranteed[vf]; + if (vf == mlx4_master_func_num(dev)) { + res_alloc->res_free = num_instances; + if (res_type == RES_MTT) { + /* reserved mtts will be taken out of the PF allocation */ + res_alloc->res_free += dev->caps.reserved_mtts; + res_alloc->guaranteed[vf] += dev->caps.reserved_mtts; + res_alloc->quota[vf] += dev->caps.reserved_mtts; + } + } +} + +void mlx4_init_quotas(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int pf; + + /* quotas for VFs are initialized in mlx4_slave_cap */ + if (mlx4_is_slave(dev)) + return; + + if (!mlx4_is_mfunc(dev)) { + dev->quotas.qp = dev->caps.num_qps - dev->caps.reserved_qps - + mlx4_num_reserved_sqps(dev); + dev->quotas.cq = dev->caps.num_cqs - dev->caps.reserved_cqs; + dev->quotas.srq = dev->caps.num_srqs - dev->caps.reserved_srqs; + dev->quotas.mtt = dev->caps.num_mtts - dev->caps.reserved_mtts; + dev->quotas.mpt = dev->caps.num_mpts - dev->caps.reserved_mrws; + return; + } + + pf = mlx4_master_func_num(dev); + dev->quotas.qp = + priv->mfunc.master.res_tracker.res_alloc[RES_QP].quota[pf]; + dev->quotas.cq = + priv->mfunc.master.res_tracker.res_alloc[RES_CQ].quota[pf]; + dev->quotas.srq = + priv->mfunc.master.res_tracker.res_alloc[RES_SRQ].quota[pf]; + dev->quotas.mtt = + priv->mfunc.master.res_tracker.res_alloc[RES_MTT].quota[pf]; + dev->quotas.mpt = + priv->mfunc.master.res_tracker.res_alloc[RES_MPT].quota[pf]; +} +int mlx4_init_resource_tracker(struct mlx4_dev *dev) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int i, j; + int t; + + priv->mfunc.master.res_tracker.slave_list = + kzalloc(dev->num_slaves * sizeof(struct slave_list), + GFP_KERNEL); + if (!priv->mfunc.master.res_tracker.slave_list) + return -ENOMEM; + + for (i = 0 ; i < dev->num_slaves; i++) { + for (t = 0; t < MLX4_NUM_OF_RESOURCE_TYPE; ++t) + INIT_LIST_HEAD(&priv->mfunc.master.res_tracker. + slave_list[i].res_list[t]); + mutex_init(&priv->mfunc.master.res_tracker.slave_list[i].mutex); + } + + mlx4_dbg(dev, "Started init_resource_tracker: %ld slaves\n", + dev->num_slaves); + for (i = 0 ; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) + priv->mfunc.master.res_tracker.res_tree[i] = RB_ROOT; + + for (i = 0; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) { + struct resource_allocator *res_alloc = + &priv->mfunc.master.res_tracker.res_alloc[i]; + res_alloc->quota = kmalloc((dev->num_vfs + 1) * sizeof(int), GFP_KERNEL); + res_alloc->guaranteed = kmalloc((dev->num_vfs + 1) * sizeof(int), GFP_KERNEL); + if (i == RES_MAC || i == RES_VLAN) + res_alloc->allocated = kzalloc(MLX4_MAX_PORTS * + (dev->num_vfs + 1) * sizeof(int), + GFP_KERNEL); + else + res_alloc->allocated = kzalloc((dev->num_vfs + 1) * sizeof(int), GFP_KERNEL); + + if (!res_alloc->quota || !res_alloc->guaranteed || + !res_alloc->allocated) + goto no_mem_err; + + spin_lock_init(&res_alloc->alloc_lock); + for (t = 0; t < dev->num_vfs + 1; t++) { + switch (i) { + case RES_QP: + initialize_res_quotas(dev, res_alloc, RES_QP, + t, dev->caps.num_qps - + dev->caps.reserved_qps - + mlx4_num_reserved_sqps(dev)); + break; + case RES_CQ: + initialize_res_quotas(dev, res_alloc, RES_CQ, + t, dev->caps.num_cqs - + dev->caps.reserved_cqs); + break; + case RES_SRQ: + initialize_res_quotas(dev, res_alloc, RES_SRQ, + t, dev->caps.num_srqs - + dev->caps.reserved_srqs); + break; + case RES_MPT: + initialize_res_quotas(dev, res_alloc, RES_MPT, + t, dev->caps.num_mpts - + dev->caps.reserved_mrws); + break; + case RES_MTT: + initialize_res_quotas(dev, res_alloc, RES_MTT, + t, dev->caps.num_mtts - + dev->caps.reserved_mtts); + break; + case RES_MAC: + if (t == mlx4_master_func_num(dev)) { + res_alloc->quota[t] = + MLX4_MAX_MAC_NUM - 2 * dev->num_vfs; + res_alloc->guaranteed[t] = res_alloc->quota[t]; + for (j = 0; j < MLX4_MAX_PORTS; j++) + res_alloc->res_port_free[j] = MLX4_MAX_MAC_NUM; + } else { + res_alloc->quota[t] = 2; + res_alloc->guaranteed[t] = 2; + } + break; + case RES_VLAN: + if (t == mlx4_master_func_num(dev)) { + res_alloc->quota[t] = MLX4_MAX_VLAN_NUM; + res_alloc->guaranteed[t] = MLX4_MAX_VLAN_NUM / 2; + for (j = 0; j < MLX4_MAX_PORTS; j++) + res_alloc->res_port_free[j] = + res_alloc->quota[t]; + } else { + res_alloc->quota[t] = MLX4_MAX_VLAN_NUM / 2; + res_alloc->guaranteed[t] = 0; + } + break; + case RES_COUNTER: + res_alloc->quota[t] = dev->caps.max_counters; + res_alloc->guaranteed[t] = 0; + if (t == mlx4_master_func_num(dev)) + res_alloc->res_free = res_alloc->quota[t]; + break; + default: + break; + } + if (i == RES_MAC || i == RES_VLAN) { + for (j = 0; j < MLX4_MAX_PORTS; j++) + res_alloc->res_port_rsvd[j] += + res_alloc->guaranteed[t]; + } else { + res_alloc->res_reserved += res_alloc->guaranteed[t]; + } + } + } + spin_lock_init(&priv->mfunc.master.res_tracker.lock); + return 0; + +no_mem_err: + for (i = 0; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) { + kfree(priv->mfunc.master.res_tracker.res_alloc[i].allocated); + priv->mfunc.master.res_tracker.res_alloc[i].allocated = NULL; + kfree(priv->mfunc.master.res_tracker.res_alloc[i].guaranteed); + priv->mfunc.master.res_tracker.res_alloc[i].guaranteed = NULL; + kfree(priv->mfunc.master.res_tracker.res_alloc[i].quota); + priv->mfunc.master.res_tracker.res_alloc[i].quota = NULL; + } + return -ENOMEM; +} + +void mlx4_free_resource_tracker(struct mlx4_dev *dev, + enum mlx4_res_tracker_free_type type) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + int i; + + if (priv->mfunc.master.res_tracker.slave_list) { + if (type != RES_TR_FREE_STRUCTS_ONLY) { + for (i = 0; i < dev->num_slaves; i++) { + if (type == RES_TR_FREE_ALL || + dev->caps.function != i) + mlx4_delete_all_resources_for_slave(dev, i); + } + /* free master's vlans */ + i = dev->caps.function; + mutex_lock(&priv->mfunc.master.res_tracker.slave_list[i].mutex); + rem_slave_vlans(dev, i); + mutex_unlock(&priv->mfunc.master.res_tracker.slave_list[i].mutex); + } + + if (type != RES_TR_FREE_SLAVES_ONLY) { + for (i = 0; i < MLX4_NUM_OF_RESOURCE_TYPE; i++) { + kfree(priv->mfunc.master.res_tracker.res_alloc[i].allocated); + priv->mfunc.master.res_tracker.res_alloc[i].allocated = NULL; + kfree(priv->mfunc.master.res_tracker.res_alloc[i].guaranteed); + priv->mfunc.master.res_tracker.res_alloc[i].guaranteed = NULL; + kfree(priv->mfunc.master.res_tracker.res_alloc[i].quota); + priv->mfunc.master.res_tracker.res_alloc[i].quota = NULL; + } + kfree(priv->mfunc.master.res_tracker.slave_list); + priv->mfunc.master.res_tracker.slave_list = NULL; + } + } +} + +static void update_pkey_index(struct mlx4_dev *dev, int slave, + struct mlx4_cmd_mailbox *inbox) +{ + u8 sched = *(u8 *)(inbox->buf + 64); + u8 orig_index = *(u8 *)(inbox->buf + 35); + u8 new_index; + struct mlx4_priv *priv = mlx4_priv(dev); + int port; + + port = (sched >> 6 & 1) + 1; + + new_index = priv->virt2phys_pkey[slave][port - 1][orig_index]; + *(u8 *)(inbox->buf + 35) = new_index; +} + +static void update_gid(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *inbox, + u8 slave) +{ + struct mlx4_qp_context *qp_ctx = inbox->buf + 8; + enum mlx4_qp_optpar optpar = be32_to_cpu(*(__be32 *) inbox->buf); + u32 ts = (be32_to_cpu(qp_ctx->flags) >> 16) & 0xff; + int port; + + if (MLX4_QP_ST_UD == ts) { + port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1; + if (mlx4_is_eth(dev, port)) + qp_ctx->pri_path.mgid_index = mlx4_get_base_gid_ix(dev, slave) | 0x80; + else + qp_ctx->pri_path.mgid_index = 0x80 | slave; + + } else if (MLX4_QP_ST_RC == ts || MLX4_QP_ST_UC == ts) { + if (optpar & MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH) { + port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1; + if (mlx4_is_eth(dev, port)) { + qp_ctx->pri_path.mgid_index += mlx4_get_base_gid_ix(dev, slave); + qp_ctx->pri_path.mgid_index &= 0x7f; + } else { + qp_ctx->pri_path.mgid_index = slave & 0x7F; + } + } + if (optpar & MLX4_QP_OPTPAR_ALT_ADDR_PATH) { + port = (qp_ctx->alt_path.sched_queue >> 6 & 1) + 1; + if (mlx4_is_eth(dev, port)) { + qp_ctx->alt_path.mgid_index += mlx4_get_base_gid_ix(dev, slave); + qp_ctx->alt_path.mgid_index &= 0x7f; + } else { + qp_ctx->alt_path.mgid_index = slave & 0x7F; + } + } + } +} + +static int check_counter_index_validity(struct mlx4_dev *dev, int slave, int port, int idx) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct counter_index *counter, *tmp_counter; + + if (slave == 0) { + list_for_each_entry_safe(counter, tmp_counter, + &priv->counters_table.global_port_list[port - 1], + list) { + if (counter->index == idx) + return 0; + } + return -EINVAL; + } else { + list_for_each_entry_safe(counter, tmp_counter, + &priv->counters_table.vf_list[slave - 1][port - 1], + list) { + if (counter->index == idx) + return 0; + } + return -EINVAL; + } +} + +static int update_vport_qp_param(struct mlx4_dev *dev, + struct mlx4_cmd_mailbox *inbox, + u8 slave, u32 qpn) +{ + struct mlx4_qp_context *qpc = inbox->buf + 8; + struct mlx4_vport_oper_state *vp_oper; + struct mlx4_priv *priv; + u32 qp_type; + int port; + + port = (qpc->pri_path.sched_queue & 0x40) ? 2 : 1; + priv = mlx4_priv(dev); + vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port]; + qp_type = (be32_to_cpu(qpc->flags) >> 16) & 0xff; + + if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH && + qpc->pri_path.counter_index != MLX4_SINK_COUNTER_INDEX) { + if (check_counter_index_validity(dev, slave, port, + qpc->pri_path.counter_index)) + return -EINVAL; + } + + mlx4_dbg(dev, "%s: QP counter_index %d for slave %d port %d\n", + __func__, qpc->pri_path.counter_index, slave, port); + + if ((dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK) && + dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH && + !mlx4_is_qp_reserved(dev, qpn) && + qp_type == MLX4_QP_ST_MLX && + qpc->pri_path.counter_index != 0xFF) { + /* disable multicast loopback to qp with same counter */ + qpc->pri_path.fl |= MLX4_FL_ETH_SRC_CHECK_MC_LB; + qpc->pri_path.vlan_control |= + MLX4_VLAN_CTRL_ETH_SRC_CHECK_IF_COUNTER; + } + + if (MLX4_VGT != vp_oper->state.default_vlan) { + /* the reserved QPs (special, proxy, tunnel) + * do not operate over vlans + */ + if (mlx4_is_qp_reserved(dev, qpn)) + return 0; + + /* force strip vlan by clear vsd */ + qpc->param3 &= ~cpu_to_be32(MLX4_STRIP_VLAN); + /* preserve IF_COUNTER flag */ + qpc->pri_path.vlan_control &= + MLX4_VLAN_CTRL_ETH_SRC_CHECK_IF_COUNTER; + if (MLX4_QP_ST_RC != qp_type) { + if (0 != vp_oper->state.default_vlan) { + qpc->pri_path.vlan_control |= + MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED; + } else { /* priority tagged */ + qpc->pri_path.vlan_control |= + MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED; + } + } + qpc->pri_path.fvl_rx |= MLX4_FVL_RX_FORCE_ETH_VLAN; + qpc->pri_path.vlan_index = vp_oper->vlan_idx; + qpc->pri_path.fl |= MLX4_FL_CV | MLX4_FL_ETH_HIDE_CQE_VLAN; + qpc->pri_path.feup |= MLX4_FEUP_FORCE_ETH_UP | MLX4_FVL_FORCE_ETH_VLAN; + qpc->pri_path.sched_queue &= 0xC7; + qpc->pri_path.sched_queue |= (vp_oper->state.default_qos) << 3; + } + if (vp_oper->state.spoofchk) { + qpc->pri_path.feup |= MLX4_FSM_FORCE_ETH_SRC_MAC; + qpc->pri_path.grh_mylmc = (0x80 & qpc->pri_path.grh_mylmc) + vp_oper->mac_idx; + } + return 0; +} + +static int mpt_mask(struct mlx4_dev *dev) +{ + return dev->caps.num_mpts - 1; +} + +static void *find_res(struct mlx4_dev *dev, u64 res_id, + enum mlx4_resource type) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + return res_tracker_lookup(&priv->mfunc.master.res_tracker.res_tree[type], + res_id); +} + +static int get_res(struct mlx4_dev *dev, int slave, u64 res_id, + enum mlx4_resource type, + void *res) +{ + struct res_common *r; + int err = 0; + + spin_lock_irq(mlx4_tlock(dev)); + r = find_res(dev, res_id, type); + if (!r) { + err = -ENONET; + goto exit; + } + + if (r->state == RES_ANY_BUSY) { + err = -EBUSY; + goto exit; + } + + if (r->owner != slave) { + err = -EPERM; + goto exit; + } + + r->from_state = r->state; + r->state = RES_ANY_BUSY; + + if (res) + *((struct res_common **)res) = r; + +exit: + spin_unlock_irq(mlx4_tlock(dev)); + return err; +} + +int mlx4_get_slave_from_resource_id(struct mlx4_dev *dev, + enum mlx4_resource type, + u64 res_id, int *slave) +{ + + struct res_common *r; + int err = -ENOENT; + int id = res_id; + + if (type == RES_QP) + id &= 0x7fffff; + spin_lock(mlx4_tlock(dev)); + + r = find_res(dev, id, type); + if (r) { + *slave = r->owner; + err = 0; + } + spin_unlock(mlx4_tlock(dev)); + + return err; +} + +static void put_res(struct mlx4_dev *dev, int slave, u64 res_id, + enum mlx4_resource type) +{ + struct res_common *r; + + spin_lock_irq(mlx4_tlock(dev)); + r = find_res(dev, res_id, type); + if (r) + r->state = r->from_state; + spin_unlock_irq(mlx4_tlock(dev)); +} + +static struct res_common *alloc_qp_tr(int id) +{ + struct res_qp *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_QP_RESERVED; + ret->local_qpn = id; + INIT_LIST_HEAD(&ret->mcg_list); + spin_lock_init(&ret->mcg_spl); + atomic_set(&ret->ref_count, 0); + + return &ret->com; +} + +static struct res_common *alloc_mtt_tr(int id, int order) +{ + struct res_mtt *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->order = order; + ret->com.state = RES_MTT_ALLOCATED; + atomic_set(&ret->ref_count, 0); + + return &ret->com; +} + +static struct res_common *alloc_mpt_tr(int id, int key) +{ + struct res_mpt *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_MPT_RESERVED; + ret->key = key; + + return &ret->com; +} + +static struct res_common *alloc_eq_tr(int id) +{ + struct res_eq *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_EQ_RESERVED; + + return &ret->com; +} + +static struct res_common *alloc_cq_tr(int id) +{ + struct res_cq *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_CQ_ALLOCATED; + atomic_set(&ret->ref_count, 0); + + return &ret->com; +} + +static struct res_common *alloc_srq_tr(int id) +{ + struct res_srq *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_SRQ_ALLOCATED; + atomic_set(&ret->ref_count, 0); + + return &ret->com; +} + +static struct res_common *alloc_counter_tr(int id) +{ + struct res_counter *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_COUNTER_ALLOCATED; + + return &ret->com; +} + +static struct res_common *alloc_xrcdn_tr(int id) +{ + struct res_xrcdn *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_XRCD_ALLOCATED; + + return &ret->com; +} + +static struct res_common *alloc_fs_rule_tr(u64 id, int qpn) +{ + struct res_fs_rule *ret; + + ret = kzalloc(sizeof *ret, GFP_KERNEL); + if (!ret) + return NULL; + + ret->com.res_id = id; + ret->com.state = RES_FS_RULE_ALLOCATED; + ret->qpn = qpn; + return &ret->com; +} + +static struct res_common *alloc_tr(u64 id, enum mlx4_resource type, int slave, + int extra) +{ + struct res_common *ret; + + switch (type) { + case RES_QP: + ret = alloc_qp_tr(id); + break; + case RES_MPT: + ret = alloc_mpt_tr(id, extra); + break; + case RES_MTT: + ret = alloc_mtt_tr(id, extra); + break; + case RES_EQ: + ret = alloc_eq_tr(id); + break; + case RES_CQ: + ret = alloc_cq_tr(id); + break; + case RES_SRQ: + ret = alloc_srq_tr(id); + break; + case RES_MAC: + printk(KERN_ERR "implementation missing\n"); + return NULL; + case RES_COUNTER: + ret = alloc_counter_tr(id); + break; + case RES_XRCD: + ret = alloc_xrcdn_tr(id); + break; + case RES_FS_RULE: + ret = alloc_fs_rule_tr(id, extra); + break; + default: + return NULL; + } + if (ret) + ret->owner = slave; + + return ret; +} + +static int add_res_range(struct mlx4_dev *dev, int slave, u64 base, int count, + enum mlx4_resource type, int extra) +{ + int i; + int err; + struct mlx4_priv *priv = mlx4_priv(dev); + struct res_common **res_arr; + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct rb_root *root = &tracker->res_tree[type]; + + res_arr = kzalloc(count * sizeof *res_arr, GFP_KERNEL); + if (!res_arr) + return -ENOMEM; + + for (i = 0; i < count; ++i) { + res_arr[i] = alloc_tr(base + i, type, slave, extra); + if (!res_arr[i]) { + for (--i; i >= 0; --i) + kfree(res_arr[i]); + + kfree(res_arr); + return -ENOMEM; + } + } + + spin_lock_irq(mlx4_tlock(dev)); + for (i = 0; i < count; ++i) { + if (find_res(dev, base + i, type)) { + err = -EEXIST; + goto undo; + } + err = res_tracker_insert(root, res_arr[i]); + if (err) + goto undo; + list_add_tail(&res_arr[i]->list, + &tracker->slave_list[slave].res_list[type]); + } + spin_unlock_irq(mlx4_tlock(dev)); + kfree(res_arr); + + return 0; + +undo: + for (--i; i >= 0; --i) { + rb_erase(&res_arr[i]->node, root); + list_del_init(&res_arr[i]->list); + } + + spin_unlock_irq(mlx4_tlock(dev)); + + for (i = 0; i < count; ++i) + kfree(res_arr[i]); + + kfree(res_arr); + + return err; +} + +static int remove_qp_ok(struct res_qp *res) +{ + if (res->com.state == RES_QP_BUSY || atomic_read(&res->ref_count) || + !list_empty(&res->mcg_list)) { + pr_err("resource tracker: fail to remove qp, state %d, ref_count %d\n", + res->com.state, atomic_read(&res->ref_count)); + return -EBUSY; + } else if (res->com.state != RES_QP_RESERVED) { + return -EPERM; + } + + return 0; +} + +static int remove_mtt_ok(struct res_mtt *res, int order) +{ + if (res->com.state == RES_MTT_BUSY || + atomic_read(&res->ref_count)) { + printk(KERN_DEBUG "%s-%d: state %s, ref_count %d\n", + __func__, __LINE__, + mtt_states_str(res->com.state), + atomic_read(&res->ref_count)); + return -EBUSY; + } else if (res->com.state != RES_MTT_ALLOCATED) + return -EPERM; + else if (res->order != order) + return -EINVAL; + + return 0; +} + +static int remove_mpt_ok(struct res_mpt *res) +{ + if (res->com.state == RES_MPT_BUSY) + return -EBUSY; + else if (res->com.state != RES_MPT_RESERVED) + return -EPERM; + + return 0; +} + +static int remove_eq_ok(struct res_eq *res) +{ + if (res->com.state == RES_MPT_BUSY) + return -EBUSY; + else if (res->com.state != RES_MPT_RESERVED) + return -EPERM; + + return 0; +} + +static int remove_counter_ok(struct res_counter *res) +{ + if (res->com.state == RES_COUNTER_BUSY) + return -EBUSY; + else if (res->com.state != RES_COUNTER_ALLOCATED) + return -EPERM; + + return 0; +} + +static int remove_xrcdn_ok(struct res_xrcdn *res) +{ + if (res->com.state == RES_XRCD_BUSY) + return -EBUSY; + else if (res->com.state != RES_XRCD_ALLOCATED) + return -EPERM; + + return 0; +} + +static int remove_fs_rule_ok(struct res_fs_rule *res) +{ + if (res->com.state == RES_FS_RULE_BUSY) + return -EBUSY; + else if (res->com.state != RES_FS_RULE_ALLOCATED) + return -EPERM; + + return 0; +} + +static int remove_cq_ok(struct res_cq *res) +{ + if (res->com.state == RES_CQ_BUSY) + return -EBUSY; + else if (res->com.state != RES_CQ_ALLOCATED) + return -EPERM; + + return 0; +} + +static int remove_srq_ok(struct res_srq *res) +{ + if (res->com.state == RES_SRQ_BUSY) + return -EBUSY; + else if (res->com.state != RES_SRQ_ALLOCATED) + return -EPERM; + + return 0; +} + +static int remove_ok(struct res_common *res, enum mlx4_resource type, int extra) +{ + switch (type) { + case RES_QP: + return remove_qp_ok((struct res_qp *)res); + case RES_CQ: + return remove_cq_ok((struct res_cq *)res); + case RES_SRQ: + return remove_srq_ok((struct res_srq *)res); + case RES_MPT: + return remove_mpt_ok((struct res_mpt *)res); + case RES_MTT: + return remove_mtt_ok((struct res_mtt *)res, extra); + case RES_MAC: + return -ENOSYS; + case RES_EQ: + return remove_eq_ok((struct res_eq *)res); + case RES_COUNTER: + return remove_counter_ok((struct res_counter *)res); + case RES_XRCD: + return remove_xrcdn_ok((struct res_xrcdn *)res); + case RES_FS_RULE: + return remove_fs_rule_ok((struct res_fs_rule *)res); + default: + return -EINVAL; + } +} + +static int rem_res_range(struct mlx4_dev *dev, int slave, u64 base, int count, + enum mlx4_resource type, int extra) +{ + u64 i; + int err; + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_common *r; + + spin_lock_irq(mlx4_tlock(dev)); + for (i = base; i < base + count; ++i) { + r = res_tracker_lookup(&tracker->res_tree[type], i); + if (!r) { + err = -ENOENT; + goto out; + } + if (r->owner != slave) { + err = -EPERM; + goto out; + } + err = remove_ok(r, type, extra); + if (err) + goto out; + } + + for (i = base; i < base + count; ++i) { + r = res_tracker_lookup(&tracker->res_tree[type], i); + rb_erase(&r->node, &tracker->res_tree[type]); + list_del(&r->list); + kfree(r); + } + err = 0; + +out: + spin_unlock_irq(mlx4_tlock(dev)); + + return err; +} + +static int qp_res_start_move_to(struct mlx4_dev *dev, int slave, int qpn, + enum res_qp_states state, struct res_qp **qp, + int alloc) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_qp *r; + int err = 0; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[RES_QP], qpn); + if (!r) + err = -ENOENT; + else if (r->com.owner != slave) + err = -EPERM; + else { + switch (state) { + case RES_QP_BUSY: + mlx4_dbg(dev, "%s: failed RES_QP, 0x%llx\n", + __func__, (unsigned long long)r->com.res_id); + err = -EBUSY; + break; + + case RES_QP_RESERVED: + if (r->com.state == RES_QP_MAPPED && !alloc) + break; + + mlx4_dbg(dev, "failed RES_QP, 0x%llx\n", (unsigned long long)r->com.res_id); + err = -EINVAL; + break; + + case RES_QP_MAPPED: + if ((r->com.state == RES_QP_RESERVED && alloc) || + r->com.state == RES_QP_HW) + break; + else { + mlx4_dbg(dev, "failed RES_QP, 0x%llx\n", + (unsigned long long)r->com.res_id); + err = -EINVAL; + } + + break; + + case RES_QP_HW: + if (r->com.state != RES_QP_MAPPED) + err = -EINVAL; + break; + default: + err = -EINVAL; + } + + if (!err) { + r->com.from_state = r->com.state; + r->com.to_state = state; + r->com.state = RES_QP_BUSY; + if (qp) + *qp = r; + } + } + + spin_unlock_irq(mlx4_tlock(dev)); + + return err; +} + +static int mr_res_start_move_to(struct mlx4_dev *dev, int slave, int index, + enum res_mpt_states state, struct res_mpt **mpt) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_mpt *r; + int err = 0; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[RES_MPT], index); + if (!r) + err = -ENOENT; + else if (r->com.owner != slave) + err = -EPERM; + else { + switch (state) { + case RES_MPT_BUSY: + err = -EINVAL; + break; + + case RES_MPT_RESERVED: + if (r->com.state != RES_MPT_MAPPED) + err = -EINVAL; + break; + + case RES_MPT_MAPPED: + if (r->com.state != RES_MPT_RESERVED && + r->com.state != RES_MPT_HW) + err = -EINVAL; + break; + + case RES_MPT_HW: + if (r->com.state != RES_MPT_MAPPED) + err = -EINVAL; + break; + default: + err = -EINVAL; + } + + if (!err) { + r->com.from_state = r->com.state; + r->com.to_state = state; + r->com.state = RES_MPT_BUSY; + if (mpt) + *mpt = r; + } + } + + spin_unlock_irq(mlx4_tlock(dev)); + + return err; +} + +static int eq_res_start_move_to(struct mlx4_dev *dev, int slave, int index, + enum res_eq_states state, struct res_eq **eq) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_eq *r; + int err = 0; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[RES_EQ], index); + if (!r) + err = -ENOENT; + else if (r->com.owner != slave) + err = -EPERM; + else { + switch (state) { + case RES_EQ_BUSY: + err = -EINVAL; + break; + + case RES_EQ_RESERVED: + if (r->com.state != RES_EQ_HW) + err = -EINVAL; + break; + + case RES_EQ_HW: + if (r->com.state != RES_EQ_RESERVED) + err = -EINVAL; + break; + + default: + err = -EINVAL; + } + + if (!err) { + r->com.from_state = r->com.state; + r->com.to_state = state; + r->com.state = RES_EQ_BUSY; + if (eq) + *eq = r; + } + } + + spin_unlock_irq(mlx4_tlock(dev)); + + return err; +} + +static int cq_res_start_move_to(struct mlx4_dev *dev, int slave, int cqn, + enum res_cq_states state, struct res_cq **cq) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_cq *r; + int err; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[RES_CQ], cqn); + if (!r) + err = -ENOENT; + else if (r->com.owner != slave) + err = -EPERM; + else { + switch (state) { + case RES_CQ_BUSY: + err = -EBUSY; + break; + + case RES_CQ_ALLOCATED: + if (r->com.state != RES_CQ_HW) + err = -EINVAL; + else if (atomic_read(&r->ref_count)) + err = -EBUSY; + else + err = 0; + break; + + case RES_CQ_HW: + if (r->com.state != RES_CQ_ALLOCATED) + err = -EINVAL; + else + err = 0; + break; + + default: + err = -EINVAL; + } + + if (!err) { + r->com.from_state = r->com.state; + r->com.to_state = state; + r->com.state = RES_CQ_BUSY; + if (cq) + *cq = r; + } + } + + spin_unlock_irq(mlx4_tlock(dev)); + + return err; +} + +static int srq_res_start_move_to(struct mlx4_dev *dev, int slave, int index, + enum res_srq_states state, struct res_srq **srq) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_srq *r; + int err = 0; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[RES_SRQ], index); + if (!r) + err = -ENOENT; + else if (r->com.owner != slave) + err = -EPERM; + else { + switch (state) { + case RES_SRQ_BUSY: + err = -EINVAL; + break; + + case RES_SRQ_ALLOCATED: + if (r->com.state != RES_SRQ_HW) + err = -EINVAL; + else if (atomic_read(&r->ref_count)) + err = -EBUSY; + break; + + case RES_SRQ_HW: + if (r->com.state != RES_SRQ_ALLOCATED) + err = -EINVAL; + break; + + default: + err = -EINVAL; + } + + if (!err) { + r->com.from_state = r->com.state; + r->com.to_state = state; + r->com.state = RES_SRQ_BUSY; + if (srq) + *srq = r; + } + } + + spin_unlock_irq(mlx4_tlock(dev)); + + return err; +} + +static void res_abort_move(struct mlx4_dev *dev, int slave, + enum mlx4_resource type, int id) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_common *r; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[type], id); + if (r && (r->owner == slave)) + r->state = r->from_state; + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void res_end_move(struct mlx4_dev *dev, int slave, + enum mlx4_resource type, int id) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_common *r; + + spin_lock_irq(mlx4_tlock(dev)); + r = res_tracker_lookup(&tracker->res_tree[type], id); + if (r && (r->owner == slave)) + r->state = r->to_state; + spin_unlock_irq(mlx4_tlock(dev)); +} + +static int valid_reserved(struct mlx4_dev *dev, int slave, int qpn) +{ + return mlx4_is_qp_reserved(dev, qpn) && + (mlx4_is_master(dev) || mlx4_is_guest_proxy(dev, slave, qpn)); +} + +static int fw_reserved(struct mlx4_dev *dev, int qpn) +{ + return qpn < dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW]; +} + +static int qp_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int err; + int count; + int align; + int base; + int qpn; + u8 flags; + + switch (op) { + case RES_OP_RESERVE: + count = get_param_l(&in_param) & 0xffffff; + flags = get_param_l(&in_param) >> 24; + align = get_param_h(&in_param); + err = mlx4_grant_resource(dev, slave, RES_QP, count, 0); + if (err) + return err; + + err = __mlx4_qp_reserve_range(dev, count, align, &base, flags); + if (err) { + mlx4_release_resource(dev, slave, RES_QP, count, 0); + return err; + } + + err = add_res_range(dev, slave, base, count, RES_QP, 0); + if (err) { + mlx4_release_resource(dev, slave, RES_QP, count, 0); + __mlx4_qp_release_range(dev, base, count); + return err; + } + set_param_l(out_param, base); + break; + case RES_OP_MAP_ICM: + qpn = get_param_l(&in_param) & 0x7fffff; + if (valid_reserved(dev, slave, qpn)) { + err = add_res_range(dev, slave, qpn, 1, RES_QP, 0); + if (err) + return err; + } + + err = qp_res_start_move_to(dev, slave, qpn, RES_QP_MAPPED, + NULL, 1); + if (err) + return err; + + if (!fw_reserved(dev, qpn)) { + err = __mlx4_qp_alloc_icm(dev, qpn); + if (err) { + res_abort_move(dev, slave, RES_QP, qpn); + return err; + } + } + + res_end_move(dev, slave, RES_QP, qpn); + break; + + default: + err = -EINVAL; + break; + } + return err; +} + +static int mtt_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int err = -EINVAL; + int base; + int order; + + if (op != RES_OP_RESERVE_AND_MAP) + return err; + + order = get_param_l(&in_param); + + err = mlx4_grant_resource(dev, slave, RES_MTT, 1 << order, 0); + if (err) + return err; + + base = __mlx4_alloc_mtt_range(dev, order); + if (base == -1) { + mlx4_release_resource(dev, slave, RES_MTT, 1 << order, 0); + return -ENOMEM; + } + + err = add_res_range(dev, slave, base, 1, RES_MTT, order); + if (err) { + mlx4_release_resource(dev, slave, RES_MTT, 1 << order, 0); + __mlx4_free_mtt_range(dev, base, order); + } else + set_param_l(out_param, base); + + return err; +} + +static int mpt_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int err = -EINVAL; + int index; + int id; + struct res_mpt *mpt; + + switch (op) { + case RES_OP_RESERVE: + err = mlx4_grant_resource(dev, slave, RES_MPT, 1, 0); + if (err) + break; + + index = __mlx4_mpt_reserve(dev); + if (index == -1) { + mlx4_release_resource(dev, slave, RES_MPT, 1, 0); + break; + } + id = index & mpt_mask(dev); + + err = add_res_range(dev, slave, id, 1, RES_MPT, index); + if (err) { + mlx4_release_resource(dev, slave, RES_MPT, 1, 0); + __mlx4_mpt_release(dev, index); + break; + } + set_param_l(out_param, index); + break; + case RES_OP_MAP_ICM: + index = get_param_l(&in_param); + id = index & mpt_mask(dev); + err = mr_res_start_move_to(dev, slave, id, + RES_MPT_MAPPED, &mpt); + if (err) + return err; + + err = __mlx4_mpt_alloc_icm(dev, mpt->key); + if (err) { + res_abort_move(dev, slave, RES_MPT, id); + return err; + } + + res_end_move(dev, slave, RES_MPT, id); + break; + } + return err; +} + +static int cq_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int cqn; + int err; + + switch (op) { + case RES_OP_RESERVE_AND_MAP: + err = mlx4_grant_resource(dev, slave, RES_CQ, 1, 0); + if (err) + break; + + err = __mlx4_cq_alloc_icm(dev, &cqn); + if (err) { + mlx4_release_resource(dev, slave, RES_CQ, 1, 0); + break; + } + + err = add_res_range(dev, slave, cqn, 1, RES_CQ, 0); + if (err) { + mlx4_release_resource(dev, slave, RES_CQ, 1, 0); + __mlx4_cq_free_icm(dev, cqn); + break; + } + + set_param_l(out_param, cqn); + break; + + default: + err = -EINVAL; + } + + return err; +} + +static int srq_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int srqn; + int err; + + switch (op) { + case RES_OP_RESERVE_AND_MAP: + err = mlx4_grant_resource(dev, slave, RES_SRQ, 1, 0); + if (err) + break; + + err = __mlx4_srq_alloc_icm(dev, &srqn); + if (err) { + mlx4_release_resource(dev, slave, RES_SRQ, 1, 0); + break; + } + + err = add_res_range(dev, slave, srqn, 1, RES_SRQ, 0); + if (err) { + mlx4_release_resource(dev, slave, RES_SRQ, 1, 0); + __mlx4_srq_free_icm(dev, srqn); + break; + } + + set_param_l(out_param, srqn); + break; + + default: + err = -EINVAL; + } + + return err; +} + +static int mac_find_smac_ix_in_slave(struct mlx4_dev *dev, int slave, int port, + u8 smac_index, u64 *mac) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *mac_list = + &tracker->slave_list[slave].res_list[RES_MAC]; + struct mac_res *res, *tmp; + + list_for_each_entry_safe(res, tmp, mac_list, list) { + if (res->smac_index == smac_index && res->port == (u8) port) { + *mac = res->mac; + return 0; + } + } + return -ENOENT; +} + +static int mac_add_to_slave(struct mlx4_dev *dev, int slave, u64 mac, int port, u8 smac_index) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *mac_list = + &tracker->slave_list[slave].res_list[RES_MAC]; + struct mac_res *res, *tmp; + + list_for_each_entry_safe(res, tmp, mac_list, list) { + if (res->mac == mac && res->port == (u8) port) { + /* mac found. update ref count */ + ++res->ref_count; + return 0; + } + } + + if (mlx4_grant_resource(dev, slave, RES_MAC, 1, port)) + return -EINVAL; + res = kzalloc(sizeof *res, GFP_KERNEL); + if (!res) { + mlx4_release_resource(dev, slave, RES_MAC, 1, port); + return -ENOMEM; + } + res->mac = mac; + res->port = (u8) port; + res->smac_index = smac_index; + res->ref_count = 1; + list_add_tail(&res->list, + &tracker->slave_list[slave].res_list[RES_MAC]); + return 0; +} + + +static void mac_del_from_slave(struct mlx4_dev *dev, int slave, u64 mac, + int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *mac_list = + &tracker->slave_list[slave].res_list[RES_MAC]; + struct mac_res *res, *tmp; + + list_for_each_entry_safe(res, tmp, mac_list, list) { + if (res->mac == mac && res->port == (u8) port) { + if (!--res->ref_count) { + list_del(&res->list); + mlx4_release_resource(dev, slave, RES_MAC, 1, port); + kfree(res); + } + break; + } + } +} + +static void rem_slave_macs(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *mac_list = + &tracker->slave_list[slave].res_list[RES_MAC]; + struct mac_res *res, *tmp; + int i; + + list_for_each_entry_safe(res, tmp, mac_list, list) { + list_del(&res->list); + /* dereference the mac the num times the slave referenced it */ + for (i = 0; i < res->ref_count; i++) + __mlx4_unregister_mac(dev, res->port, res->mac); + mlx4_release_resource(dev, slave, RES_MAC, 1, res->port); + kfree(res); + } +} + +static int mac_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param, int in_port) +{ + int err = -EINVAL; + int port; + u64 mac; + u8 smac_index = 0; + + if (op != RES_OP_RESERVE_AND_MAP) + return err; + + port = !in_port ? get_param_l(out_param) : in_port; + mac = in_param; + + err = __mlx4_register_mac(dev, port, mac); + if (err >= 0) { + smac_index = err; + set_param_l(out_param, err); + err = 0; + } + + if (!err) { + err = mac_add_to_slave(dev, slave, mac, port, smac_index); + if (err) + __mlx4_unregister_mac(dev, port, mac); + } + return err; +} + +static int vlan_add_to_slave(struct mlx4_dev *dev, int slave, u16 vlan, + int port, int vlan_index) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *vlan_list = + &tracker->slave_list[slave].res_list[RES_VLAN]; + struct vlan_res *res, *tmp; + + list_for_each_entry_safe(res, tmp, vlan_list, list) { + if (res->vlan == vlan && res->port == (u8) port) { + /* vlan found. update ref count */ + ++res->ref_count; + return 0; + } + } + + if (mlx4_grant_resource(dev, slave, RES_VLAN, 1, port)) + return -EINVAL; + res = kzalloc(sizeof(*res), GFP_KERNEL); + if (!res) { + mlx4_release_resource(dev, slave, RES_VLAN, 1, port); + return -ENOMEM; + } + res->vlan = vlan; + res->port = (u8) port; + res->vlan_index = vlan_index; + res->ref_count = 1; + list_add_tail(&res->list, + &tracker->slave_list[slave].res_list[RES_VLAN]); + return 0; +} + + +static void vlan_del_from_slave(struct mlx4_dev *dev, int slave, u16 vlan, + int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *vlan_list = + &tracker->slave_list[slave].res_list[RES_VLAN]; + struct vlan_res *res, *tmp; + + list_for_each_entry_safe(res, tmp, vlan_list, list) { + if (res->vlan == vlan && res->port == (u8) port) { + if (!--res->ref_count) { + list_del(&res->list); + mlx4_release_resource(dev, slave, RES_VLAN, + 1, port); + kfree(res); + } + break; + } + } +} + +static void rem_slave_vlans(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *vlan_list = + &tracker->slave_list[slave].res_list[RES_VLAN]; + struct vlan_res *res, *tmp; + int i; + + list_for_each_entry_safe(res, tmp, vlan_list, list) { + list_del(&res->list); + /* dereference the vlan the num times the slave referenced it */ + for (i = 0; i < res->ref_count; i++) + __mlx4_unregister_vlan(dev, res->port, res->vlan); + mlx4_release_resource(dev, slave, RES_VLAN, 1, res->port); + kfree(res); + } +} + +static int vlan_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param, int in_port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; + int err = -EINVAL; + u16 vlan; + int vlan_index; + int port; + + port = !in_port ? get_param_l(out_param) : in_port; + + if (!port) + return err; + + if (op != RES_OP_RESERVE_AND_MAP) + return err; + + /* upstream kernels had NOP for reg/unreg vlan. Continue this. */ + if (!in_port && port > 0 && port <= dev->caps.num_ports) { + slave_state[slave].old_vlan_api = true; + return 0; + } + + vlan = (u16) in_param; + + err = __mlx4_register_vlan(dev, port, vlan, &vlan_index); + if (!err) { + set_param_l(out_param, (u32) vlan_index); + err = vlan_add_to_slave(dev, slave, vlan, port, vlan_index); + if (err) + __mlx4_unregister_vlan(dev, port, vlan); + } + return err; +} + +static int counter_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param, int port) +{ + u32 index; + int err; + + if (op != RES_OP_RESERVE) + return -EINVAL; + + err = __mlx4_counter_alloc(dev, slave, port, &index); + if (!err) + set_param_l(out_param, index); + + return err; +} + +static int xrcdn_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + u32 xrcdn; + int err; + + if (op != RES_OP_RESERVE) + return -EINVAL; + + err = __mlx4_xrcd_alloc(dev, &xrcdn); + if (err) + return err; + + err = add_res_range(dev, slave, xrcdn, 1, RES_XRCD, 0); + if (err) + __mlx4_xrcd_free(dev, xrcdn); + else + set_param_l(out_param, xrcdn); + + return err; +} + +int mlx4_ALLOC_RES_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int alop = vhcr->op_modifier; + + switch (vhcr->in_modifier & 0xFF) { + case RES_QP: + err = qp_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_MTT: + err = mtt_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_MPT: + err = mpt_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_CQ: + err = cq_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_SRQ: + err = srq_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_MAC: + err = mac_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param, + (vhcr->in_modifier >> 8) & 0xFF); + break; + + case RES_VLAN: + err = vlan_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param, + (vhcr->in_modifier >> 8) & 0xFF); + break; + + case RES_COUNTER: + err = counter_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param, + (vhcr->in_modifier >> 8) & 0xFF); + break; + + case RES_XRCD: + err = xrcdn_alloc_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + default: + err = -EINVAL; + break; + } + + return err; +} + +static int qp_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param) +{ + int err; + int count; + int base; + int qpn; + + switch (op) { + case RES_OP_RESERVE: + base = get_param_l(&in_param) & 0x7fffff; + count = get_param_h(&in_param); + err = rem_res_range(dev, slave, base, count, RES_QP, 0); + if (err) + break; + mlx4_release_resource(dev, slave, RES_QP, count, 0); + __mlx4_qp_release_range(dev, base, count); + break; + case RES_OP_MAP_ICM: + qpn = get_param_l(&in_param) & 0x7fffff; + err = qp_res_start_move_to(dev, slave, qpn, RES_QP_RESERVED, + NULL, 0); + if (err) + return err; + + if (!fw_reserved(dev, qpn)) + __mlx4_qp_free_icm(dev, qpn); + + res_end_move(dev, slave, RES_QP, qpn); + + if (valid_reserved(dev, slave, qpn)) + err = rem_res_range(dev, slave, qpn, 1, RES_QP, 0); + break; + default: + err = -EINVAL; + break; + } + return err; +} + +static int mtt_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int err = -EINVAL; + int base; + int order; + + if (op != RES_OP_RESERVE_AND_MAP) + return err; + + base = get_param_l(&in_param); + order = get_param_h(&in_param); + err = rem_res_range(dev, slave, base, 1, RES_MTT, order); + if (!err) { + mlx4_release_resource(dev, slave, RES_MTT, 1 << order, 0); + __mlx4_free_mtt_range(dev, base, order); + } + return err; +} + +static int mpt_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param) +{ + int err = -EINVAL; + int index; + int id; + struct res_mpt *mpt; + + switch (op) { + case RES_OP_RESERVE: + index = get_param_l(&in_param); + id = index & mpt_mask(dev); + err = get_res(dev, slave, id, RES_MPT, &mpt); + if (err) + break; + index = mpt->key; + put_res(dev, slave, id, RES_MPT); + + err = rem_res_range(dev, slave, id, 1, RES_MPT, 0); + if (err) + break; + mlx4_release_resource(dev, slave, RES_MPT, 1, 0); + __mlx4_mpt_release(dev, index); + break; + case RES_OP_MAP_ICM: + index = get_param_l(&in_param); + id = index & mpt_mask(dev); + err = mr_res_start_move_to(dev, slave, id, + RES_MPT_RESERVED, &mpt); + if (err) + return err; + + __mlx4_mpt_free_icm(dev, mpt->key); + res_end_move(dev, slave, RES_MPT, id); + return err; + break; + default: + err = -EINVAL; + break; + } + return err; +} + +static int cq_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int cqn; + int err; + + switch (op) { + case RES_OP_RESERVE_AND_MAP: + cqn = get_param_l(&in_param); + err = rem_res_range(dev, slave, cqn, 1, RES_CQ, 0); + if (err) + break; + + mlx4_release_resource(dev, slave, RES_CQ, 1, 0); + __mlx4_cq_free_icm(dev, cqn); + break; + + default: + err = -EINVAL; + break; + } + + return err; +} + +static int srq_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int srqn; + int err; + + switch (op) { + case RES_OP_RESERVE_AND_MAP: + srqn = get_param_l(&in_param); + err = rem_res_range(dev, slave, srqn, 1, RES_SRQ, 0); + if (err) + break; + + mlx4_release_resource(dev, slave, RES_SRQ, 1, 0); + __mlx4_srq_free_icm(dev, srqn); + break; + + default: + err = -EINVAL; + break; + } + + return err; +} + +static int mac_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param, int in_port) +{ + int port; + int err = 0; + + switch (op) { + case RES_OP_RESERVE_AND_MAP: + port = !in_port ? get_param_l(out_param) : in_port; + mac_del_from_slave(dev, slave, in_param, port); + __mlx4_unregister_mac(dev, port, in_param); + break; + default: + err = -EINVAL; + break; + } + + return err; + +} + +static int vlan_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param, int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; + int err = 0; + + switch (op) { + case RES_OP_RESERVE_AND_MAP: + if (slave_state[slave].old_vlan_api == true) + return 0; + if (!port) + return -EINVAL; + vlan_del_from_slave(dev, slave, in_param, port); + __mlx4_unregister_vlan(dev, port, in_param); + break; + default: + err = -EINVAL; + break; + } + + return err; +} + +static int counter_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param, int port) +{ + int index; + + if (op != RES_OP_RESERVE) + return -EINVAL; + + index = get_param_l(&in_param); + + __mlx4_counter_free(dev, slave, port, index); + + return 0; +} + +static int xrcdn_free_res(struct mlx4_dev *dev, int slave, int op, int cmd, + u64 in_param, u64 *out_param) +{ + int xrcdn; + int err; + + if (op != RES_OP_RESERVE) + return -EINVAL; + + xrcdn = get_param_l(&in_param); + err = rem_res_range(dev, slave, xrcdn, 1, RES_XRCD, 0); + if (err) + return err; + + __mlx4_xrcd_free(dev, xrcdn); + + return err; +} + +int mlx4_FREE_RES_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err = -EINVAL; + int alop = vhcr->op_modifier; + + switch (vhcr->in_modifier & 0xFF) { + case RES_QP: + err = qp_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param); + break; + + case RES_MTT: + err = mtt_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_MPT: + err = mpt_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param); + break; + + case RES_CQ: + err = cq_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_SRQ: + err = srq_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + break; + + case RES_MAC: + err = mac_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param, + (vhcr->in_modifier >> 8) & 0xFF); + break; + + case RES_VLAN: + err = vlan_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param, + (vhcr->in_modifier >> 8) & 0xFF); + break; + + case RES_COUNTER: + err = counter_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param, + (vhcr->in_modifier >> 8) & 0xFF); + break; + + case RES_XRCD: + err = xrcdn_free_res(dev, slave, vhcr->op_modifier, alop, + vhcr->in_param, &vhcr->out_param); + + default: + break; + } + return err; +} + +/* ugly but other choices are uglier */ +static int mr_phys_mpt(struct mlx4_mpt_entry *mpt) +{ + return (be32_to_cpu(mpt->flags) >> 9) & 1; +} + +static int mr_get_mtt_addr(struct mlx4_mpt_entry *mpt) +{ + return (int)be64_to_cpu(mpt->mtt_addr) & 0xfffffff8; +} + +static int mr_get_mtt_size(struct mlx4_mpt_entry *mpt) +{ + return be32_to_cpu(mpt->mtt_sz); +} + +static u32 mr_get_pd(struct mlx4_mpt_entry *mpt) +{ + return be32_to_cpu(mpt->pd_flags) & 0x00ffffff; +} + +static int mr_is_fmr(struct mlx4_mpt_entry *mpt) +{ + return be32_to_cpu(mpt->pd_flags) & MLX4_MPT_PD_FLAG_FAST_REG; +} + +static int mr_is_bind_enabled(struct mlx4_mpt_entry *mpt) +{ + return be32_to_cpu(mpt->flags) & MLX4_MPT_FLAG_BIND_ENABLE; +} + +static int mr_is_region(struct mlx4_mpt_entry *mpt) +{ + return be32_to_cpu(mpt->flags) & MLX4_MPT_FLAG_REGION; +} + +static int qp_get_mtt_addr(struct mlx4_qp_context *qpc) +{ + return be32_to_cpu(qpc->mtt_base_addr_l) & 0xfffffff8; +} + +static int srq_get_mtt_addr(struct mlx4_srq_context *srqc) +{ + return be32_to_cpu(srqc->mtt_base_addr_l) & 0xfffffff8; +} + +static int qp_get_mtt_size(struct mlx4_qp_context *qpc) +{ + int page_shift = (qpc->log_page_size & 0x3f) + 12; + int log_sq_size = (qpc->sq_size_stride >> 3) & 0xf; + int log_sq_sride = qpc->sq_size_stride & 7; + int log_rq_size = (qpc->rq_size_stride >> 3) & 0xf; + int log_rq_stride = qpc->rq_size_stride & 7; + int srq = (be32_to_cpu(qpc->srqn) >> 24) & 1; + int rss = (be32_to_cpu(qpc->flags) >> 13) & 1; + u32 ts = (be32_to_cpu(qpc->flags) >> 16) & 0xff; + int xrc = (ts == MLX4_QP_ST_XRC) ? 1 : 0; + int sq_size; + int rq_size; + int total_pages; + int total_mem; + int page_offset = (be32_to_cpu(qpc->params2) >> 6) & 0x3f; + + sq_size = 1 << (log_sq_size + log_sq_sride + 4); + rq_size = (srq|rss|xrc) ? 0 : (1 << (log_rq_size + log_rq_stride + 4)); + total_mem = sq_size + rq_size; + total_pages = + roundup_pow_of_two((total_mem + (page_offset << 6)) >> + page_shift); + + return total_pages; +} + +static int check_mtt_range(struct mlx4_dev *dev, int slave, int start, + int size, struct res_mtt *mtt) +{ + int res_start = mtt->com.res_id; + int res_size = (1 << mtt->order); + + if (start < res_start || start + size > res_start + res_size) + return -EPERM; + return 0; +} + +int mlx4_SW2HW_MPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int index = vhcr->in_modifier; + struct res_mtt *mtt; + struct res_mpt *mpt; + int mtt_base = mr_get_mtt_addr(inbox->buf) / dev->caps.mtt_entry_sz; + int phys; + int id; + u32 pd; + int pd_slave; + + id = index & mpt_mask(dev); + err = mr_res_start_move_to(dev, slave, id, RES_MPT_HW, &mpt); + if (err) + return err; + + /* Currently disable memory windows since this feature isn't tested yet + * under virtualization. + */ + if (!mr_is_region(inbox->buf)) { + err = -ENOSYS; + goto ex_abort; + } + + /* Make sure that the PD bits related to the slave id are zeros. */ + pd = mr_get_pd(inbox->buf); + pd_slave = (pd >> 17) & 0x7f; + if (pd_slave != 0 && pd_slave != slave) { + err = -EPERM; + goto ex_abort; + } + + if (mr_is_fmr(inbox->buf)) { + /* FMR and Bind Enable are forbidden in slave devices. */ + if (mr_is_bind_enabled(inbox->buf)) { + err = -EPERM; + goto ex_abort; + } + /* FMR and Memory Windows are also forbidden. */ + if (!mr_is_region(inbox->buf)) { + err = -EPERM; + goto ex_abort; + } + } + + phys = mr_phys_mpt(inbox->buf); + if (!phys) { + err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); + if (err) + goto ex_abort; + + err = check_mtt_range(dev, slave, mtt_base, + mr_get_mtt_size(inbox->buf), mtt); + if (err) + goto ex_put; + + mpt->mtt = mtt; + } + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_put; + + if (!phys) { + atomic_inc(&mtt->ref_count); + put_res(dev, slave, mtt->com.res_id, RES_MTT); + } + + res_end_move(dev, slave, RES_MPT, id); + return 0; + +ex_put: + if (!phys) + put_res(dev, slave, mtt->com.res_id, RES_MTT); +ex_abort: + res_abort_move(dev, slave, RES_MPT, id); + + return err; +} + +int mlx4_HW2SW_MPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int index = vhcr->in_modifier; + struct res_mpt *mpt; + int id; + + id = index & mpt_mask(dev); + err = mr_res_start_move_to(dev, slave, id, RES_MPT_MAPPED, &mpt); + if (err) + return err; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_abort; + + if (mpt->mtt) + atomic_dec(&mpt->mtt->ref_count); + + res_end_move(dev, slave, RES_MPT, id); + return 0; + +ex_abort: + res_abort_move(dev, slave, RES_MPT, id); + + return err; +} + +int mlx4_QUERY_MPT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int index = vhcr->in_modifier; + struct res_mpt *mpt; + int id; + + id = index & mpt_mask(dev); + err = get_res(dev, slave, id, RES_MPT, &mpt); + if (err) + return err; + + if (mpt->com.from_state != RES_MPT_HW) { + err = -EBUSY; + goto out; + } + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + +out: + put_res(dev, slave, id, RES_MPT); + return err; +} + +static int qp_get_rcqn(struct mlx4_qp_context *qpc) +{ + return be32_to_cpu(qpc->cqn_recv) & 0xffffff; +} + +static int qp_get_scqn(struct mlx4_qp_context *qpc) +{ + return be32_to_cpu(qpc->cqn_send) & 0xffffff; +} + +static u32 qp_get_srqn(struct mlx4_qp_context *qpc) +{ + return be32_to_cpu(qpc->srqn) & 0x1ffffff; +} + +static void adjust_proxy_tun_qkey(struct mlx4_dev *dev, struct mlx4_vhcr *vhcr, + struct mlx4_qp_context *context) +{ + u32 qpn = vhcr->in_modifier & 0xffffff; + u32 qkey = 0; + + if (mlx4_get_parav_qkey(dev, qpn, &qkey)) + return; + + /* adjust qkey in qp context */ + context->qkey = cpu_to_be32(qkey); +} + +int mlx4_RST2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int qpn = vhcr->in_modifier & 0x7fffff; + struct res_mtt *mtt; + struct res_qp *qp; + struct mlx4_qp_context *qpc = inbox->buf + 8; + int mtt_base = qp_get_mtt_addr(qpc) / dev->caps.mtt_entry_sz; + int mtt_size = qp_get_mtt_size(qpc); + struct res_cq *rcq; + struct res_cq *scq; + int rcqn = qp_get_rcqn(qpc); + int scqn = qp_get_scqn(qpc); + u32 srqn = qp_get_srqn(qpc) & 0xffffff; + int use_srq = (qp_get_srqn(qpc) >> 24) & 1; + struct res_srq *srq; + int local_qpn = be32_to_cpu(qpc->local_qpn) & 0xffffff; + + err = qp_res_start_move_to(dev, slave, qpn, RES_QP_HW, &qp, 0); + if (err) + return err; + qp->local_qpn = local_qpn; + qp->sched_queue = 0; + qp->param3 = 0; + qp->vlan_control = 0; + qp->fvl_rx = 0; + qp->pri_path_fl = 0; + qp->vlan_index = 0; + qp->feup = 0; + qp->qpc_flags = be32_to_cpu(qpc->flags); + + err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); + if (err) + goto ex_abort; + + err = check_mtt_range(dev, slave, mtt_base, mtt_size, mtt); + if (err) + goto ex_put_mtt; + + err = get_res(dev, slave, rcqn, RES_CQ, &rcq); + if (err) + goto ex_put_mtt; + + if (scqn != rcqn) { + err = get_res(dev, slave, scqn, RES_CQ, &scq); + if (err) + goto ex_put_rcq; + } else + scq = rcq; + + if (use_srq) { + err = get_res(dev, slave, srqn, RES_SRQ, &srq); + if (err) + goto ex_put_scq; + } + + adjust_proxy_tun_qkey(dev, vhcr, qpc); + update_pkey_index(dev, slave, inbox); + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_put_srq; + atomic_inc(&mtt->ref_count); + qp->mtt = mtt; + atomic_inc(&rcq->ref_count); + qp->rcq = rcq; + atomic_inc(&scq->ref_count); + qp->scq = scq; + + if (scqn != rcqn) + put_res(dev, slave, scqn, RES_CQ); + + if (use_srq) { + atomic_inc(&srq->ref_count); + put_res(dev, slave, srqn, RES_SRQ); + qp->srq = srq; + } + put_res(dev, slave, rcqn, RES_CQ); + put_res(dev, slave, mtt_base, RES_MTT); + res_end_move(dev, slave, RES_QP, qpn); + + return 0; + +ex_put_srq: + if (use_srq) + put_res(dev, slave, srqn, RES_SRQ); +ex_put_scq: + if (scqn != rcqn) + put_res(dev, slave, scqn, RES_CQ); +ex_put_rcq: + put_res(dev, slave, rcqn, RES_CQ); +ex_put_mtt: + put_res(dev, slave, mtt_base, RES_MTT); +ex_abort: + res_abort_move(dev, slave, RES_QP, qpn); + + return err; +} + +static int eq_get_mtt_addr(struct mlx4_eq_context *eqc) +{ + return be32_to_cpu(eqc->mtt_base_addr_l) & 0xfffffff8; +} + +static int eq_get_mtt_size(struct mlx4_eq_context *eqc) +{ + int log_eq_size = eqc->log_eq_size & 0x1f; + int page_shift = (eqc->log_page_size & 0x3f) + 12; + + if (log_eq_size + 5 < page_shift) + return 1; + + return 1 << (log_eq_size + 5 - page_shift); +} + +static int cq_get_mtt_addr(struct mlx4_cq_context *cqc) +{ + return be32_to_cpu(cqc->mtt_base_addr_l) & 0xfffffff8; +} + +static int cq_get_mtt_size(struct mlx4_cq_context *cqc) +{ + int log_cq_size = (be32_to_cpu(cqc->logsize_usrpage) >> 24) & 0x1f; + int page_shift = (cqc->log_page_size & 0x3f) + 12; + + if (log_cq_size + 5 < page_shift) + return 1; + + return 1 << (log_cq_size + 5 - page_shift); +} + +int mlx4_SW2HW_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int eqn = vhcr->in_modifier; + int res_id = (slave << 8) | eqn; + struct mlx4_eq_context *eqc = inbox->buf; + int mtt_base = eq_get_mtt_addr(eqc) / dev->caps.mtt_entry_sz; + int mtt_size = eq_get_mtt_size(eqc); + struct res_eq *eq; + struct res_mtt *mtt; + + err = add_res_range(dev, slave, res_id, 1, RES_EQ, 0); + if (err) + return err; + err = eq_res_start_move_to(dev, slave, res_id, RES_EQ_HW, &eq); + if (err) + goto out_add; + + err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); + if (err) + goto out_move; + + err = check_mtt_range(dev, slave, mtt_base, mtt_size, mtt); + if (err) + goto out_put; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto out_put; + + atomic_inc(&mtt->ref_count); + eq->mtt = mtt; + put_res(dev, slave, mtt->com.res_id, RES_MTT); + res_end_move(dev, slave, RES_EQ, res_id); + return 0; + +out_put: + put_res(dev, slave, mtt->com.res_id, RES_MTT); +out_move: + res_abort_move(dev, slave, RES_EQ, res_id); +out_add: + rem_res_range(dev, slave, res_id, 1, RES_EQ, 0); + return err; +} + +static int get_containing_mtt(struct mlx4_dev *dev, int slave, int start, + int len, struct res_mtt **res) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct res_mtt *mtt; + int err = -EINVAL; + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry(mtt, &tracker->slave_list[slave].res_list[RES_MTT], + com.list) { + if (!check_mtt_range(dev, slave, start, len, mtt)) { + *res = mtt; + mtt->com.from_state = mtt->com.state; + mtt->com.state = RES_MTT_BUSY; + err = 0; + break; + } + } + spin_unlock_irq(mlx4_tlock(dev)); + + return err; +} + +static int verify_qp_parameters(struct mlx4_dev *dev, + struct mlx4_cmd_mailbox *inbox, + enum qp_transition transition, u8 slave) +{ + u32 qp_type; + struct mlx4_qp_context *qp_ctx; + enum mlx4_qp_optpar optpar; + int port; + int num_gids; + + qp_ctx = inbox->buf + 8; + qp_type = (be32_to_cpu(qp_ctx->flags) >> 16) & 0xff; + optpar = be32_to_cpu(*(__be32 *) inbox->buf); + + switch (qp_type) { + case MLX4_QP_ST_RC: + case MLX4_QP_ST_UC: + switch (transition) { + case QP_TRANS_INIT2RTR: + case QP_TRANS_RTR2RTS: + case QP_TRANS_RTS2RTS: + case QP_TRANS_SQD2SQD: + case QP_TRANS_SQD2RTS: + if (slave != mlx4_master_func_num(dev)) + if (optpar & MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH) { + port = (qp_ctx->pri_path.sched_queue >> 6 & 1) + 1; + if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB) + num_gids = mlx4_get_slave_num_gids(dev, slave); + else + num_gids = 1; + if (qp_ctx->pri_path.mgid_index >= num_gids) + return -EINVAL; + } + if (optpar & MLX4_QP_OPTPAR_ALT_ADDR_PATH) { + port = (qp_ctx->alt_path.sched_queue >> 6 & 1) + 1; + if (dev->caps.port_mask[port] != MLX4_PORT_TYPE_IB) + num_gids = mlx4_get_slave_num_gids(dev, slave); + else + num_gids = 1; + if (qp_ctx->alt_path.mgid_index >= num_gids) + return -EINVAL; + } + break; + default: + break; + } + + break; + default: + break; + } + + return 0; +} + +int mlx4_WRITE_MTT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_mtt mtt; + __be64 *page_list = inbox->buf; + u64 *pg_list = (u64 *)page_list; + int i; + struct res_mtt *rmtt = NULL; + int start = be64_to_cpu(page_list[0]); + int npages = vhcr->in_modifier; + int err; + + err = get_containing_mtt(dev, slave, start, npages, &rmtt); + if (err) + return err; + + /* Call the SW implementation of write_mtt: + * - Prepare a dummy mtt struct + * - Translate inbox contents to simple addresses in host endianess */ + mtt.offset = 0; /* TBD this is broken but I don't handle it since + we don't really use it */ + mtt.order = 0; + mtt.page_shift = 0; + for (i = 0; i < npages; ++i) + pg_list[i + 2] = (be64_to_cpu(page_list[i + 2]) & ~1ULL); + + err = __mlx4_write_mtt(dev, &mtt, be64_to_cpu(page_list[0]), npages, + ((u64 *)page_list + 2)); + + if (rmtt) + put_res(dev, slave, rmtt->com.res_id, RES_MTT); + + return err; +} + +int mlx4_HW2SW_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int eqn = vhcr->in_modifier; + int res_id = eqn | (slave << 8); + struct res_eq *eq; + int err; + + err = eq_res_start_move_to(dev, slave, res_id, RES_EQ_RESERVED, &eq); + if (err) + return err; + + err = get_res(dev, slave, eq->mtt->com.res_id, RES_MTT, NULL); + if (err) + goto ex_abort; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_put; + + atomic_dec(&eq->mtt->ref_count); + put_res(dev, slave, eq->mtt->com.res_id, RES_MTT); + res_end_move(dev, slave, RES_EQ, res_id); + rem_res_range(dev, slave, res_id, 1, RES_EQ, 0); + + return 0; + +ex_put: + put_res(dev, slave, eq->mtt->com.res_id, RES_MTT); +ex_abort: + res_abort_move(dev, slave, RES_EQ, res_id); + + return err; +} + +int mlx4_GEN_EQE(struct mlx4_dev *dev, int slave, struct mlx4_eqe *eqe) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_slave_event_eq_info *event_eq; + struct mlx4_cmd_mailbox *mailbox; + u32 in_modifier = 0; + int err; + int res_id; + struct res_eq *req; + + if (!priv->mfunc.master.slave_state) + return -EINVAL; + + /* check for slave valid, slave not PF, and slave active */ + if (slave < 0 || slave >= dev->num_slaves || + slave == dev->caps.function || + !priv->mfunc.master.slave_state[slave].active) + return 0; + + event_eq = &priv->mfunc.master.slave_state[slave].event_eq[eqe->type]; + + /* Create the event only if the slave is registered */ + if (event_eq->eqn < 0) + return 0; + + mutex_lock(&priv->mfunc.master.gen_eqe_mutex[slave]); + res_id = (slave << 8) | event_eq->eqn; + err = get_res(dev, slave, res_id, RES_EQ, &req); + if (err) + goto unlock; + + if (req->com.from_state != RES_EQ_HW) { + err = -EINVAL; + goto put; + } + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + err = PTR_ERR(mailbox); + goto put; + } + + if (eqe->type == MLX4_EVENT_TYPE_CMD) { + ++event_eq->token; + eqe->event.cmd.token = cpu_to_be16(event_eq->token); + } + + memcpy(mailbox->buf, (u8 *) eqe, 28); + + in_modifier = (slave & 0xff) | ((event_eq->eqn & 0xff) << 16); + + err = mlx4_cmd(dev, mailbox->dma, in_modifier, 0, + MLX4_CMD_GEN_EQE, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_NATIVE); + + put_res(dev, slave, res_id, RES_EQ); + mutex_unlock(&priv->mfunc.master.gen_eqe_mutex[slave]); + mlx4_free_cmd_mailbox(dev, mailbox); + return err; + +put: + put_res(dev, slave, res_id, RES_EQ); + +unlock: + mutex_unlock(&priv->mfunc.master.gen_eqe_mutex[slave]); + return err; +} + +int mlx4_QUERY_EQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int eqn = vhcr->in_modifier; + int res_id = eqn | (slave << 8); + struct res_eq *eq; + int err; + + err = get_res(dev, slave, res_id, RES_EQ, &eq); + if (err) + return err; + + if (eq->com.from_state != RES_EQ_HW) { + err = -EINVAL; + goto ex_put; + } + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + +ex_put: + put_res(dev, slave, res_id, RES_EQ); + return err; +} + +int mlx4_SW2HW_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int cqn = vhcr->in_modifier; + struct mlx4_cq_context *cqc = inbox->buf; + int mtt_base = cq_get_mtt_addr(cqc) / dev->caps.mtt_entry_sz; + struct res_cq *cq; + struct res_mtt *mtt; + + err = cq_res_start_move_to(dev, slave, cqn, RES_CQ_HW, &cq); + if (err) + return err; + err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); + if (err) + goto out_move; + err = check_mtt_range(dev, slave, mtt_base, cq_get_mtt_size(cqc), mtt); + if (err) + goto out_put; + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto out_put; + atomic_inc(&mtt->ref_count); + cq->mtt = mtt; + put_res(dev, slave, mtt->com.res_id, RES_MTT); + res_end_move(dev, slave, RES_CQ, cqn); + return 0; + +out_put: + put_res(dev, slave, mtt->com.res_id, RES_MTT); +out_move: + res_abort_move(dev, slave, RES_CQ, cqn); + return err; +} + +int mlx4_HW2SW_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int cqn = vhcr->in_modifier; + struct res_cq *cq; + + err = cq_res_start_move_to(dev, slave, cqn, RES_CQ_ALLOCATED, &cq); + if (err) + return err; + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto out_move; + atomic_dec(&cq->mtt->ref_count); + res_end_move(dev, slave, RES_CQ, cqn); + return 0; + +out_move: + res_abort_move(dev, slave, RES_CQ, cqn); + return err; +} + +int mlx4_QUERY_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int cqn = vhcr->in_modifier; + struct res_cq *cq; + int err; + + err = get_res(dev, slave, cqn, RES_CQ, &cq); + if (err) + return err; + + if (cq->com.from_state != RES_CQ_HW) + goto ex_put; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +ex_put: + put_res(dev, slave, cqn, RES_CQ); + + return err; +} + +static int handle_resize(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd, + struct res_cq *cq) +{ + int err; + struct res_mtt *orig_mtt; + struct res_mtt *mtt; + struct mlx4_cq_context *cqc = inbox->buf; + int mtt_base = cq_get_mtt_addr(cqc) / dev->caps.mtt_entry_sz; + + err = get_res(dev, slave, cq->mtt->com.res_id, RES_MTT, &orig_mtt); + if (err) + return err; + + if (orig_mtt != cq->mtt) { + err = -EINVAL; + goto ex_put; + } + + err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); + if (err) + goto ex_put; + + err = check_mtt_range(dev, slave, mtt_base, cq_get_mtt_size(cqc), mtt); + if (err) + goto ex_put1; + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_put1; + atomic_dec(&orig_mtt->ref_count); + put_res(dev, slave, orig_mtt->com.res_id, RES_MTT); + atomic_inc(&mtt->ref_count); + cq->mtt = mtt; + put_res(dev, slave, mtt->com.res_id, RES_MTT); + return 0; + +ex_put1: + put_res(dev, slave, mtt->com.res_id, RES_MTT); +ex_put: + put_res(dev, slave, orig_mtt->com.res_id, RES_MTT); + + return err; + +} + +int mlx4_MODIFY_CQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int cqn = vhcr->in_modifier; + struct res_cq *cq; + int err; + + err = get_res(dev, slave, cqn, RES_CQ, &cq); + if (err) + return err; + + if (cq->com.from_state != RES_CQ_HW) + goto ex_put; + + if (vhcr->op_modifier == 0) { + err = handle_resize(dev, slave, vhcr, inbox, outbox, cmd, cq); + goto ex_put; + } + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +ex_put: + put_res(dev, slave, cqn, RES_CQ); + + return err; +} + +static int srq_get_mtt_size(struct mlx4_srq_context *srqc) +{ + int log_srq_size = (be32_to_cpu(srqc->state_logsize_srqn) >> 24) & 0xf; + int log_rq_stride = srqc->logstride & 7; + int page_shift = (srqc->log_page_size & 0x3f) + 12; + + if (log_srq_size + log_rq_stride + 4 < page_shift) + return 1; + + return 1 << (log_srq_size + log_rq_stride + 4 - page_shift); +} + +int mlx4_SW2HW_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int srqn = vhcr->in_modifier; + struct res_mtt *mtt; + struct res_srq *srq; + struct mlx4_srq_context *srqc = inbox->buf; + int mtt_base = srq_get_mtt_addr(srqc) / dev->caps.mtt_entry_sz; + + if (srqn != (be32_to_cpu(srqc->state_logsize_srqn) & 0xffffff)) + return -EINVAL; + + err = srq_res_start_move_to(dev, slave, srqn, RES_SRQ_HW, &srq); + if (err) + return err; + err = get_res(dev, slave, mtt_base, RES_MTT, &mtt); + if (err) + goto ex_abort; + err = check_mtt_range(dev, slave, mtt_base, srq_get_mtt_size(srqc), + mtt); + if (err) + goto ex_put_mtt; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_put_mtt; + + atomic_inc(&mtt->ref_count); + srq->mtt = mtt; + put_res(dev, slave, mtt->com.res_id, RES_MTT); + res_end_move(dev, slave, RES_SRQ, srqn); + return 0; + +ex_put_mtt: + put_res(dev, slave, mtt->com.res_id, RES_MTT); +ex_abort: + res_abort_move(dev, slave, RES_SRQ, srqn); + + return err; +} + +int mlx4_HW2SW_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int srqn = vhcr->in_modifier; + struct res_srq *srq; + + err = srq_res_start_move_to(dev, slave, srqn, RES_SRQ_ALLOCATED, &srq); + if (err) + return err; + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_abort; + atomic_dec(&srq->mtt->ref_count); + if (srq->cq) + atomic_dec(&srq->cq->ref_count); + res_end_move(dev, slave, RES_SRQ, srqn); + + return 0; + +ex_abort: + res_abort_move(dev, slave, RES_SRQ, srqn); + + return err; +} + +int mlx4_QUERY_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int srqn = vhcr->in_modifier; + struct res_srq *srq; + + err = get_res(dev, slave, srqn, RES_SRQ, &srq); + if (err) + return err; + if (srq->com.from_state != RES_SRQ_HW) { + err = -EBUSY; + goto out; + } + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +out: + put_res(dev, slave, srqn, RES_SRQ); + return err; +} + +int mlx4_ARM_SRQ_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int srqn = vhcr->in_modifier; + struct res_srq *srq; + + err = get_res(dev, slave, srqn, RES_SRQ, &srq); + if (err) + return err; + + if (srq->com.from_state != RES_SRQ_HW) { + err = -EBUSY; + goto out; + } + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +out: + put_res(dev, slave, srqn, RES_SRQ); + return err; +} + +int mlx4_GEN_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int qpn = vhcr->in_modifier & 0x7fffff; + struct res_qp *qp; + + err = get_res(dev, slave, qpn, RES_QP, &qp); + if (err) + return err; + if (qp->com.from_state != RES_QP_HW) { + err = -EBUSY; + goto out; + } + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +out: + put_res(dev, slave, qpn, RES_QP); + return err; +} + +int mlx4_INIT2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_qp_context *context = inbox->buf + 8; + adjust_proxy_tun_qkey(dev, vhcr, context); + update_pkey_index(dev, slave, inbox); + return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +} + +static int roce_verify_mac(struct mlx4_dev *dev, int slave, + struct mlx4_qp_context *qpc, + struct mlx4_cmd_mailbox *inbox) +{ + u64 mac; + int port; + u32 ts = (be32_to_cpu(qpc->flags) >> 16) & 0xff; + u8 sched = *(u8 *)(inbox->buf + 64); + u8 smac_ix; + + port = (sched >> 6 & 1) + 1; + if (mlx4_is_eth(dev, port) && (ts != MLX4_QP_ST_MLX)) { + smac_ix = qpc->pri_path.grh_mylmc & 0x7f; + if (mac_find_smac_ix_in_slave(dev, slave, port, smac_ix, &mac)) + return -ENOENT; + } + return 0; +} + +int mlx4_INIT2RTR_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + struct mlx4_qp_context *qpc = inbox->buf + 8; + int qpn = vhcr->in_modifier & 0x7fffff; + struct res_qp *qp; + u8 orig_sched_queue; + __be32 orig_param3 = qpc->param3; + u8 orig_vlan_control = qpc->pri_path.vlan_control; + u8 orig_fvl_rx = qpc->pri_path.fvl_rx; + u8 orig_pri_path_fl = qpc->pri_path.fl; + u8 orig_vlan_index = qpc->pri_path.vlan_index; + u8 orig_feup = qpc->pri_path.feup; + + err = verify_qp_parameters(dev, inbox, QP_TRANS_INIT2RTR, slave); + if (err) + return err; + + if (roce_verify_mac(dev, slave, qpc, inbox)) + return -EINVAL; + + update_pkey_index(dev, slave, inbox); + update_gid(dev, inbox, (u8)slave); + adjust_proxy_tun_qkey(dev, vhcr, qpc); + orig_sched_queue = qpc->pri_path.sched_queue; + + err = get_res(dev, slave, qpn, RES_QP, &qp); + if (err) + return err; + if (qp->com.from_state != RES_QP_HW) { + err = -EBUSY; + goto out; + } + + /* do not modify vport QP params for RSS QPs */ + if (!(qp->qpc_flags & (1 << MLX4_RSS_QPC_FLAG_OFFSET))) { + err = update_vport_qp_param(dev, inbox, slave, qpn); + if (err) + goto out; + } + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +out: + /* if no error, save sched queue value passed in by VF. This is + * essentially the QOS value provided by the VF. This will be useful + * if we allow dynamic changes from VST back to VGT + */ + if (!err) { + qp->sched_queue = orig_sched_queue; + qp->param3 = orig_param3; + qp->vlan_control = orig_vlan_control; + qp->fvl_rx = orig_fvl_rx; + qp->pri_path_fl = orig_pri_path_fl; + qp->vlan_index = orig_vlan_index; + qp->feup = orig_feup; + } + put_res(dev, slave, qpn, RES_QP); + return err; +} + +int mlx4_RTR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + struct mlx4_qp_context *context = inbox->buf + 8; + + err = verify_qp_parameters(dev, inbox, QP_TRANS_RTR2RTS, slave); + if (err) + return err; + + update_pkey_index(dev, slave, inbox); + update_gid(dev, inbox, (u8)slave); + adjust_proxy_tun_qkey(dev, vhcr, context); + return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +} + +int mlx4_RTS2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + struct mlx4_qp_context *context = inbox->buf + 8; + + err = verify_qp_parameters(dev, inbox, QP_TRANS_RTS2RTS, slave); + if (err) + return err; + + update_pkey_index(dev, slave, inbox); + update_gid(dev, inbox, (u8)slave); + adjust_proxy_tun_qkey(dev, vhcr, context); + return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +} + + +int mlx4_SQERR2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_qp_context *context = inbox->buf + 8; + adjust_proxy_tun_qkey(dev, vhcr, context); + return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +} + +int mlx4_SQD2SQD_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + struct mlx4_qp_context *context = inbox->buf + 8; + + err = verify_qp_parameters(dev, inbox, QP_TRANS_SQD2SQD, slave); + if (err) + return err; + + adjust_proxy_tun_qkey(dev, vhcr, context); + update_gid(dev, inbox, (u8)slave); + update_pkey_index(dev, slave, inbox); + return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +} + +int mlx4_SQD2RTS_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + struct mlx4_qp_context *context = inbox->buf + 8; + + err = verify_qp_parameters(dev, inbox, QP_TRANS_SQD2RTS, slave); + if (err) + return err; + + adjust_proxy_tun_qkey(dev, vhcr, context); + update_gid(dev, inbox, (u8)slave); + update_pkey_index(dev, slave, inbox); + return mlx4_GEN_QP_wrapper(dev, slave, vhcr, inbox, outbox, cmd); +} + +int mlx4_2RST_QP_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + int qpn = vhcr->in_modifier & 0x7fffff; + struct res_qp *qp; + + err = qp_res_start_move_to(dev, slave, qpn, RES_QP_MAPPED, &qp, 0); + if (err) + return err; + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + if (err) + goto ex_abort; + + atomic_dec(&qp->mtt->ref_count); + atomic_dec(&qp->rcq->ref_count); + atomic_dec(&qp->scq->ref_count); + if (qp->srq) + atomic_dec(&qp->srq->ref_count); + res_end_move(dev, slave, RES_QP, qpn); + return 0; + +ex_abort: + res_abort_move(dev, slave, RES_QP, qpn); + + return err; +} + +static struct res_gid *find_gid(struct mlx4_dev *dev, int slave, + struct res_qp *rqp, u8 *gid) +{ + struct res_gid *res; + + list_for_each_entry(res, &rqp->mcg_list, list) { + if (!memcmp(res->gid, gid, 16)) + return res; + } + return NULL; +} + +static int add_mcg_res(struct mlx4_dev *dev, int slave, struct res_qp *rqp, + u8 *gid, enum mlx4_protocol prot, + enum mlx4_steer_type steer, u64 reg_id) +{ + struct res_gid *res; + int err; + + res = kzalloc(sizeof *res, GFP_KERNEL); + if (!res) + return -ENOMEM; + + spin_lock_irq(&rqp->mcg_spl); + if (find_gid(dev, slave, rqp, gid)) { + kfree(res); + err = -EEXIST; + } else { + memcpy(res->gid, gid, 16); + res->prot = prot; + res->steer = steer; + res->reg_id = reg_id; + list_add_tail(&res->list, &rqp->mcg_list); + err = 0; + } + spin_unlock_irq(&rqp->mcg_spl); + + return err; +} + +static int rem_mcg_res(struct mlx4_dev *dev, int slave, struct res_qp *rqp, + u8 *gid, enum mlx4_protocol prot, + enum mlx4_steer_type steer, u64 *reg_id) +{ + struct res_gid *res; + int err; + + spin_lock_irq(&rqp->mcg_spl); + res = find_gid(dev, slave, rqp, gid); + if (!res || res->prot != prot || res->steer != steer) + err = -EINVAL; + else { + *reg_id = res->reg_id; + list_del(&res->list); + kfree(res); + err = 0; + } + spin_unlock_irq(&rqp->mcg_spl); + + return err; +} + +static int qp_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + int block_loopback, enum mlx4_protocol prot, + enum mlx4_steer_type type, u64 *reg_id) +{ + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: + return mlx4_trans_to_dmfs_attach(dev, qp, gid, gid[5], + block_loopback, prot, + reg_id); + case MLX4_STEERING_MODE_B0: + return mlx4_qp_attach_common(dev, qp, gid, + block_loopback, prot, type); + default: + return -EINVAL; + } +} + +static int qp_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + enum mlx4_protocol prot, enum mlx4_steer_type type, + u64 reg_id) +{ + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: + return mlx4_flow_detach(dev, reg_id); + case MLX4_STEERING_MODE_B0: + return mlx4_qp_detach_common(dev, qp, gid, prot, type); + default: + return -EINVAL; + } +} + +int mlx4_QP_ATTACH_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + struct mlx4_qp qp; /* dummy for calling attach/detach */ + u8 *gid = inbox->buf; + enum mlx4_protocol prot = (vhcr->in_modifier >> 28) & 0x7; + int err; + int qpn; + struct res_qp *rqp; + u64 reg_id = 0; + int attach = vhcr->op_modifier; + int block_loopback = vhcr->in_modifier >> 31; + u8 steer_type_mask = 2; + enum mlx4_steer_type type = (gid[7] & steer_type_mask) >> 1; + + qpn = vhcr->in_modifier & 0xffffff; + err = get_res(dev, slave, qpn, RES_QP, &rqp); + if (err) + return err; + + qp.qpn = qpn; + if (attach) { + err = qp_attach(dev, &qp, gid, block_loopback, prot, + type, ®_id); + if (err) { + pr_err("Fail to attach rule to qp 0x%x\n", qpn); + goto ex_put; + } + err = add_mcg_res(dev, slave, rqp, gid, prot, type, reg_id); + if (err) + goto ex_detach; + } else { + err = rem_mcg_res(dev, slave, rqp, gid, prot, type, ®_id); + if (err) + goto ex_put; + + err = qp_detach(dev, &qp, gid, prot, type, reg_id); + if (err) + pr_err("Fail to detach rule from qp 0x%x reg_id = 0x%llx\n", + qpn, (unsigned long long)reg_id); + } + put_res(dev, slave, qpn, RES_QP); + return err; + +ex_detach: + qp_detach(dev, &qp, gid, prot, type, reg_id); +ex_put: + put_res(dev, slave, qpn, RES_QP); + return err; +} + +/* + * MAC validation for Flow Steering rules. + * VF can attach rules only with a mac address which is assigned to it. + */ +static int validate_eth_header_mac(int slave, struct _rule_hw *eth_header, + struct list_head *rlist) +{ + struct mac_res *res, *tmp; + __be64 be_mac; + + /* make sure it isn't multicast or broadcast mac*/ + if (!is_multicast_ether_addr(eth_header->eth.dst_mac) && + !is_broadcast_ether_addr(eth_header->eth.dst_mac)) { + list_for_each_entry_safe(res, tmp, rlist, list) { + be_mac = cpu_to_be64(res->mac << 16); + if (!memcmp(&be_mac, eth_header->eth.dst_mac, ETH_ALEN)) + return 0; + } + pr_err("MAC %pM doesn't belong to VF %d, Steering rule rejected\n", + eth_header->eth.dst_mac, slave); + return -EINVAL; + } + return 0; +} + +/* + * In case of missing eth header, append eth header with a MAC address + * assigned to the VF. + */ +static int add_eth_header(struct mlx4_dev *dev, int slave, + struct mlx4_cmd_mailbox *inbox, + struct list_head *rlist, int header_id) +{ + struct mac_res *res, *tmp; + u8 port; + struct mlx4_net_trans_rule_hw_ctrl *ctrl; + struct mlx4_net_trans_rule_hw_eth *eth_header; + struct mlx4_net_trans_rule_hw_ipv4 *ip_header; + struct mlx4_net_trans_rule_hw_tcp_udp *l4_header; + __be64 be_mac = 0; + __be64 mac_msk = cpu_to_be64(MLX4_MAC_MASK << 16); + + ctrl = (struct mlx4_net_trans_rule_hw_ctrl *)inbox->buf; + port = ctrl->port; + eth_header = (struct mlx4_net_trans_rule_hw_eth *)(ctrl + 1); + + /* Clear a space in the inbox for eth header */ + switch (header_id) { + case MLX4_NET_TRANS_RULE_ID_IPV4: + ip_header = + (struct mlx4_net_trans_rule_hw_ipv4 *)(eth_header + 1); + memmove(ip_header, eth_header, + sizeof(*ip_header) + sizeof(*l4_header)); + break; + case MLX4_NET_TRANS_RULE_ID_TCP: + case MLX4_NET_TRANS_RULE_ID_UDP: + l4_header = (struct mlx4_net_trans_rule_hw_tcp_udp *) + (eth_header + 1); + memmove(l4_header, eth_header, sizeof(*l4_header)); + break; + default: + return -EINVAL; + } + list_for_each_entry_safe(res, tmp, rlist, list) { + if (port == res->port) { + be_mac = cpu_to_be64(res->mac << 16); + break; + } + } + if (!be_mac) { + pr_err("Failed adding eth header to FS rule, Can't find matching MAC for port %d .\n", + port); + return -EINVAL; + } + + memset(eth_header, 0, sizeof(*eth_header)); + eth_header->size = sizeof(*eth_header) >> 2; + eth_header->id = cpu_to_be16(__sw_id_hw[MLX4_NET_TRANS_RULE_ID_ETH]); + memcpy(eth_header->dst_mac, &be_mac, ETH_ALEN); + memcpy(eth_header->dst_mac_msk, &mac_msk, ETH_ALEN); + + return 0; + +} + +int mlx4_QP_FLOW_STEERING_ATTACH_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *rlist = &tracker->slave_list[slave].res_list[RES_MAC]; + int err; + int qpn; + struct res_qp *rqp; + struct mlx4_net_trans_rule_hw_ctrl *ctrl; + struct _rule_hw *rule_header; + int header_id; + + if (dev->caps.steering_mode != + MLX4_STEERING_MODE_DEVICE_MANAGED) + return -EOPNOTSUPP; + + ctrl = (struct mlx4_net_trans_rule_hw_ctrl *)inbox->buf; + qpn = be32_to_cpu(ctrl->qpn) & 0xffffff; + err = get_res(dev, slave, qpn, RES_QP, &rqp); + if (err) { + pr_err("Steering rule with qpn 0x%x rejected.\n", qpn); + return err; + } + rule_header = (struct _rule_hw *)(ctrl + 1); + header_id = map_hw_to_sw_id(be16_to_cpu(rule_header->id)); + + switch (header_id) { + case MLX4_NET_TRANS_RULE_ID_ETH: + if (validate_eth_header_mac(slave, rule_header, rlist)) { + err = -EINVAL; + goto err_put; + } + break; + case MLX4_NET_TRANS_RULE_ID_IB: + break; + case MLX4_NET_TRANS_RULE_ID_IPV4: + case MLX4_NET_TRANS_RULE_ID_TCP: + case MLX4_NET_TRANS_RULE_ID_UDP: + pr_warn("Can't attach FS rule without L2 headers, adding L2 header.\n"); + if (add_eth_header(dev, slave, inbox, rlist, header_id)) { + err = -EINVAL; + goto err_put; + } + vhcr->in_modifier += + sizeof(struct mlx4_net_trans_rule_hw_eth) >> 2; + break; + default: + pr_err("Corrupted mailbox.\n"); + err = -EINVAL; + goto err_put; + } + + err = mlx4_cmd_imm(dev, inbox->dma, &vhcr->out_param, + vhcr->in_modifier, 0, + MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + goto err_put; + + err = add_res_range(dev, slave, vhcr->out_param, 1, RES_FS_RULE, qpn); + if (err) { + mlx4_err(dev, "Fail to add flow steering resources.\n "); + /* detach rule*/ + mlx4_cmd(dev, vhcr->out_param, 0, 0, + MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + goto err_put; + } + atomic_inc(&rqp->ref_count); +err_put: + put_res(dev, slave, qpn, RES_QP); + return err; +} + +int mlx4_QP_FLOW_STEERING_DETACH_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + struct res_qp *rqp; + struct res_fs_rule *rrule; + + if (dev->caps.steering_mode != + MLX4_STEERING_MODE_DEVICE_MANAGED) + return -EOPNOTSUPP; + + err = get_res(dev, slave, vhcr->in_param, RES_FS_RULE, &rrule); + if (err) + return err; + /* Release the rule form busy state before removal */ + put_res(dev, slave, vhcr->in_param, RES_FS_RULE); + err = get_res(dev, slave, rrule->qpn, RES_QP, &rqp); + if (err) + return err; + + err = mlx4_cmd(dev, vhcr->in_param, 0, 0, + MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (!err) { + err = rem_res_range(dev, slave, vhcr->in_param, 1, RES_FS_RULE, + 0); + atomic_dec(&rqp->ref_count); + + if (err) { + mlx4_err(dev, "Fail to remove flow steering resources.\n "); + goto out; + } + } + +out: + put_res(dev, slave, rrule->qpn, RES_QP); + return err; +} + +enum { + BUSY_MAX_RETRIES = 10 +}; + +int mlx4_QUERY_IF_STAT_wrapper(struct mlx4_dev *dev, int slave, + struct mlx4_vhcr *vhcr, + struct mlx4_cmd_mailbox *inbox, + struct mlx4_cmd_mailbox *outbox, + struct mlx4_cmd_info *cmd) +{ + int err; + + err = mlx4_DMA_wrapper(dev, slave, vhcr, inbox, outbox, cmd); + + return err; +} + +static void detach_qp(struct mlx4_dev *dev, int slave, struct res_qp *rqp) +{ + struct res_gid *rgid; + struct res_gid *tmp; + struct mlx4_qp qp; /* dummy for calling attach/detach */ + + list_for_each_entry_safe(rgid, tmp, &rqp->mcg_list, list) { + switch (dev->caps.steering_mode) { + case MLX4_STEERING_MODE_DEVICE_MANAGED: + mlx4_flow_detach(dev, rgid->reg_id); + break; + case MLX4_STEERING_MODE_B0: + qp.qpn = rqp->local_qpn; + (void) mlx4_qp_detach_common(dev, &qp, rgid->gid, + rgid->prot, rgid->steer); + break; + } + list_del(&rgid->list); + kfree(rgid); + } +} + +static int _move_all_busy(struct mlx4_dev *dev, int slave, + enum mlx4_resource type, int print) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = + &priv->mfunc.master.res_tracker; + struct list_head *rlist = &tracker->slave_list[slave].res_list[type]; + struct res_common *r; + struct res_common *tmp; + int busy; + + busy = 0; + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(r, tmp, rlist, list) { + if (r->owner == slave) { + if (!r->removing) { + if (r->state == RES_ANY_BUSY) { + if (print) + mlx4_dbg(dev, + "%s id 0x%llx is busy\n", + ResourceType(type), + (unsigned long long)r->res_id); + ++busy; + } else { + r->from_state = r->state; + r->state = RES_ANY_BUSY; + r->removing = 1; + } + } + } + } + spin_unlock_irq(mlx4_tlock(dev)); + + return busy; +} + +static int move_all_busy(struct mlx4_dev *dev, int slave, + enum mlx4_resource type) +{ + unsigned long begin; + int busy; + + begin = jiffies; + do { + busy = _move_all_busy(dev, slave, type, 0); + if (time_after(jiffies, begin + 5 * HZ)) + break; + if (busy) + cond_resched(); + } while (busy); + + if (busy) + busy = _move_all_busy(dev, slave, type, 1); + + return busy; +} +static void rem_slave_qps(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *qp_list = + &tracker->slave_list[slave].res_list[RES_QP]; + struct res_qp *qp; + struct res_qp *tmp; + int state; + u64 in_param; + int qpn; + int err; + + err = move_all_busy(dev, slave, RES_QP); + if (err) + mlx4_warn(dev, "rem_slave_qps: Could not move all qps to busy" + "for slave %d\n", slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(qp, tmp, qp_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (qp->com.owner == slave) { + qpn = qp->com.res_id; + detach_qp(dev, slave, qp); + state = qp->com.from_state; + while (state != 0) { + switch (state) { + case RES_QP_RESERVED: + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&qp->com.node, + &tracker->res_tree[RES_QP]); + list_del(&qp->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + if (!valid_reserved(dev, slave, qpn)) { + __mlx4_qp_release_range(dev, qpn, 1); + mlx4_release_resource(dev, slave, + RES_QP, 1, 0); + } + kfree(qp); + state = 0; + break; + case RES_QP_MAPPED: + if (!valid_reserved(dev, slave, qpn)) + __mlx4_qp_free_icm(dev, qpn); + state = RES_QP_RESERVED; + break; + case RES_QP_HW: + in_param = slave; + err = mlx4_cmd(dev, in_param, + qp->local_qpn, 2, + MLX4_CMD_2RST_QP, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + mlx4_dbg(dev, "rem_slave_qps: failed" + " to move slave %d qpn %d to" + " reset\n", slave, + qp->local_qpn); + atomic_dec(&qp->rcq->ref_count); + atomic_dec(&qp->scq->ref_count); + atomic_dec(&qp->mtt->ref_count); + if (qp->srq) + atomic_dec(&qp->srq->ref_count); + state = RES_QP_MAPPED; + break; + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void rem_slave_srqs(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *srq_list = + &tracker->slave_list[slave].res_list[RES_SRQ]; + struct res_srq *srq; + struct res_srq *tmp; + int state; + u64 in_param; + LIST_HEAD(tlist); + int srqn; + int err; + + err = move_all_busy(dev, slave, RES_SRQ); + if (err) + mlx4_warn(dev, "rem_slave_srqs: Could not move all srqs to " + "busy for slave %d\n", slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(srq, tmp, srq_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (srq->com.owner == slave) { + srqn = srq->com.res_id; + state = srq->com.from_state; + while (state != 0) { + switch (state) { + case RES_SRQ_ALLOCATED: + __mlx4_srq_free_icm(dev, srqn); + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&srq->com.node, + &tracker->res_tree[RES_SRQ]); + list_del(&srq->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + mlx4_release_resource(dev, slave, + RES_SRQ, 1, 0); + kfree(srq); + state = 0; + break; + + case RES_SRQ_HW: + in_param = slave; + err = mlx4_cmd(dev, in_param, srqn, 1, + MLX4_CMD_HW2SW_SRQ, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + mlx4_dbg(dev, "rem_slave_srqs: failed" + " to move slave %d srq %d to" + " SW ownership\n", + slave, srqn); + + atomic_dec(&srq->mtt->ref_count); + if (srq->cq) + atomic_dec(&srq->cq->ref_count); + state = RES_SRQ_ALLOCATED; + break; + + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void rem_slave_cqs(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *cq_list = + &tracker->slave_list[slave].res_list[RES_CQ]; + struct res_cq *cq; + struct res_cq *tmp; + int state; + u64 in_param; + LIST_HEAD(tlist); + int cqn; + int err; + + err = move_all_busy(dev, slave, RES_CQ); + if (err) + mlx4_warn(dev, "rem_slave_cqs: Could not move all cqs to " + "busy for slave %d\n", slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(cq, tmp, cq_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (cq->com.owner == slave && !atomic_read(&cq->ref_count)) { + cqn = cq->com.res_id; + state = cq->com.from_state; + while (state != 0) { + switch (state) { + case RES_CQ_ALLOCATED: + __mlx4_cq_free_icm(dev, cqn); + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&cq->com.node, + &tracker->res_tree[RES_CQ]); + list_del(&cq->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + mlx4_release_resource(dev, slave, + RES_CQ, 1, 0); + kfree(cq); + state = 0; + break; + + case RES_CQ_HW: + in_param = slave; + err = mlx4_cmd(dev, in_param, cqn, 1, + MLX4_CMD_HW2SW_CQ, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + mlx4_dbg(dev, "rem_slave_cqs: failed" + " to move slave %d cq %d to" + " SW ownership\n", + slave, cqn); + atomic_dec(&cq->mtt->ref_count); + state = RES_CQ_ALLOCATED; + break; + + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void rem_slave_mrs(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *mpt_list = + &tracker->slave_list[slave].res_list[RES_MPT]; + struct res_mpt *mpt; + struct res_mpt *tmp; + int state; + u64 in_param; + LIST_HEAD(tlist); + int mptn; + int err; + + err = move_all_busy(dev, slave, RES_MPT); + if (err) + mlx4_warn(dev, "rem_slave_mrs: Could not move all mpts to " + "busy for slave %d\n", slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(mpt, tmp, mpt_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (mpt->com.owner == slave) { + mptn = mpt->com.res_id; + state = mpt->com.from_state; + while (state != 0) { + switch (state) { + case RES_MPT_RESERVED: + __mlx4_mpt_release(dev, mpt->key); + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&mpt->com.node, + &tracker->res_tree[RES_MPT]); + list_del(&mpt->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + mlx4_release_resource(dev, slave, + RES_MPT, 1, 0); + kfree(mpt); + state = 0; + break; + + case RES_MPT_MAPPED: + __mlx4_mpt_free_icm(dev, mpt->key); + state = RES_MPT_RESERVED; + break; + + case RES_MPT_HW: + in_param = slave; + err = mlx4_cmd(dev, in_param, mptn, 0, + MLX4_CMD_HW2SW_MPT, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + mlx4_dbg(dev, "rem_slave_mrs: failed" + " to move slave %d mpt %d to" + " SW ownership\n", + slave, mptn); + if (mpt->mtt) + atomic_dec(&mpt->mtt->ref_count); + state = RES_MPT_MAPPED; + break; + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void rem_slave_mtts(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = + &priv->mfunc.master.res_tracker; + struct list_head *mtt_list = + &tracker->slave_list[slave].res_list[RES_MTT]; + struct res_mtt *mtt; + struct res_mtt *tmp; + int state; + LIST_HEAD(tlist); + int base; + int err; + + err = move_all_busy(dev, slave, RES_MTT); + if (err) + mlx4_warn(dev, "rem_slave_mtts: Could not move all mtts to " + "busy for slave %d\n", slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(mtt, tmp, mtt_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (mtt->com.owner == slave) { + base = mtt->com.res_id; + state = mtt->com.from_state; + while (state != 0) { + switch (state) { + case RES_MTT_ALLOCATED: + __mlx4_free_mtt_range(dev, base, + mtt->order); + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&mtt->com.node, + &tracker->res_tree[RES_MTT]); + list_del(&mtt->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + mlx4_release_resource(dev, slave, RES_MTT, + 1 << mtt->order, 0); + kfree(mtt); + state = 0; + break; + + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void rem_slave_fs_rule(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = + &priv->mfunc.master.res_tracker; + struct list_head *fs_rule_list = + &tracker->slave_list[slave].res_list[RES_FS_RULE]; + struct res_fs_rule *fs_rule; + struct res_fs_rule *tmp; + int state; + u64 base; + int err; + + err = move_all_busy(dev, slave, RES_FS_RULE); + if (err) + mlx4_warn(dev, "rem_slave_fs_rule: Could not move all mtts to busy for slave %d\n", + slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(fs_rule, tmp, fs_rule_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (fs_rule->com.owner == slave) { + base = fs_rule->com.res_id; + state = fs_rule->com.from_state; + while (state != 0) { + switch (state) { + case RES_FS_RULE_ALLOCATED: + /* detach rule */ + err = mlx4_cmd(dev, base, 0, 0, + MLX4_QP_FLOW_STEERING_DETACH, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&fs_rule->com.node, + &tracker->res_tree[RES_FS_RULE]); + list_del(&fs_rule->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + kfree(fs_rule); + state = 0; + break; + + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void rem_slave_eqs(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *eq_list = + &tracker->slave_list[slave].res_list[RES_EQ]; + struct res_eq *eq; + struct res_eq *tmp; + int err; + int state; + LIST_HEAD(tlist); + int eqn; + struct mlx4_cmd_mailbox *mailbox; + + err = move_all_busy(dev, slave, RES_EQ); + if (err) + mlx4_warn(dev, "rem_slave_eqs: Could not move all eqs to " + "busy for slave %d\n", slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(eq, tmp, eq_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (eq->com.owner == slave) { + eqn = eq->com.res_id; + state = eq->com.from_state; + while (state != 0) { + switch (state) { + case RES_EQ_RESERVED: + spin_lock_irq(mlx4_tlock(dev)); + rb_erase(&eq->com.node, + &tracker->res_tree[RES_EQ]); + list_del(&eq->com.list); + spin_unlock_irq(mlx4_tlock(dev)); + kfree(eq); + state = 0; + break; + + case RES_EQ_HW: + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) { + cond_resched(); + continue; + } + err = mlx4_cmd_box(dev, slave, 0, + eqn & 0xff, 0, + MLX4_CMD_HW2SW_EQ, + MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_NATIVE); + if (err) + mlx4_dbg(dev, "rem_slave_eqs: failed" + " to move slave %d eqs %d to" + " SW ownership\n", slave, eqn); + mlx4_free_cmd_mailbox(dev, mailbox); + atomic_dec(&eq->mtt->ref_count); + state = RES_EQ_RESERVED; + break; + + default: + state = 0; + } + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +static void rem_slave_counters(struct mlx4_dev *dev, int slave) +{ + __mlx4_slave_counters_free(dev, slave); +} + +static void rem_slave_xrcdns(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker; + struct list_head *xrcdn_list = + &tracker->slave_list[slave].res_list[RES_XRCD]; + struct res_xrcdn *xrcd; + struct res_xrcdn *tmp; + int err; + int xrcdn; + + err = move_all_busy(dev, slave, RES_XRCD); + if (err) + mlx4_warn(dev, "rem_slave_xrcdns: Could not move all xrcdns to " + "busy for slave %d\n", slave); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(xrcd, tmp, xrcdn_list, com.list) { + if (xrcd->com.owner == slave) { + xrcdn = xrcd->com.res_id; + rb_erase(&xrcd->com.node, &tracker->res_tree[RES_XRCD]); + list_del(&xrcd->com.list); + kfree(xrcd); + __mlx4_xrcd_free(dev, xrcdn); + } + } + spin_unlock_irq(mlx4_tlock(dev)); +} + +void mlx4_delete_all_resources_for_slave(struct mlx4_dev *dev, int slave) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + + mutex_lock(&priv->mfunc.master.res_tracker.slave_list[slave].mutex); + rem_slave_macs(dev, slave); + rem_slave_vlans(dev, slave); + rem_slave_fs_rule(dev, slave); + rem_slave_qps(dev, slave); + rem_slave_srqs(dev, slave); + rem_slave_cqs(dev, slave); + rem_slave_mrs(dev, slave); + rem_slave_eqs(dev, slave); + rem_slave_mtts(dev, slave); + rem_slave_counters(dev, slave); + rem_slave_xrcdns(dev, slave); + mutex_unlock(&priv->mfunc.master.res_tracker.slave_list[slave].mutex); +} + +void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work) +{ + struct mlx4_vf_immed_vlan_work *work = + container_of(_work, struct mlx4_vf_immed_vlan_work, work); + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_update_qp_context *upd_context; + struct mlx4_dev *dev = &work->priv->dev; + struct mlx4_resource_tracker *tracker = + &work->priv->mfunc.master.res_tracker; + struct list_head *qp_list = + &tracker->slave_list[work->slave].res_list[RES_QP]; + struct res_qp *qp; + struct res_qp *tmp; + u64 qp_path_mask_vlan_ctrl = + ((1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_UNTAGGED) | + (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_1P) | + (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_TAGGED) | + (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_UNTAGGED) | + (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_1P) | + (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_TAGGED)); + + u64 qp_path_mask = ((1ULL << MLX4_UPD_QP_PATH_MASK_VLAN_INDEX) | + (1ULL << MLX4_UPD_QP_PATH_MASK_FVL) | + (1ULL << MLX4_UPD_QP_PATH_MASK_CV) | + (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_HIDE_CQE_VLAN) | + (1ULL << MLX4_UPD_QP_PATH_MASK_FEUP) | + (1ULL << MLX4_UPD_QP_PATH_MASK_FVL_RX) | + (1ULL << MLX4_UPD_QP_PATH_MASK_SCHED_QUEUE)); + + int err; + int port, errors = 0; + u8 vlan_control; + + if (mlx4_is_slave(dev)) { + mlx4_warn(dev, "Trying to update-qp in slave %d\n", + work->slave); + goto out; + } + + mailbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + goto out; + + if (!work->vlan_id) + vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED; + else + vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED | + MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED; + + upd_context = mailbox->buf; + upd_context->qp_mask = cpu_to_be64(MLX4_UPD_QP_MASK_VSD); + + spin_lock_irq(mlx4_tlock(dev)); + list_for_each_entry_safe(qp, tmp, qp_list, com.list) { + spin_unlock_irq(mlx4_tlock(dev)); + if (qp->com.owner == work->slave) { + if (qp->com.from_state != RES_QP_HW || + !qp->sched_queue || /* no INIT2RTR trans yet */ + mlx4_is_qp_reserved(dev, qp->local_qpn) || + qp->qpc_flags & (1 << MLX4_RSS_QPC_FLAG_OFFSET)) { + spin_lock_irq(mlx4_tlock(dev)); + continue; + } + port = (qp->sched_queue >> 6 & 1) + 1; + if (port != work->port) { + spin_lock_irq(mlx4_tlock(dev)); + continue; + } + if (MLX4_QP_ST_RC == ((qp->qpc_flags >> 16) & 0xff)) + upd_context->primary_addr_path_mask = cpu_to_be64(qp_path_mask); + else + upd_context->primary_addr_path_mask = + cpu_to_be64(qp_path_mask | qp_path_mask_vlan_ctrl); + if (work->vlan_id == MLX4_VGT) { + upd_context->qp_context.param3 = qp->param3; + upd_context->qp_context.pri_path.vlan_control = qp->vlan_control; + upd_context->qp_context.pri_path.fvl_rx = qp->fvl_rx; + upd_context->qp_context.pri_path.vlan_index = qp->vlan_index; + upd_context->qp_context.pri_path.fl = qp->pri_path_fl; + upd_context->qp_context.pri_path.feup = qp->feup; + upd_context->qp_context.pri_path.sched_queue = + qp->sched_queue; + } else { + upd_context->qp_context.param3 = qp->param3 & ~cpu_to_be32(MLX4_STRIP_VLAN); + upd_context->qp_context.pri_path.vlan_control = vlan_control; + upd_context->qp_context.pri_path.vlan_index = work->vlan_ix; + upd_context->qp_context.pri_path.fvl_rx = + qp->fvl_rx | MLX4_FVL_RX_FORCE_ETH_VLAN; + upd_context->qp_context.pri_path.fl = + qp->pri_path_fl | MLX4_FL_CV | MLX4_FL_ETH_HIDE_CQE_VLAN; + upd_context->qp_context.pri_path.feup = + qp->feup | MLX4_FEUP_FORCE_ETH_UP | MLX4_FVL_FORCE_ETH_VLAN; + upd_context->qp_context.pri_path.sched_queue = + qp->sched_queue & 0xC7; + upd_context->qp_context.pri_path.sched_queue |= + ((work->qos & 0x7) << 3); + } + + err = mlx4_cmd(dev, mailbox->dma, + qp->local_qpn & 0xffffff, + 0, MLX4_CMD_UPDATE_QP, + MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); + if (err) { + mlx4_info(dev, "UPDATE_QP failed for slave %d, " + "port %d, qpn %d (%d)\n", + work->slave, port, qp->local_qpn, + err); + errors++; + } + } + spin_lock_irq(mlx4_tlock(dev)); + } + spin_unlock_irq(mlx4_tlock(dev)); + mlx4_free_cmd_mailbox(dev, mailbox); + + if (errors) + mlx4_err(dev, "%d UPDATE_QP failures for slave %d, port %d\n", + errors, work->slave, work->port); + + /* unregister previous vlan_id if needed and we had no errors + * while updating the QPs + */ + if (work->flags & MLX4_VF_IMMED_VLAN_FLAG_VLAN && !errors && + NO_INDX != work->orig_vlan_ix) + __mlx4_unregister_vlan(&work->priv->dev, work->port, + work->orig_vlan_id); +out: + kfree(work); + return; +} + Property changes on: stable/9/sys/ofed/drivers/net/mlx4/resource_tracker.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/drivers/net/mlx4/sense.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/sense.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/sense.c (revision 273864) @@ -1,172 +1,143 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ #include #include #include #include "mlx4.h" -static int mlx4_SENSE_PORT(struct mlx4_dev *dev, int port, - enum mlx4_port_type *type) +int mlx4_SENSE_PORT(struct mlx4_dev *dev, int port, + enum mlx4_port_type *type) { u64 out_param; int err = 0; err = mlx4_cmd_imm(dev, 0, &out_param, port, 0, - MLX4_CMD_SENSE_PORT, MLX4_CMD_TIME_CLASS_B); + MLX4_CMD_SENSE_PORT, MLX4_CMD_TIME_CLASS_B, + MLX4_CMD_WRAPPED); if (err) { mlx4_err(dev, "Sense command failed for port: %d\n", port); return err; } if (out_param > 2) { - mlx4_err(dev, "Sense returned illegal value: 0x%llx\n", out_param); - return EINVAL; + mlx4_err(dev, "Sense returned illegal value: 0x%llx\n", (unsigned long long)out_param); + return -EINVAL; } *type = out_param; return 0; } void mlx4_do_sense_ports(struct mlx4_dev *dev, enum mlx4_port_type *stype, enum mlx4_port_type *defaults) { struct mlx4_sense *sense = &mlx4_priv(dev)->sense; int err; int i; for (i = 1; i <= dev->caps.num_ports; i++) { stype[i - 1] = 0; if (sense->do_sense_port[i] && sense->sense_allowed[i] && dev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) { err = mlx4_SENSE_PORT(dev, i, &stype[i - 1]); if (err) stype[i - 1] = defaults[i - 1]; } else stype[i - 1] = defaults[i - 1]; } /* - * Adjust port configuration: - * If port 1 sensed nothing and port 2 is IB, set both as IB - * If port 2 sensed nothing and port 1 is Eth, set both as Eth - */ - if (stype[0] == MLX4_PORT_TYPE_ETH) { - for (i = 1; i < dev->caps.num_ports; i++) - stype[i] = stype[i] ? stype[i] : MLX4_PORT_TYPE_ETH; - } - if (stype[dev->caps.num_ports - 1] == MLX4_PORT_TYPE_IB) { - for (i = 0; i < dev->caps.num_ports - 1; i++) - stype[i] = stype[i] ? stype[i] : MLX4_PORT_TYPE_IB; - } - - /* * If sensed nothing, remain in current configuration. */ for (i = 0; i < dev->caps.num_ports; i++) stype[i] = stype[i] ? stype[i] : defaults[i]; } static void mlx4_sense_port(struct work_struct *work) { struct delayed_work *delay = to_delayed_work(work); struct mlx4_sense *sense = container_of(delay, struct mlx4_sense, sense_poll); struct mlx4_dev *dev = sense->dev; struct mlx4_priv *priv = mlx4_priv(dev); enum mlx4_port_type stype[MLX4_MAX_PORTS]; mutex_lock(&priv->port_mutex); mlx4_do_sense_ports(dev, stype, &dev->caps.port_type[1]); if (mlx4_check_port_params(dev, stype)) goto sense_again; if (mlx4_change_port_types(dev, stype)) mlx4_err(dev, "Failed to change port_types\n"); sense_again: mutex_unlock(&priv->port_mutex); - if (sense->resched) - queue_delayed_work(sense->sense_wq , &sense->sense_poll, - round_jiffies(MLX4_SENSE_RANGE)); + queue_delayed_work(mlx4_wq , &sense->sense_poll, + round_jiffies_relative(MLX4_SENSE_RANGE)); } void mlx4_start_sense(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_sense *sense = &priv->sense; if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) return; - sense->resched = 1; - queue_delayed_work(sense->sense_wq , &sense->sense_poll, - round_jiffies(MLX4_SENSE_RANGE)); + queue_delayed_work(mlx4_wq , &sense->sense_poll, + round_jiffies_relative(MLX4_SENSE_RANGE)); } - void mlx4_stop_sense(struct mlx4_dev *dev) { - mlx4_priv(dev)->sense.resched = 0; + cancel_delayed_work_sync(&mlx4_priv(dev)->sense.sense_poll); } -int mlx4_sense_init(struct mlx4_dev *dev) +void mlx4_sense_init(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); struct mlx4_sense *sense = &priv->sense; int port; sense->dev = dev; - sense->sense_wq = create_singlethread_workqueue("mlx4_sense"); - if (!sense->sense_wq) - return -ENOMEM; - for (port = 1; port <= dev->caps.num_ports; port++) sense->do_sense_port[port] = 1; - INIT_DELAYED_WORK_DEFERRABLE(&sense->sense_poll, mlx4_sense_port); - return 0; + INIT_DEFERRABLE_WORK(&sense->sense_poll, mlx4_sense_port); } - -void mlx4_sense_cleanup(struct mlx4_dev *dev) -{ - mlx4_stop_sense(dev); - cancel_delayed_work(&mlx4_priv(dev)->sense.sense_poll); - destroy_workqueue(mlx4_priv(dev)->sense.sense_wq); -} - Index: stable/9/sys/ofed/drivers/net/mlx4/srq.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/srq.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/srq.c (revision 273864) @@ -1,273 +1,314 @@ /* * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include - #include #include +#include +#include #include "mlx4.h" #include "icm.h" -struct mlx4_srq_context { - __be32 state_logsize_srqn; - u8 logstride; - u8 reserved1; - __be16 xrc_domain; - __be32 pg_offset_cqn; - u32 reserved2; - u8 log_page_size; - u8 reserved3[2]; - u8 mtt_base_addr_h; - __be32 mtt_base_addr_l; - __be32 pd; - __be16 limit_watermark; - __be16 wqe_cnt; - u16 reserved4; - __be16 wqe_counter; - u32 reserved5; - __be64 db_rec_addr; -}; - void mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type) { struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; struct mlx4_srq *srq; spin_lock(&srq_table->lock); - srq = radix_tree_lookup(&dev->srq_table_tree, - srqn & (dev->caps.num_srqs - 1)); + srq = radix_tree_lookup(&srq_table->tree, srqn & (dev->caps.num_srqs - 1)); if (srq) atomic_inc(&srq->refcount); spin_unlock(&srq_table->lock); if (!srq) { mlx4_warn(dev, "Async event for bogus SRQ %08x\n", srqn); return; } srq->event(srq, event_type); if (atomic_dec_and_test(&srq->refcount)) complete(&srq->free); } static int mlx4_SW2HW_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, int srq_num) { - return mlx4_cmd(dev, mailbox->dma, srq_num, 0, MLX4_CMD_SW2HW_SRQ, - MLX4_CMD_TIME_CLASS_A); + return mlx4_cmd(dev, mailbox->dma, srq_num, 0, + MLX4_CMD_SW2HW_SRQ, MLX4_CMD_TIME_CLASS_A, + MLX4_CMD_WRAPPED); } static int mlx4_HW2SW_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, int srq_num) { return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, srq_num, mailbox ? 0 : 1, MLX4_CMD_HW2SW_SRQ, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); } static int mlx4_ARM_SRQ(struct mlx4_dev *dev, int srq_num, int limit_watermark) { return mlx4_cmd(dev, limit_watermark, srq_num, 0, MLX4_CMD_ARM_SRQ, - MLX4_CMD_TIME_CLASS_B); + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); } static int mlx4_QUERY_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, int srq_num) { return mlx4_cmd_box(dev, 0, mailbox->dma, srq_num, 0, MLX4_CMD_QUERY_SRQ, - MLX4_CMD_TIME_CLASS_A); + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); } -int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, u32 cqn, u16 xrcd, - struct mlx4_mtt *mtt, u64 db_rec, struct mlx4_srq *srq) +int __mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn) { struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; - struct mlx4_cmd_mailbox *mailbox; - struct mlx4_srq_context *srq_context; - u64 mtt_addr; int err; - srq->srqn = mlx4_bitmap_alloc(&srq_table->bitmap); - if (srq->srqn == -1) + + *srqn = mlx4_bitmap_alloc(&srq_table->bitmap); + if (*srqn == -1) return -ENOMEM; - err = mlx4_table_get(dev, &srq_table->table, srq->srqn); + err = mlx4_table_get(dev, &srq_table->table, *srqn); if (err) goto err_out; - err = mlx4_table_get(dev, &srq_table->cmpt_table, srq->srqn); + err = mlx4_table_get(dev, &srq_table->cmpt_table, *srqn); if (err) goto err_put; + return 0; +err_put: + mlx4_table_put(dev, &srq_table->table, *srqn); + +err_out: + mlx4_bitmap_free(&srq_table->bitmap, *srqn, MLX4_NO_RR); + return err; +} + +static int mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn) +{ + u64 out_param; + int err; + + if (mlx4_is_mfunc(dev)) { + err = mlx4_cmd_imm(dev, 0, &out_param, RES_SRQ, + RES_OP_RESERVE_AND_MAP, + MLX4_CMD_ALLOC_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); + if (!err) + *srqn = get_param_l(&out_param); + + return err; + } + return __mlx4_srq_alloc_icm(dev, srqn); +} + +void __mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn) +{ + struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; + + mlx4_table_put(dev, &srq_table->cmpt_table, srqn); + mlx4_table_put(dev, &srq_table->table, srqn); + mlx4_bitmap_free(&srq_table->bitmap, srqn, MLX4_NO_RR); +} + +static void mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn) +{ + u64 in_param = 0; + + if (mlx4_is_mfunc(dev)) { + set_param_l(&in_param, srqn); + if (mlx4_cmd(dev, in_param, RES_SRQ, RES_OP_RESERVE_AND_MAP, + MLX4_CMD_FREE_RES, + MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) + mlx4_warn(dev, "Failed freeing cq:%d\n", srqn); + return; + } + __mlx4_srq_free_icm(dev, srqn); +} + +int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, u32 cqn, u16 xrcd, + struct mlx4_mtt *mtt, u64 db_rec, struct mlx4_srq *srq) +{ + struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; + struct mlx4_cmd_mailbox *mailbox; + struct mlx4_srq_context *srq_context; + u64 mtt_addr; + int err; + + err = mlx4_srq_alloc_icm(dev, &srq->srqn); + if (err) + return err; + spin_lock_irq(&srq_table->lock); - err = radix_tree_insert(&dev->srq_table_tree, srq->srqn, srq); + err = radix_tree_insert(&srq_table->tree, srq->srqn, srq); spin_unlock_irq(&srq_table->lock); if (err) - goto err_cmpt_put; + goto err_icm; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) { err = PTR_ERR(mailbox); goto err_radix; } srq_context = mailbox->buf; memset(srq_context, 0, sizeof *srq_context); srq_context->state_logsize_srqn = cpu_to_be32((ilog2(srq->max) << 24) | srq->srqn); srq_context->logstride = srq->wqe_shift - 4; - srq_context->xrc_domain = cpu_to_be16(xrcd); + srq_context->xrcd = cpu_to_be16(xrcd); srq_context->pg_offset_cqn = cpu_to_be32(cqn & 0xffffff); srq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; mtt_addr = mlx4_mtt_addr(dev, mtt); srq_context->mtt_base_addr_h = mtt_addr >> 32; srq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); srq_context->pd = cpu_to_be32(pdn); srq_context->db_rec_addr = cpu_to_be64(db_rec); err = mlx4_SW2HW_SRQ(dev, mailbox, srq->srqn); mlx4_free_cmd_mailbox(dev, mailbox); if (err) goto err_radix; atomic_set(&srq->refcount, 1); init_completion(&srq->free); return 0; err_radix: spin_lock_irq(&srq_table->lock); - radix_tree_delete(&dev->srq_table_tree, srq->srqn); + radix_tree_delete(&srq_table->tree, srq->srqn); spin_unlock_irq(&srq_table->lock); -err_cmpt_put: - mlx4_table_put(dev, &srq_table->cmpt_table, srq->srqn); - -err_put: - mlx4_table_put(dev, &srq_table->table, srq->srqn); - -err_out: - mlx4_bitmap_free(&srq_table->bitmap, srq->srqn); - +err_icm: + mlx4_srq_free_icm(dev, srq->srqn); return err; } EXPORT_SYMBOL_GPL(mlx4_srq_alloc); -void mlx4_srq_invalidate(struct mlx4_dev *dev, struct mlx4_srq *srq) +void mlx4_srq_free(struct mlx4_dev *dev, struct mlx4_srq *srq) { + struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; int err; err = mlx4_HW2SW_SRQ(dev, NULL, srq->srqn); if (err) mlx4_warn(dev, "HW2SW_SRQ failed (%d) for SRQN %06x\n", err, srq->srqn); -} -EXPORT_SYMBOL_GPL(mlx4_srq_invalidate); -void mlx4_srq_remove(struct mlx4_dev *dev, struct mlx4_srq *srq) -{ - struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; - spin_lock_irq(&srq_table->lock); - radix_tree_delete(&dev->srq_table_tree, srq->srqn); + radix_tree_delete(&srq_table->tree, srq->srqn); spin_unlock_irq(&srq_table->lock); -} -EXPORT_SYMBOL_GPL(mlx4_srq_remove); -void mlx4_srq_free(struct mlx4_dev *dev, struct mlx4_srq *srq) -{ - struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; - if (atomic_dec_and_test(&srq->refcount)) complete(&srq->free); wait_for_completion(&srq->free); - mlx4_table_put(dev, &srq_table->table, srq->srqn); - mlx4_bitmap_free(&srq_table->bitmap, srq->srqn); + mlx4_srq_free_icm(dev, srq->srqn); } EXPORT_SYMBOL_GPL(mlx4_srq_free); int mlx4_srq_arm(struct mlx4_dev *dev, struct mlx4_srq *srq, int limit_watermark) { return mlx4_ARM_SRQ(dev, srq->srqn, limit_watermark); } EXPORT_SYMBOL_GPL(mlx4_srq_arm); int mlx4_srq_query(struct mlx4_dev *dev, struct mlx4_srq *srq, int *limit_watermark) { struct mlx4_cmd_mailbox *mailbox; struct mlx4_srq_context *srq_context; int err; mailbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); srq_context = mailbox->buf; err = mlx4_QUERY_SRQ(dev, mailbox, srq->srqn); if (err) goto err_out; *limit_watermark = be16_to_cpu(srq_context->limit_watermark); err_out: mlx4_free_cmd_mailbox(dev, mailbox); return err; } EXPORT_SYMBOL_GPL(mlx4_srq_query); int mlx4_init_srq_table(struct mlx4_dev *dev) { struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; int err; spin_lock_init(&srq_table->lock); - INIT_RADIX_TREE(&dev->srq_table_tree, GFP_ATOMIC); + INIT_RADIX_TREE(&srq_table->tree, GFP_ATOMIC); + if (mlx4_is_slave(dev)) + return 0; err = mlx4_bitmap_init(&srq_table->bitmap, dev->caps.num_srqs, dev->caps.num_srqs - 1, dev->caps.reserved_srqs, 0); if (err) return err; return 0; } void mlx4_cleanup_srq_table(struct mlx4_dev *dev) { + if (mlx4_is_slave(dev)) + return; mlx4_bitmap_cleanup(&mlx4_priv(dev)->srq_table.bitmap); } + +struct mlx4_srq *mlx4_srq_lookup(struct mlx4_dev *dev, u32 srqn) +{ + struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; + struct mlx4_srq *srq; + unsigned long flags; + + spin_lock_irqsave(&srq_table->lock, flags); + srq = radix_tree_lookup(&srq_table->tree, + srqn & (dev->caps.num_srqs - 1)); + spin_unlock_irqrestore(&srq_table->lock, flags); + + return srq; +} +EXPORT_SYMBOL_GPL(mlx4_srq_lookup); Index: stable/9/sys/ofed/drivers/net/mlx4/sys_tune.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/sys_tune.c (nonexistent) +++ stable/9/sys/ofed/drivers/net/mlx4/sys_tune.c (revision 273864) @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2010, 2014 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#include +#include +#include + +#include "mlx4.h" + +#if defined(CONFIG_X86) && defined(CONFIG_APM_MODULE) + +/* Each CPU is put into a group. In most cases, the group number is + * equal to the CPU number of one of the CPUs in the group. The + * exception is group NR_CPUS which is the default group. This is + * protected by sys_tune_startup_mutex. */ +DEFINE_PER_CPU(int, idle_cpu_group) = NR_CPUS; + +/* For each group, a count of the number of CPUs in the group which + * are known to be busy. A busy CPU might be running the busy loop + * below or general kernel code. The count is decremented on entry to + * the old pm_idle handler and incremented on exit. The aim is to + * avoid the count going to zero or negative. This situation can + * occur temporarily during module unload or CPU hot-plug but + * normality will be restored when the affected CPUs next exit the + * idle loop. */ +static atomic_t busy_cpu_count[NR_CPUS+1]; + +/* A workqueue item to be executed to cause the CPU to exit from the + * idle loop. */ +DEFINE_PER_CPU(struct work_struct, sys_tune_cpu_work); + +#define sys_tune_set_state(CPU,STATE) \ + do { } while(0) + + +/* A mutex to protect most of the module datastructures. */ +static DEFINE_MUTEX(sys_tune_startup_mutex); + +/* The old pm_idle handler. */ +static void (*old_pm_idle)(void) = NULL; + +static void sys_tune_pm_idle(void) +{ + atomic_t *busy_cpus_ptr; + int busy_cpus; + int cpu = smp_processor_id(); + + busy_cpus_ptr = &(busy_cpu_count[per_cpu(idle_cpu_group, cpu)]); + + sys_tune_set_state(cpu, 2); + + local_irq_enable(); + while (!need_resched()) { + busy_cpus = atomic_read(busy_cpus_ptr); + + /* If other CPUs in this group are busy then let this + * CPU go idle. We mustn't let the number of busy + * CPUs drop below 1. */ + if ( busy_cpus > 1 && + old_pm_idle != NULL && + ( atomic_cmpxchg(busy_cpus_ptr, busy_cpus, + busy_cpus-1) == busy_cpus ) ) { + local_irq_disable(); + sys_tune_set_state(cpu, 3); + /* This check might not be necessary, but it + * seems safest to include it because there + * might be a kernel version which requires + * it. */ + if (need_resched()) + local_irq_enable(); + else + old_pm_idle(); + /* This CPU is busy again. */ + sys_tune_set_state(cpu, 1); + atomic_add(1, busy_cpus_ptr); + return; + } + + cpu_relax(); + } + sys_tune_set_state(cpu, 0); +} + + +void sys_tune_work_func(struct work_struct *work) +{ + /* Do nothing. Since this function is running in process + * context, the idle thread isn't running on this CPU. */ +} + + +#ifdef CONFIG_SMP +static void sys_tune_smp_call(void *info) +{ + schedule_work(&get_cpu_var(sys_tune_cpu_work)); + put_cpu_var(sys_tune_cpu_work); +} +#endif + + +#ifdef CONFIG_SMP +static void sys_tune_refresh(void) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) + on_each_cpu(&sys_tune_smp_call, NULL, 0, 1); +#else + on_each_cpu(&sys_tune_smp_call, NULL, 1); +#endif +} +#else +static void sys_tune_refresh(void) +{ + /* The current thread is executing on the one and only CPU so + * the idle thread isn't running. */ +} +#endif + + + +static int sys_tune_cpu_group(int cpu) +{ +#ifdef CONFIG_SMP + const cpumask_t *mask; + int other_cpu; + int group; + +#if defined(topology_thread_cpumask) && defined(ST_HAVE_EXPORTED_CPU_SIBLING_MAP) + /* Keep one hyperthread busy per core. */ + mask = topology_thread_cpumask(cpu); +#else + return cpu; +#endif + for_each_cpu_mask(cpu, *(mask)) { + group = per_cpu(idle_cpu_group, other_cpu); + if (group != NR_CPUS) + return group; + } +#endif + + return cpu; +} + + +static void sys_tune_add_cpu(int cpu) +{ + int group; + + /* Do nothing if this CPU has already been added. */ + if (per_cpu(idle_cpu_group, cpu) != NR_CPUS) + return; + + group = sys_tune_cpu_group(cpu); + per_cpu(idle_cpu_group, cpu) = group; + atomic_inc(&(busy_cpu_count[group])); + +} + +static void sys_tune_del_cpu(int cpu) +{ + + int group; + + if (per_cpu(idle_cpu_group, cpu) == NR_CPUS) + return; + + group = per_cpu(idle_cpu_group, cpu); + /* If the CPU was busy, this can cause the count to drop to + * zero. To rectify this, we need to cause one of the other + * CPUs in the group to exit the idle loop. If the CPU was + * not busy then this causes the contribution for this CPU to + * go to -1 which can cause the overall count to drop to zero + * or go negative. To rectify this situation we need to cause + * this CPU to exit the idle loop. */ + atomic_dec(&(busy_cpu_count[group])); + per_cpu(idle_cpu_group, cpu) = NR_CPUS; + +} + + +static int sys_tune_cpu_notify(struct notifier_block *self, + unsigned long action, void *hcpu) +{ + int cpu = (long)hcpu; + + switch(action) { +#ifdef CPU_ONLINE_FROZEN + case CPU_ONLINE_FROZEN: +#endif + case CPU_ONLINE: + mutex_lock(&sys_tune_startup_mutex); + sys_tune_add_cpu(cpu); + mutex_unlock(&sys_tune_startup_mutex); + /* The CPU might have already entered the idle loop in + * the wrong group. Make sure it exits the idle loop + * so that it picks up the correct group. */ + sys_tune_refresh(); + break; + +#ifdef CPU_DEAD_FROZEN + case CPU_DEAD_FROZEN: +#endif + case CPU_DEAD: + mutex_lock(&sys_tune_startup_mutex); + sys_tune_del_cpu(cpu); + mutex_unlock(&sys_tune_startup_mutex); + /* The deleted CPU may have been the only busy CPU in + * the group. Make sure one of the other CPUs in the + * group exits the idle loop. */ + sys_tune_refresh(); + break; + } + return NOTIFY_OK; +} + + +static struct notifier_block sys_tune_cpu_nb = { + .notifier_call = sys_tune_cpu_notify, +}; + + +static void sys_tune_ensure_init(void) +{ + BUG_ON (old_pm_idle != NULL); + + /* Atomically update pm_idle to &sys_tune_pm_idle. The old value + * is stored in old_pm_idle before installing the new + * handler. */ + do { + old_pm_idle = pm_idle; + } while (cmpxchg(&pm_idle, old_pm_idle, &sys_tune_pm_idle) != + old_pm_idle); +} +#endif + +void sys_tune_fini(void) +{ +#if defined(CONFIG_X86) && defined(CONFIG_APM_MODULE) + void (*old)(void); + int cpu; + + unregister_cpu_notifier(&sys_tune_cpu_nb); + + mutex_lock(&sys_tune_startup_mutex); + + + old = cmpxchg(&pm_idle, &sys_tune_pm_idle, old_pm_idle); + + for_each_online_cpu(cpu) + sys_tune_del_cpu(cpu); + + mutex_unlock(&sys_tune_startup_mutex); + + /* Our handler may still be executing on other CPUs. + * Schedule this thread on all CPUs to make sure all + * idle threads get interrupted. */ + sys_tune_refresh(); + + /* Make sure the work item has finished executing on all CPUs. + * This in turn ensures that all idle threads have been + * interrupted. */ + flush_scheduled_work(); +#endif /* CONFIG_X86 */ +} + +void sys_tune_init(void) +{ +#if defined(CONFIG_X86) && defined(CONFIG_APM_MODULE) + int cpu; + + for_each_possible_cpu(cpu) { + INIT_WORK(&per_cpu(sys_tune_cpu_work, cpu), + sys_tune_work_func); + } + + /* Start by registering the handler to ensure we don't miss + * any updates. */ + register_cpu_notifier(&sys_tune_cpu_nb); + + mutex_lock(&sys_tune_startup_mutex); + + for_each_online_cpu(cpu) + sys_tune_add_cpu(cpu); + + sys_tune_ensure_init(); + + + mutex_unlock(&sys_tune_startup_mutex); + + /* Ensure our idle handler starts to run. */ + sys_tune_refresh(); +#endif +} + Property changes on: stable/9/sys/ofed/drivers/net/mlx4/sys_tune.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/drivers/net/mlx4/utils.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/utils.c (nonexistent) +++ stable/9/sys/ofed/drivers/net/mlx4/utils.c (revision 273864) @@ -0,0 +1,189 @@ +/* $OpenBSD: if_trunk.c,v 1.30 2007/01/31 06:20:19 reyk Exp $ */ + +/* + * Copyright (c) 2005, 2006 Reyk Floeter + * Copyright (c) 2007 Andrew Thompson + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(INET) || defined(INET6) +#include +#endif +#ifdef INET +#include +#include +#include +#endif + +#ifdef INET6 +#include +#include +#include +#endif + +#include + +#include "utils.h" + +/* XXX this code should be factored out */ +/* XXX copied from if_lagg.c */ + +static const void * +mlx4_en_gethdr(struct mbuf *m, u_int off, u_int len, void *buf) +{ + if (m->m_pkthdr.len < (off + len)) { + return (NULL); + } else if (m->m_len < (off + len)) { + m_copydata(m, off, len, buf); + return (buf); + } + return (mtod(m, char *) + off); +} + +uint32_t +mlx4_en_hashmbuf(uint32_t flags, struct mbuf *m, uint32_t key) +{ + uint16_t etype; + uint32_t p = key; + int off; + struct ether_header *eh; + const struct ether_vlan_header *vlan; +#ifdef INET + const struct ip *ip; + const uint32_t *ports; + int iphlen; +#endif +#ifdef INET6 + const struct ip6_hdr *ip6; + uint32_t flow; +#endif + union { +#ifdef INET + struct ip ip; +#endif +#ifdef INET6 + struct ip6_hdr ip6; +#endif + struct ether_vlan_header vlan; + uint32_t port; + } buf; + + + off = sizeof(*eh); + if (m->m_len < off) + goto out; + eh = mtod(m, struct ether_header *); + etype = ntohs(eh->ether_type); + if (flags & MLX4_F_HASHL2) { + p = hash32_buf(&eh->ether_shost, ETHER_ADDR_LEN, p); + p = hash32_buf(&eh->ether_dhost, ETHER_ADDR_LEN, p); + } + + /* Special handling for encapsulating VLAN frames */ + if ((m->m_flags & M_VLANTAG) && (flags & MLX4_F_HASHL2)) { + p = hash32_buf(&m->m_pkthdr.ether_vtag, + sizeof(m->m_pkthdr.ether_vtag), p); + } else if (etype == ETHERTYPE_VLAN) { + vlan = mlx4_en_gethdr(m, off, sizeof(*vlan), &buf); + if (vlan == NULL) + goto out; + + if (flags & MLX4_F_HASHL2) + p = hash32_buf(&vlan->evl_tag, sizeof(vlan->evl_tag), p); + etype = ntohs(vlan->evl_proto); + off += sizeof(*vlan) - sizeof(*eh); + } + + switch (etype) { +#ifdef INET + case ETHERTYPE_IP: + ip = mlx4_en_gethdr(m, off, sizeof(*ip), &buf); + if (ip == NULL) + goto out; + + if (flags & MLX4_F_HASHL3) { + p = hash32_buf(&ip->ip_src, sizeof(struct in_addr), p); + p = hash32_buf(&ip->ip_dst, sizeof(struct in_addr), p); + } + if (!(flags & MLX4_F_HASHL4)) + break; + switch (ip->ip_p) { + case IPPROTO_TCP: + case IPPROTO_UDP: + case IPPROTO_SCTP: + iphlen = ip->ip_hl << 2; + if (iphlen < sizeof(*ip)) + break; + off += iphlen; + ports = mlx4_en_gethdr(m, off, sizeof(*ports), &buf); + if (ports == NULL) + break; + p = hash32_buf(ports, sizeof(*ports), p); + break; + } + break; +#endif +#ifdef INET6 + case ETHERTYPE_IPV6: + if (!(flags & MLX4_F_HASHL3)) + break; + ip6 = mlx4_en_gethdr(m, off, sizeof(*ip6), &buf); + if (ip6 == NULL) + goto out; + + p = hash32_buf(&ip6->ip6_src, sizeof(struct in6_addr), p); + p = hash32_buf(&ip6->ip6_dst, sizeof(struct in6_addr), p); + flow = ip6->ip6_flow & IPV6_FLOWLABEL_MASK; + p = hash32_buf(&flow, sizeof(flow), p); /* IPv6 flow label */ + break; +#endif + } +out: + return (p); +} Property changes on: stable/9/sys/ofed/drivers/net/mlx4/utils.c ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/drivers/net/mlx4/utils.h =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/utils.h (nonexistent) +++ stable/9/sys/ofed/drivers/net/mlx4/utils.h (revision 273864) @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2014 Mellanox Technologies Ltd. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _MLX4_UTILS_H_ +#define _MLX4_UTILS_H_ + +/* Lagg flags */ +#define MLX4_F_HASHL2 0x00000001 /* hash layer 2 */ +#define MLX4_F_HASHL3 0x00000002 /* hash layer 3 */ +#define MLX4_F_HASHL4 0x00000004 /* hash layer 4 */ +#define MLX4_F_HASHMASK 0x00000007 + +uint32_t mlx4_en_hashmbuf(uint32_t flags, struct mbuf *m, uint32_t key); + +#endif /* _MLX4_UTILS_H_ */ Property changes on: stable/9/sys/ofed/drivers/net/mlx4/utils.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/drivers/net/mlx4/xrcd.c =================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/xrcd.c (revision 273863) +++ stable/9/sys/ofed/drivers/net/mlx4/xrcd.c (revision 273864) @@ -1,70 +1,69 @@ /* * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include #include #include "mlx4.h" int mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn) { struct mlx4_priv *priv = mlx4_priv(dev); *xrcdn = mlx4_bitmap_alloc(&priv->xrcd_bitmap); if (*xrcdn == -1) return -ENOMEM; return 0; } EXPORT_SYMBOL_GPL(mlx4_xrcd_alloc); void mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn) { mlx4_bitmap_free(&mlx4_priv(dev)->xrcd_bitmap, xrcdn); } EXPORT_SYMBOL_GPL(mlx4_xrcd_free); int __devinit mlx4_init_xrcd_table(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); return mlx4_bitmap_init(&priv->xrcd_bitmap, (1 << 16), (1 << 16) - 1, dev->caps.reserved_xrcds + 1, 0); } void mlx4_cleanup_xrcd_table(struct mlx4_dev *dev) { mlx4_bitmap_cleanup(&mlx4_priv(dev)->xrcd_bitmap); } Index: stable/9/sys/ofed/include/asm/current.h =================================================================== --- stable/9/sys/ofed/include/asm/current.h (revision 273863) +++ stable/9/sys/ofed/include/asm/current.h (nonexistent) @@ -1,32 +0,0 @@ -/*- - * Copyright (c) 2010 Isilon Systems, Inc. - * Copyright (c) 2010 iX Systems, Inc. - * Copyright (c) 2010 Panasas, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _ASM_CURRENT_H_ -#define _ASM_CURRENT_H_ - -#endif /* _ASM_CURRENT_H_ */ Property changes on: stable/9/sys/ofed/include/asm/current.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/9/sys/ofed/include/asm/system.h =================================================================== --- stable/9/sys/ofed/include/asm/system.h (revision 273863) +++ stable/9/sys/ofed/include/asm/system.h (nonexistent) @@ -1,27 +0,0 @@ -/*- - * Copyright (c) 2010 Isilon Systems, Inc. - * Copyright (c) 2010 iX Systems, Inc. - * Copyright (c) 2010 Panasas, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ Property changes on: stable/9/sys/ofed/include/asm/system.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/9/sys/ofed/include/asm/semaphore.h =================================================================== --- stable/9/sys/ofed/include/asm/semaphore.h (revision 273863) +++ stable/9/sys/ofed/include/asm/semaphore.h (nonexistent) @@ -1,34 +0,0 @@ -/*- - * Copyright (c) 2010 Isilon Systems, Inc. - * Copyright (c) 2010 iX Systems, Inc. - * Copyright (c) 2010 Panasas, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _ASM_SEMAPHORE_H_ -#define _ASM_SEMAPHORE_H_ - -#include - -#endif /* _ASM_SEMAPHORE_H_ */ Property changes on: stable/9/sys/ofed/include/asm/semaphore.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/9/sys/ofed/include/asm/atomic-long.h =================================================================== --- stable/9/sys/ofed/include/asm/atomic-long.h (revision 273863) +++ stable/9/sys/ofed/include/asm/atomic-long.h (revision 273864) @@ -1,79 +1,81 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef _ATOMIC_LONG_H_ #define _ATOMIC_LONG_H_ #include #include #include typedef struct { volatile u_long counter; } atomic_long_t; #define atomic_long_add(i, v) atomic_long_add_return((i), (v)) #define atomic_long_inc_return(v) atomic_long_add_return(1, (v)) static inline long atomic_long_add_return(long i, atomic_long_t *v) { return i + atomic_fetchadd_long(&v->counter, i); } static inline void atomic_long_set(atomic_long_t *v, long i) { atomic_store_rel_long(&v->counter, i); } static inline long atomic_long_read(atomic_long_t *v) { return atomic_load_acq_long(&v->counter); } static inline long atomic_long_inc(atomic_long_t *v) { return atomic_fetchadd_long(&v->counter, 1) + 1; } static inline long atomic_long_dec(atomic_long_t *v) { return atomic_fetchadd_long(&v->counter, -1) - 1; } static inline long atomic_long_dec_and_test(atomic_long_t *v) { long i = atomic_long_add(-1, v); return i == 0 ; } #endif /* _ATOMIC_LONG_H_ */ Index: stable/9/sys/ofed/include/asm/atomic.h =================================================================== --- stable/9/sys/ofed/include/asm/atomic.h (revision 273863) +++ stable/9/sys/ofed/include/asm/atomic.h (revision 273864) @@ -1,85 +1,106 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _ASM_ATOMIC_H_ #define _ASM_ATOMIC_H_ #include #include #include -#include typedef struct { volatile u_int counter; } atomic_t; #define atomic_add(i, v) atomic_add_return((i), (v)) #define atomic_sub(i, v) atomic_sub_return((i), (v)) #define atomic_inc_return(v) atomic_add_return(1, (v)) #define atomic_add_negative(i, v) (atomic_add_return((i), (v)) < 0) #define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0) #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) #define atomic_inc_and_test(v) (atomic_add_return(1, (v)) == 0) +#define atomic_dec_return(v) atomic_sub_return(1, (v)) static inline int atomic_add_return(int i, atomic_t *v) { return i + atomic_fetchadd_int(&v->counter, i); } static inline int atomic_sub_return(int i, atomic_t *v) { return atomic_fetchadd_int(&v->counter, -i) - i; } static inline void atomic_set(atomic_t *v, int i) { atomic_store_rel_int(&v->counter, i); } static inline int atomic_read(atomic_t *v) { return atomic_load_acq_int(&v->counter); } static inline int atomic_inc(atomic_t *v) { return atomic_fetchadd_int(&v->counter, 1) + 1; } static inline int atomic_dec(atomic_t *v) { return atomic_fetchadd_int(&v->counter, -1) - 1; } + +static inline int atomic_add_unless(atomic_t *v, int a, int u) +{ + int c, old; + c = atomic_read(v); + for (;;) { + if (unlikely(c == (u))) + break; + old = atomic_cmpset_int(&v->counter, c, c + (a)); + if (likely(old == c)) + break; + c = old; + } + return c != (u); +} + +#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) + + + #endif /* _ASM_ATOMIC_H_ */ Index: stable/9/sys/ofed/include/asm/byteorder.h =================================================================== --- stable/9/sys/ofed/include/asm/byteorder.h (revision 273863) +++ stable/9/sys/ofed/include/asm/byteorder.h (revision 273864) @@ -1,90 +1,93 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef _ASM_BYTEORDER_H_ #define _ASM_BYTEORDER_H_ #include #include +#include #if BYTE_ORDER == LITTLE_ENDIAN #define __LITTLE_ENDIAN #else #define __BIG_ENDIAN #endif #define cpu_to_le64 htole64 #define le64_to_cpu le64toh #define cpu_to_le32 htole32 #define le32_to_cpu le32toh #define cpu_to_le16 htole16 #define le16_to_cpu le16toh #define cpu_to_be64 htobe64 #define be64_to_cpu be64toh #define cpu_to_be32 htobe32 #define be32_to_cpu be32toh #define cpu_to_be16 htobe16 #define be16_to_cpu be16toh #define __be16_to_cpu be16toh #define cpu_to_le64p(x) htole64(*((uint64_t *)x)) #define le64_to_cpup(x) le64toh(*((uint64_t *)x)) #define cpu_to_le32p(x) htole32(*((uint32_t *)x)) #define le32_to_cpup(x) le32toh(*((uint32_t *)x)) #define cpu_to_le16p(x) htole16(*((uint16_t *)x)) #define le16_to_cpup(x) le16toh(*((uint16_t *)x)) #define cpu_to_be64p(x) htobe64(*((uint64_t *)x)) #define be64_to_cpup(x) be64toh(*((uint64_t *)x)) #define cpu_to_be32p(x) htobe32(*((uint32_t *)x)) #define be32_to_cpup(x) be32toh(*((uint32_t *)x)) #define cpu_to_be16p(x) htobe16(*((uint16_t *)x)) #define be16_to_cpup(x) be16toh(*((uint16_t *)x)) #define cpu_to_le64s(x) do { *((uint64_t *)x) = cpu_to_le64p((x)) } while (0) #define le64_to_cpus(x) do { *((uint64_t *)x) = le64_to_cpup((x)) } while (0) #define cpu_to_le32s(x) do { *((uint32_t *)x) = cpu_to_le32p((x)) } while (0) #define le32_to_cpus(x) do { *((uint32_t *)x) = le32_to_cpup((x)) } while (0) #define cpu_to_le16s(x) do { *((uint16_t *)x) = cpu_to_le16p((x)) } while (0) #define le16_to_cpus(x) do { *((uint16_t *)x) = le16_to_cpup((x)) } while (0) #define cpu_to_be64s(x) do { *((uint64_t *)x) = cpu_to_be64p((x)) } while (0) #define be64_to_cpus(x) do { *((uint64_t *)x) = be64_to_cpup((x)) } while (0) #define cpu_to_be32s(x) do { *((uint32_t *)x) = cpu_to_be32p((x)) } while (0) #define be32_to_cpus(x) do { *((uint32_t *)x) = be32_to_cpup((x)) } while (0) #define cpu_to_be16s(x) do { *((uint16_t *)x) = cpu_to_be16p((x)) } while (0) #define be16_to_cpus(x) do { *((uint16_t *)x) = be16_to_cpup((x)) } while (0) #define swab16 bswap16 #define swab32 bswap32 #define swab64 bswap64 static inline void be16_add_cpu(u16 *var, u16 val) { *var = cpu_to_be16(be16_to_cpu(*var) + val); } #endif /* _ASM_BYTEORDER_H_ */ Index: stable/9/sys/ofed/include/asm/fcntl.h =================================================================== --- stable/9/sys/ofed/include/asm/fcntl.h (revision 273863) +++ stable/9/sys/ofed/include/asm/fcntl.h (revision 273864) @@ -1,33 +1,34 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _ASM_FCNTL_H_ #define _ASM_FCNTL_H_ #include #endif /* _ASM_FCNTL_H_ */ Index: stable/9/sys/ofed/include/asm/io.h =================================================================== --- stable/9/sys/ofed/include/asm/io.h (revision 273863) +++ stable/9/sys/ofed/include/asm/io.h (revision 273864) @@ -1,29 +1,35 @@ -/*- +/* * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#ifndef _ASM_IO_H_ +#define _ASM_IO_H_ + #include + +#endif /* _ASM_IO_H_ */ Index: stable/9/sys/ofed/include/asm/page.h =================================================================== --- stable/9/sys/ofed/include/asm/page.h (revision 273863) +++ stable/9/sys/ofed/include/asm/page.h (revision 273864) @@ -1,29 +1,35 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#ifndef _ASM_PAGE_H_ +#define _ASM_PAGE_H_ + #include + +#endif /*_ASM_PAGE_H_*/ Index: stable/9/sys/ofed/include/asm/pgtable.h =================================================================== --- stable/9/sys/ofed/include/asm/pgtable.h (revision 273863) +++ stable/9/sys/ofed/include/asm/pgtable.h (revision 273864) @@ -1,33 +1,34 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _ASM_PGTABLE_H_ #define _ASM_PGTABLE_H_ typedef int pgprot_t; #endif /* _ASM_PGTABLE_H_ */ Index: stable/9/sys/ofed/include/asm/types.h =================================================================== --- stable/9/sys/ofed/include/asm/types.h (revision 273863) +++ stable/9/sys/ofed/include/asm/types.h (revision 273864) @@ -1,67 +1,61 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef _ASM_TYPES_H_ #define _ASM_TYPES_H_ -typedef unsigned short umode_t; - -typedef __signed__ char __s8; -typedef unsigned char __u8; - -typedef __signed__ short __s16; -typedef unsigned short __u16; - -typedef __signed__ int __s32; -typedef unsigned int __u32; - -#if defined(__GNUC__) // && !defined(__STRICT_ANSI__) -typedef __signed__ long long __s64; -typedef unsigned long long __u64; -#endif - #ifdef _KERNEL -typedef signed char s8; -typedef unsigned char u8; +typedef uint8_t u8; +typedef uint8_t __u8; +typedef uint16_t u16; +typedef uint16_t __u16; +typedef uint32_t u32; +typedef uint32_t __u32; +typedef uint64_t u64; +typedef uint64_t __u64; -typedef signed short s16; -typedef unsigned short u16; +typedef int8_t s8; +typedef int8_t __s8; +typedef int16_t s16; +typedef int16_t __s16; +typedef int32_t s32; +typedef int32_t __s32; +typedef int64_t s64; +typedef int64_t __s64; -typedef signed int s32; -typedef unsigned int u32; - -typedef signed long long s64; -typedef unsigned long long u64; - /* DMA addresses come in generic and 64-bit flavours. */ typedef vm_paddr_t dma_addr_t; typedef vm_paddr_t dma64_addr_t; + +typedef unsigned short umode_t; #endif /* _KERNEL */ #endif /* _ASM_TYPES_H_ */ Index: stable/9/sys/ofed/include/asm/uaccess.h =================================================================== --- stable/9/sys/ofed/include/asm/uaccess.h (revision 273863) +++ stable/9/sys/ofed/include/asm/uaccess.h (revision 273864) @@ -1,49 +1,51 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef _ASM_UACCESS_H_ #define _ASM_UACCESS_H_ #include static inline long copy_to_user(void *to, const void *from, unsigned long n) { if (copyout(from, to, n) != 0) return n; return 0; } static inline long copy_from_user(void *to, const void *from, unsigned long n) { if (copyin(from, to, n) != 0) return n; return 0; } #endif /* _ASM_UACCESS_H_ */ Index: stable/9/sys/ofed/include/linux/bitmap.h =================================================================== --- stable/9/sys/ofed/include/linux/bitmap.h (revision 273863) +++ stable/9/sys/ofed/include/linux/bitmap.h (nonexistent) @@ -1,34 +0,0 @@ -/*- - * Copyright (c) 2010 Isilon Systems, Inc. - * Copyright (c) 2010 iX Systems, Inc. - * Copyright (c) 2010 Panasas, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#ifndef _LINUX_BITMAP_H_ -#define _LINUX_BITMAP_H_ - -#include -#include - -#endif /* _LINUX_BITMAP_H_ */ Property changes on: stable/9/sys/ofed/include/linux/bitmap.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/9/sys/ofed/include/linux/stddef.h =================================================================== --- stable/9/sys/ofed/include/linux/stddef.h (revision 273863) +++ stable/9/sys/ofed/include/linux/stddef.h (nonexistent) @@ -1,34 +0,0 @@ -/*- - * Copyright (c) 2010 Isilon Systems, Inc. - * Copyright (c) 2010 iX Systems, Inc. - * Copyright (c) 2010 Panasas, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _LINUX_STDDEF_H_ -#define _LINUX_STDDEF_H_ - -#include - -#endif /* _LINUX_STDDEF_H_ */ Property changes on: stable/9/sys/ofed/include/linux/stddef.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/9/sys/ofed/include/linux/rtnetlink.h =================================================================== --- stable/9/sys/ofed/include/linux/rtnetlink.h (revision 273863) +++ stable/9/sys/ofed/include/linux/rtnetlink.h (nonexistent) @@ -1,27 +0,0 @@ -/*- - * Copyright (c) 2010 Isilon Systems, Inc. - * Copyright (c) 2010 iX Systems, Inc. - * Copyright (c) 2010 Panasas, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ Property changes on: stable/9/sys/ofed/include/linux/rtnetlink.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/9/sys/ofed/include/linux/init.h =================================================================== --- stable/9/sys/ofed/include/linux/init.h (revision 273863) +++ stable/9/sys/ofed/include/linux/init.h (nonexistent) @@ -1,31 +0,0 @@ -/*- - * Copyright (c) 2010 Isilon Systems, Inc. - * Copyright (c) 2010 iX Systems, Inc. - * Copyright (c) 2010 Panasas, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#ifndef _LINUX_INIT_H_ -#define _LINUX_INIT_H_ - -#endif /* _LINUX_INIT_H_ */ Property changes on: stable/9/sys/ofed/include/linux/init.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/9/sys/ofed/include/linux/ctype.h =================================================================== --- stable/9/sys/ofed/include/linux/ctype.h (revision 273863) +++ stable/9/sys/ofed/include/linux/ctype.h (nonexistent) @@ -1,34 +0,0 @@ -/*- - * Copyright (c) 2010 Isilon Systems, Inc. - * Copyright (c) 2010 iX Systems, Inc. - * Copyright (c) 2010 Panasas, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _LINUX_CTYPE_H_ -#define _LINUX_CTYPE_H_ - -#include - -#endif /* _LINUX_CTYPE_H_ */ Property changes on: stable/9/sys/ofed/include/linux/ctype.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/9/sys/ofed/include/linux/bitops.h =================================================================== --- stable/9/sys/ofed/include/linux/bitops.h (revision 273863) +++ stable/9/sys/ofed/include/linux/bitops.h (revision 273864) @@ -1,317 +1,512 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_BITOPS_H_ #define _LINUX_BITOPS_H_ #ifdef __LP64__ #define BITS_PER_LONG 64 #else #define BITS_PER_LONG 32 #endif #define BIT_MASK(n) (~0UL >> (BITS_PER_LONG - (n))) #define BITS_TO_LONGS(n) howmany((n), BITS_PER_LONG) +#define BIT_WORD(nr) ((nr) / BITS_PER_LONG) +#define BITS_PER_BYTE 8 + static inline int __ffs(int mask) { return (ffs(mask) - 1); } static inline int __fls(int mask) { return (fls(mask) - 1); } static inline int __ffsl(long mask) { return (ffsl(mask) - 1); } static inline int __flsl(long mask) { return (flsl(mask) - 1); } #define ffz(mask) __ffs(~(mask)) +static inline int get_count_order(unsigned int count) +{ + int order; + + order = fls(count) - 1; + if (count & (count - 1)) + order++; + return order; +} + static inline unsigned long find_first_bit(unsigned long *addr, unsigned long size) { long mask; int bit; for (bit = 0; size >= BITS_PER_LONG; size -= BITS_PER_LONG, bit += BITS_PER_LONG, addr++) { if (*addr == 0) continue; return (bit + __ffsl(*addr)); } if (size) { mask = (*addr) & BIT_MASK(size); if (mask) bit += __ffsl(mask); else bit += size; } return (bit); } static inline unsigned long find_first_zero_bit(unsigned long *addr, unsigned long size) { long mask; int bit; for (bit = 0; size >= BITS_PER_LONG; size -= BITS_PER_LONG, bit += BITS_PER_LONG, addr++) { if (~(*addr) == 0) continue; return (bit + __ffsl(~(*addr))); } if (size) { mask = ~(*addr) & BIT_MASK(size); if (mask) bit += __ffsl(mask); else bit += size; } return (bit); } static inline unsigned long find_last_bit(unsigned long *addr, unsigned long size) { long mask; int offs; int bit; int pos; pos = size / BITS_PER_LONG; offs = size % BITS_PER_LONG; bit = BITS_PER_LONG * pos; addr += pos; if (offs) { mask = (*addr) & BIT_MASK(offs); if (mask) return (bit + __flsl(mask)); } while (--pos) { addr--; bit -= BITS_PER_LONG; if (*addr) return (bit + __flsl(mask)); } return (size); } static inline unsigned long find_next_bit(unsigned long *addr, unsigned long size, unsigned long offset) { long mask; int offs; int bit; int pos; if (offset >= size) return (size); pos = offset / BITS_PER_LONG; offs = offset % BITS_PER_LONG; bit = BITS_PER_LONG * pos; addr += pos; if (offs) { mask = (*addr) & ~BIT_MASK(offs); if (mask) return (bit + __ffsl(mask)); bit += BITS_PER_LONG; addr++; } for (size -= bit; size >= BITS_PER_LONG; size -= BITS_PER_LONG, bit += BITS_PER_LONG, addr++) { if (*addr == 0) continue; return (bit + __ffsl(*addr)); } if (size) { mask = (*addr) & BIT_MASK(size); if (mask) bit += __ffsl(mask); else bit += size; } return (bit); } static inline unsigned long find_next_zero_bit(unsigned long *addr, unsigned long size, unsigned long offset) { long mask; int offs; int bit; int pos; if (offset >= size) return (size); pos = offset / BITS_PER_LONG; offs = offset % BITS_PER_LONG; bit = BITS_PER_LONG * pos; addr += pos; if (offs) { mask = ~(*addr) & ~BIT_MASK(offs); if (mask) return (bit + __ffsl(mask)); bit += BITS_PER_LONG; addr++; } for (size -= bit; size >= BITS_PER_LONG; size -= BITS_PER_LONG, bit += BITS_PER_LONG, addr++) { if (~(*addr) == 0) continue; return (bit + __ffsl(~(*addr))); } if (size) { mask = ~(*addr) & BIT_MASK(size); if (mask) bit += __ffsl(mask); else bit += size; } return (bit); } static inline void bitmap_zero(unsigned long *addr, int size) { int len; len = BITS_TO_LONGS(size) * sizeof(long); memset(addr, 0, len); } static inline void bitmap_fill(unsigned long *addr, int size) { int tail; int len; len = (size / BITS_PER_LONG) * sizeof(long); memset(addr, 0xff, len); tail = size & (BITS_PER_LONG - 1); if (tail) addr[size / BITS_PER_LONG] = BIT_MASK(tail); } static inline int bitmap_full(unsigned long *addr, int size) { long mask; int tail; int len; int i; len = size / BITS_PER_LONG; for (i = 0; i < len; i++) if (addr[i] != ~0UL) return (0); tail = size & (BITS_PER_LONG - 1); if (tail) { mask = BIT_MASK(tail); if ((addr[i] & mask) != mask) return (0); } return (1); } static inline int bitmap_empty(unsigned long *addr, int size) { long mask; int tail; int len; int i; len = size / BITS_PER_LONG; for (i = 0; i < len; i++) if (addr[i] != 0) return (0); tail = size & (BITS_PER_LONG - 1); if (tail) { mask = BIT_MASK(tail); if ((addr[i] & mask) != 0) return (0); } return (1); } #define NBLONG (NBBY * sizeof(long)) #define set_bit(i, a) \ atomic_set_long(&((volatile long *)(a))[(i)/NBLONG], 1UL << ((i) % NBLONG)) #define clear_bit(i, a) \ atomic_clear_long(&((volatile long *)(a))[(i)/NBLONG], 1UL << ((i) % NBLONG)) #define test_bit(i, a) \ !!(atomic_load_acq_long(&((volatile long *)(a))[(i)/NBLONG]) & \ (1UL << ((i) % NBLONG))) static inline long test_and_clear_bit(long bit, long *var) { long val; var += bit / (sizeof(long) * NBBY); bit %= sizeof(long) * NBBY; bit = (1UL << bit); do { val = *(volatile long *)var; } while (atomic_cmpset_long(var, val, val & ~bit) == 0); return !!(val & bit); } static inline long test_and_set_bit(long bit, long *var) { long val; var += bit / (sizeof(long) * NBBY); bit %= sizeof(long) * NBBY; bit = (1UL << bit); do { val = *(volatile long *)var; } while (atomic_cmpset_long(var, val, val | bit) == 0); return !!(val & bit); } + + +#define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) % BITS_PER_LONG)) +#define BITMAP_LAST_WORD_MASK(nbits) \ +( \ + ((nbits) % BITS_PER_LONG) ? \ + (1UL<<((nbits) % BITS_PER_LONG))-1 : ~0UL \ +) + + +static inline void +bitmap_set(unsigned long *map, int start, int nr) +{ + unsigned long *p = map + BIT_WORD(start); + const int size = start + nr; + int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG); + unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start); + + while (nr - bits_to_set >= 0) { + *p |= mask_to_set; + nr -= bits_to_set; + bits_to_set = BITS_PER_LONG; + mask_to_set = ~0UL; + p++; + } + if (nr) { + mask_to_set &= BITMAP_LAST_WORD_MASK(size); + *p |= mask_to_set; + } +} + +static inline void +bitmap_clear(unsigned long *map, int start, int nr) +{ + unsigned long *p = map + BIT_WORD(start); + const int size = start + nr; + int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG); + unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start); + + while (nr - bits_to_clear >= 0) { + *p &= ~mask_to_clear; + nr -= bits_to_clear; + bits_to_clear = BITS_PER_LONG; + mask_to_clear = ~0UL; + p++; + } + if (nr) { + mask_to_clear &= BITMAP_LAST_WORD_MASK(size); + *p &= ~mask_to_clear; + } +} + +enum { + REG_OP_ISFREE, /* true if region is all zero bits */ + REG_OP_ALLOC, /* set all bits in region */ + REG_OP_RELEASE, /* clear all bits in region */ +}; + +static int __reg_op(unsigned long *bitmap, int pos, int order, int reg_op) +{ + int nbits_reg; /* number of bits in region */ + int index; /* index first long of region in bitmap */ + int offset; /* bit offset region in bitmap[index] */ + int nlongs_reg; /* num longs spanned by region in bitmap */ + int nbitsinlong; /* num bits of region in each spanned long */ + unsigned long mask; /* bitmask for one long of region */ + int i; /* scans bitmap by longs */ + int ret = 0; /* return value */ + + /* + * Either nlongs_reg == 1 (for small orders that fit in one long) + * or (offset == 0 && mask == ~0UL) (for larger multiword orders.) + */ + nbits_reg = 1 << order; + index = pos / BITS_PER_LONG; + offset = pos - (index * BITS_PER_LONG); + nlongs_reg = BITS_TO_LONGS(nbits_reg); + nbitsinlong = min(nbits_reg, BITS_PER_LONG); + + /* + * Can't do "mask = (1UL << nbitsinlong) - 1", as that + * overflows if nbitsinlong == BITS_PER_LONG. + */ + mask = (1UL << (nbitsinlong - 1)); + mask += mask - 1; + mask <<= offset; + + switch (reg_op) { + case REG_OP_ISFREE: + for (i = 0; i < nlongs_reg; i++) { + if (bitmap[index + i] & mask) + goto done; + } + ret = 1; /* all bits in region free (zero) */ + break; + + case REG_OP_ALLOC: + for (i = 0; i < nlongs_reg; i++) + bitmap[index + i] |= mask; + break; + + case REG_OP_RELEASE: + for (i = 0; i < nlongs_reg; i++) + bitmap[index + i] &= ~mask; + break; + } +done: + return ret; +} + +/** + * bitmap_find_free_region - find a contiguous aligned mem region + * @bitmap: array of unsigned longs corresponding to the bitmap + * @bits: number of bits in the bitmap + * @order: region size (log base 2 of number of bits) to find + * + * Find a region of free (zero) bits in a @bitmap of @bits bits and + * allocate them (set them to one). Only consider regions of length + * a power (@order) of two, aligned to that power of two, which + * makes the search algorithm much faster. + * + * Return the bit offset in bitmap of the allocated region, + * or -errno on failure. + */ +static inline int +bitmap_find_free_region(unsigned long *bitmap, int bits, int order) +{ + int pos, end; /* scans bitmap by regions of size order */ + + for (pos = 0 ; (end = pos + (1 << order)) <= bits; pos = end) { + if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) + continue; + __reg_op(bitmap, pos, order, REG_OP_ALLOC); + return pos; + } + return -ENOMEM; +} + +/** + * bitmap_allocate_region - allocate bitmap region + * @bitmap: array of unsigned longs corresponding to the bitmap + * @pos: beginning of bit region to allocate + * @order: region size (log base 2 of number of bits) to allocate + * + * Allocate (set bits in) a specified region of a bitmap. + * + * Return 0 on success, or %-EBUSY if specified region wasn't + * free (not all bits were zero). + */ + +static inline int +bitmap_allocate_region(unsigned long *bitmap, int pos, int order) +{ + if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) + return -EBUSY; + __reg_op(bitmap, pos, order, REG_OP_ALLOC); + return 0; +} + +/** + * bitmap_release_region - release allocated bitmap region + * @bitmap: array of unsigned longs corresponding to the bitmap + * @pos: beginning of bit region to release + * @order: region size (log base 2 of number of bits) to release + * + * This is the complement to __bitmap_find_free_region() and releases + * the found region (by clearing it in the bitmap). + * + * No return value. + */ +static inline void +bitmap_release_region(unsigned long *bitmap, int pos, int order) +{ + __reg_op(bitmap, pos, order, REG_OP_RELEASE); +} + + +#define for_each_set_bit(bit, addr, size) \ + for ((bit) = find_first_bit((addr), (size)); \ + (bit) < (size); \ + (bit) = find_next_bit((addr), (size), (bit) + 1)) #endif /* _LINUX_BITOPS_H_ */ Index: stable/9/sys/ofed/include/linux/cache.h =================================================================== --- stable/9/sys/ofed/include/linux/cache.h (nonexistent) +++ stable/9/sys/ofed/include/linux/cache.h (revision 273864) @@ -0,0 +1,37 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LINUX_CACHE_H_ +#define _LINUX_CACHE_H_ + + +#define cache_line_size() CACHE_LINE_SIZE + + +#endif /* _LINUX_CACHE_H_ */ Property changes on: stable/9/sys/ofed/include/linux/cache.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/include/linux/cdev.h =================================================================== --- stable/9/sys/ofed/include/linux/cdev.h (revision 273863) +++ stable/9/sys/ofed/include/linux/cdev.h (revision 273864) @@ -1,129 +1,130 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_CDEV_H_ #define _LINUX_CDEV_H_ #include #include #include struct file_operations; struct inode; struct module; extern struct cdevsw linuxcdevsw; struct linux_cdev { struct kobject kobj; struct module *owner; struct cdev *cdev; dev_t dev; const struct file_operations *ops; }; static inline void cdev_release(struct kobject *kobj) { struct linux_cdev *cdev; cdev = container_of(kobj, struct linux_cdev, kobj); if (cdev->cdev) destroy_dev(cdev->cdev); kfree(cdev); } static inline void cdev_static_release(struct kobject *kobj) { struct linux_cdev *cdev; cdev = container_of(kobj, struct linux_cdev, kobj); if (cdev->cdev) destroy_dev(cdev->cdev); } static struct kobj_type cdev_ktype = { .release = cdev_release, }; static struct kobj_type cdev_static_ktype = { .release = cdev_static_release, }; static inline void cdev_init(struct linux_cdev *cdev, const struct file_operations *ops) { kobject_init(&cdev->kobj, &cdev_static_ktype); cdev->ops = ops; } static inline struct linux_cdev * cdev_alloc(void) { struct linux_cdev *cdev; cdev = kzalloc(sizeof(struct linux_cdev), M_WAITOK); if (cdev) kobject_init(&cdev->kobj, &cdev_ktype); return (cdev); } static inline void cdev_put(struct linux_cdev *p) { kobject_put(&p->kobj); } static inline int cdev_add(struct linux_cdev *cdev, dev_t dev, unsigned count) { if (count != 1) panic("cdev_add: Unsupported count: %d", count); cdev->cdev = make_dev(&linuxcdevsw, MINOR(dev), 0, 0, 0700, "%s", kobject_name(&cdev->kobj)); cdev->dev = dev; cdev->cdev->si_drv1 = cdev; return (0); } static inline void cdev_del(struct linux_cdev *cdev) { if (cdev->cdev) { destroy_dev(cdev->cdev); cdev->cdev = NULL; } kobject_put(&cdev->kobj); } #define cdev linux_cdev #endif /* _LINUX_CDEV_H_ */ Index: stable/9/sys/ofed/include/linux/clocksource.h =================================================================== --- stable/9/sys/ofed/include/linux/clocksource.h (nonexistent) +++ stable/9/sys/ofed/include/linux/clocksource.h (revision 273864) @@ -0,0 +1,37 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LINUX_CLOCKSOURCE_H +#define _LINUX_CLOCKSOURCE_H + +/* clocksource cycle base type */ +typedef u64 cycle_t; + + +#endif /* _LINUX_CLOCKSOURCE_H */ Property changes on: stable/9/sys/ofed/include/linux/clocksource.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/include/linux/compat.h =================================================================== --- stable/9/sys/ofed/include/linux/compat.h (revision 273863) +++ stable/9/sys/ofed/include/linux/compat.h (revision 273864) @@ -1,33 +1,37 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_COMPAT_H_ #define _LINUX_COMPAT_H_ + +#define is_multicast_ether_addr(x) 0 +#define is_broadcast_ether_addr(x) 0 #endif /* _LINUX_COMPAT_H_ */ Index: stable/9/sys/ofed/include/linux/compiler.h =================================================================== --- stable/9/sys/ofed/include/linux/compiler.h (revision 273863) +++ stable/9/sys/ofed/include/linux/compiler.h (revision 273864) @@ -1,65 +1,66 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_COMPILER_H_ #define _LINUX_COMPILER_H_ #include #define __user #define __kernel #define __safe #define __force #define __nocast #define __iomem #define __chk_user_ptr(x) 0 #define __chk_io_ptr(x) 0 #define __builtin_warning(x, y...) (1) #define __acquires(x) #define __releases(x) #define __acquire(x) 0 #define __release(x) 0 #define __cond_lock(x,c) (c) #define __bitwise #define __devinitdata #define __init #define __devinit #define __devexit #define __exit #define __stringify(x) #x #define __attribute_const__ __attribute__((__const__)) #undef __always_inline #define __always_inline inline #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) #define typeof(x) __typeof(x) #define uninitialized_var(x) x = x #endif /* _LINUX_COMPILER_H_ */ Index: stable/9/sys/ofed/include/linux/completion.h =================================================================== --- stable/9/sys/ofed/include/linux/completion.h (revision 273863) +++ stable/9/sys/ofed/include/linux/completion.h (revision 273864) @@ -1,155 +1,155 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _LINUX_COMPLETION_H_ -#define _LINUX_COMPLETION_H_ +#ifndef _FBSD_COMPLETION_H_ +#define _FBSD_COMPLETION_H_ + #include -#include -#include #include #include #include #include #include struct completion { unsigned int done; }; #define INIT_COMPLETION(c) ((c).done = 0) #define init_completion(c) ((c)->done = 0) static inline void _complete_common(struct completion *c, int all) { int wakeup_swapper; sleepq_lock(c); c->done++; if (all) wakeup_swapper = sleepq_broadcast(c, SLEEPQ_SLEEP, 0, 0); else wakeup_swapper = sleepq_signal(c, SLEEPQ_SLEEP, 0, 0); sleepq_release(c); if (wakeup_swapper) kick_proc0(); } #define complete(c) _complete_common(c, 0) #define complete_all(c) _complete_common(c, 1) /* * Indefinite wait for done != 0 with or without signals. */ static inline long _wait_for_common(struct completion *c, int flags) { flags |= SLEEPQ_SLEEP; for (;;) { sleepq_lock(c); if (c->done) break; sleepq_add(c, NULL, "completion", flags, 0); if (flags & SLEEPQ_INTERRUPTIBLE) { if (sleepq_wait_sig(c, 0) != 0) return (-ERESTARTSYS); } else sleepq_wait(c, 0); } c->done--; sleepq_release(c); return (0); } #define wait_for_completion(c) _wait_for_common(c, 0) #define wait_for_completion_interuptible(c) \ _wait_for_common(c, SLEEPQ_INTERRUPTIBLE) static inline long _wait_for_timeout_common(struct completion *c, long timeout, int flags) { long end; end = ticks + timeout; flags |= SLEEPQ_SLEEP; for (;;) { sleepq_lock(c); if (c->done) break; sleepq_add(c, NULL, "completion", flags, 0); sleepq_set_timeout(c, end - ticks); if (flags & SLEEPQ_INTERRUPTIBLE) { if (sleepq_timedwait_sig(c, 0) != 0) return (-ERESTARTSYS); } else sleepq_timedwait(c, 0); } c->done--; sleepq_release(c); timeout = end - ticks; return (timeout > 0 ? timeout : 1); } #define wait_for_completion_timeout(c, timeout) \ _wait_for_timeout_common(c, timeout, 0) #define wait_for_completion_interruptible_timeout(c, timeout) \ _wait_for_timeout_common(c, timeout, SLEEPQ_INTERRUPTIBLE) static inline int try_wait_for_completion(struct completion *c) { int isdone; isdone = 1; sleepq_lock(c); if (c->done) c->done--; else isdone = 0; sleepq_release(c); return (isdone); } static inline int completion_done(struct completion *c) { int isdone; isdone = 1; sleepq_lock(c); if (c->done == 0) isdone = 0; sleepq_release(c); return (isdone); } #endif /* _LINUX_COMPLETION_H_ */ Index: stable/9/sys/ofed/include/linux/delay.h =================================================================== --- stable/9/sys/ofed/include/linux/delay.h (revision 273863) +++ stable/9/sys/ofed/include/linux/delay.h (revision 273864) @@ -1,43 +1,44 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_DELAY_H_ #define _LINUX_DELAY_H_ #include static inline void linux_msleep(int ms) { pause("lnxsleep", msecs_to_jiffies(ms)); } #undef msleep #define msleep linux_msleep #endif /* _LINUX_DELAY_H_ */ Index: stable/9/sys/ofed/include/linux/device.h =================================================================== --- stable/9/sys/ofed/include/linux/device.h (revision 273863) +++ stable/9/sys/ofed/include/linux/device.h (revision 273864) @@ -1,388 +1,447 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_DEVICE_H_ #define _LINUX_DEVICE_H_ #include #include #include #include #include #include #include #include #include #include #include enum irqreturn { IRQ_NONE = 0, IRQ_HANDLED, IRQ_WAKE_THREAD, }; typedef enum irqreturn irqreturn_t; struct class { const char *name; struct module *owner; struct kobject kobj; devclass_t bsdclass; void (*class_release)(struct class *class); void (*dev_release)(struct device *dev); + char * (*devnode)(struct device *dev, umode_t *mode); }; struct device { struct device *parent; struct list_head irqents; device_t bsddev; dev_t devt; struct class *class; void (*release)(struct device *dev); struct kobject kobj; uint64_t *dma_mask; void *driver_data; unsigned int irq; unsigned int msix; unsigned int msix_max; }; extern struct device linux_rootdev; extern struct kobject class_root; struct class_attribute { - struct attribute attr; - ssize_t (*show)(struct class *, char *); - ssize_t (*store)(struct class *, const char *, size_t); + struct attribute attr; + ssize_t (*show)(struct class *, struct class_attribute *, char *); + ssize_t (*store)(struct class *, struct class_attribute *, const char *, size_t); + const void *(*namespace)(struct class *, const struct class_attribute *); }; + #define CLASS_ATTR(_name, _mode, _show, _store) \ struct class_attribute class_attr_##_name = \ { { #_name, NULL, _mode }, _show, _store } struct device_attribute { struct attribute attr; ssize_t (*show)(struct device *, - struct device_attribute *, char *); + struct device_attribute *, char *); ssize_t (*store)(struct device *, - struct device_attribute *, const char *, - size_t); + struct device_attribute *, const char *, + size_t); }; #define DEVICE_ATTR(_name, _mode, _show, _store) \ struct device_attribute dev_attr_##_name = \ { { #_name, NULL, _mode }, _show, _store } +/* Simple class attribute that is just a static string */ +struct class_attribute_string { + struct class_attribute attr; + char *str; +}; + +static inline ssize_t +show_class_attr_string(struct class *class, + struct class_attribute *attr, char *buf) +{ + struct class_attribute_string *cs; + cs = container_of(attr, struct class_attribute_string, attr); + return snprintf(buf, PAGE_SIZE, "%s\n", cs->str); +} + +/* Currently read-only only */ +#define _CLASS_ATTR_STRING(_name, _mode, _str) \ + { __ATTR(_name, _mode, show_class_attr_string, NULL), _str } +#define CLASS_ATTR_STRING(_name, _mode, _str) \ + struct class_attribute_string class_attr_##_name = \ + _CLASS_ATTR_STRING(_name, _mode, _str) + #define dev_err(dev, fmt, ...) device_printf((dev)->bsddev, fmt, ##__VA_ARGS__) #define dev_warn(dev, fmt, ...) device_printf((dev)->bsddev, fmt, ##__VA_ARGS__) #define dev_info(dev, fmt, ...) device_printf((dev)->bsddev, fmt, ##__VA_ARGS__) #define dev_printk(lvl, dev, fmt, ...) \ device_printf((dev)->bsddev, fmt, ##__VA_ARGS__) static inline void * dev_get_drvdata(struct device *dev) { return dev->driver_data; } static inline void dev_set_drvdata(struct device *dev, void *data) { dev->driver_data = data; } static inline struct device * get_device(struct device *dev) { if (dev) kobject_get(&dev->kobj); return (dev); } static inline char * dev_name(const struct device *dev) { return kobject_name(&dev->kobj); } #define dev_set_name(_dev, _fmt, ...) \ kobject_set_name(&(_dev)->kobj, (_fmt), ##__VA_ARGS__) static inline void put_device(struct device *dev) { if (dev) kobject_put(&dev->kobj); } static inline ssize_t class_show(struct kobject *kobj, struct attribute *attr, char *buf) { struct class_attribute *dattr; ssize_t error; dattr = container_of(attr, struct class_attribute, attr); error = -EIO; if (dattr->show) error = dattr->show(container_of(kobj, struct class, kobj), - buf); + dattr, buf); return (error); } static inline ssize_t class_store(struct kobject *kobj, struct attribute *attr, const char *buf, size_t count) { struct class_attribute *dattr; ssize_t error; dattr = container_of(attr, struct class_attribute, attr); error = -EIO; if (dattr->store) error = dattr->store(container_of(kobj, struct class, kobj), - buf, count); + dattr, buf, count); return (error); } static inline void class_release(struct kobject *kobj) { struct class *class; class = container_of(kobj, struct class, kobj); if (class->class_release) class->class_release(class); } static struct sysfs_ops class_sysfs = { .show = class_show, .store = class_store, }; static struct kobj_type class_ktype = { .release = class_release, .sysfs_ops = &class_sysfs }; static inline int class_register(struct class *class) { class->bsdclass = devclass_create(class->name); kobject_init(&class->kobj, &class_ktype); kobject_set_name(&class->kobj, class->name); kobject_add(&class->kobj, &class_root, class->name); return (0); } static inline void class_unregister(struct class *class) { kobject_put(&class->kobj); } static inline void device_release(struct kobject *kobj) { struct device *dev; dev = container_of(kobj, struct device, kobj); /* This is the precedence defined by linux. */ if (dev->release) dev->release(dev); else if (dev->class && dev->class->dev_release) dev->class->dev_release(dev); } static inline ssize_t dev_show(struct kobject *kobj, struct attribute *attr, char *buf) { struct device_attribute *dattr; ssize_t error; dattr = container_of(attr, struct device_attribute, attr); error = -EIO; if (dattr->show) error = dattr->show(container_of(kobj, struct device, kobj), dattr, buf); return (error); } static inline ssize_t dev_store(struct kobject *kobj, struct attribute *attr, const char *buf, size_t count) { struct device_attribute *dattr; ssize_t error; dattr = container_of(attr, struct device_attribute, attr); error = -EIO; if (dattr->store) error = dattr->store(container_of(kobj, struct device, kobj), dattr, buf, count); return (error); } static struct sysfs_ops dev_sysfs = { .show = dev_show, .store = dev_store, }; static struct kobj_type dev_ktype = { .release = device_release, .sysfs_ops = &dev_sysfs }; /* * Devices are registered and created for exporting to sysfs. create * implies register and register assumes the device fields have been * setup appropriately before being called. */ static inline int device_register(struct device *dev) { device_t bsddev; int unit; bsddev = NULL; if (dev->devt) { unit = MINOR(dev->devt); bsddev = devclass_get_device(dev->class->bsdclass, unit); } else unit = -1; if (bsddev == NULL) bsddev = device_add_child(dev->parent->bsddev, dev->class->kobj.name, unit); if (bsddev) { if (dev->devt == 0) dev->devt = makedev(0, device_get_unit(bsddev)); device_set_softc(bsddev, dev); } dev->bsddev = bsddev; kobject_init(&dev->kobj, &dev_ktype); kobject_add(&dev->kobj, &dev->class->kobj, dev_name(dev)); return (0); } static inline void device_unregister(struct device *dev) { device_t bsddev; bsddev = dev->bsddev; mtx_lock(&Giant); if (bsddev) device_delete_child(device_get_parent(bsddev), bsddev); mtx_unlock(&Giant); put_device(dev); } struct device *device_create(struct class *class, struct device *parent, dev_t devt, void *drvdata, const char *fmt, ...); static inline void device_destroy(struct class *class, dev_t devt) { device_t bsddev; int unit; unit = MINOR(devt); bsddev = devclass_get_device(class->bsdclass, unit); if (bsddev) device_unregister(device_get_softc(bsddev)); } static inline void class_kfree(struct class *class) { kfree(class); } static inline struct class * class_create(struct module *owner, const char *name) { struct class *class; int error; class = kzalloc(sizeof(*class), M_WAITOK); class->owner = owner; class->name= name; class->class_release = class_kfree; error = class_register(class); if (error) { kfree(class); return (NULL); } return (class); } static inline void class_destroy(struct class *class) { if (class == NULL) return; class_unregister(class); } static inline int device_create_file(struct device *dev, const struct device_attribute *attr) { if (dev) return sysfs_create_file(&dev->kobj, &attr->attr); return -EINVAL; } static inline void device_remove_file(struct device *dev, const struct device_attribute *attr) { if (dev) sysfs_remove_file(&dev->kobj, &attr->attr); } static inline int class_create_file(struct class *class, const struct class_attribute *attr) { if (class) return sysfs_create_file(&class->kobj, &attr->attr); return -EINVAL; } static inline void class_remove_file(struct class *class, const struct class_attribute *attr) { if (class) sysfs_remove_file(&class->kobj, &attr->attr); } + +static inline int dev_to_node(struct device *dev) +{ + return -1; +} + +static inline char *kvasprintf(gfp_t gfp, const char *fmt, va_list ap) +{ + unsigned int len; + char *p = NULL; + va_list aq; + + va_copy(aq, ap); + len = vsnprintf(NULL, 0, fmt, aq); + va_end(aq); + + vsnprintf(p, len+1, fmt, ap); + + return p; +} + +static inline char *kasprintf(gfp_t gfp, const char *fmt, ...) +{ + va_list ap; + char *p; + + va_start(ap, fmt); + p = kvasprintf(gfp, fmt, ap); + va_end(ap); + + return p; +} + #endif /* _LINUX_DEVICE_H_ */ Index: stable/9/sys/ofed/include/linux/dma-attrs.h =================================================================== --- stable/9/sys/ofed/include/linux/dma-attrs.h (revision 273863) +++ stable/9/sys/ofed/include/linux/dma-attrs.h (revision 273864) @@ -1,47 +1,48 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_DMA_ATTR_H_ #define _LINUX_DMA_ATTR_H_ enum dma_attr { DMA_ATTR_WRITE_BARRIER, DMA_ATTR_WEAK_ORDERING, DMA_ATTR_MAX, }; #define __DMA_ATTRS_LONGS BITS_TO_LONGS(DMA_ATTR_MAX) struct dma_attrs { unsigned long flags; }; #define DEFINE_DMA_ATTRS(x) struct dma_attrs x = { } static inline void init_dma_attrs(struct dma_attrs *attrs) { attrs->flags = 0; } #endif /* _LINUX_DMA_ATTR_H_ */ Index: stable/9/sys/ofed/include/linux/dma-mapping.h =================================================================== --- stable/9/sys/ofed/include/linux/dma-mapping.h (revision 273863) +++ stable/9/sys/ofed/include/linux/dma-mapping.h (revision 273864) @@ -1,263 +1,271 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_DMA_MAPPING_H_ #define _LINUX_DMA_MAPPING_H_ #include #include #include #include #include #include #include #include #include #include #include #include #include #include enum dma_data_direction { DMA_BIDIRECTIONAL = 0, DMA_TO_DEVICE = 1, DMA_FROM_DEVICE = 2, DMA_NONE = 3, }; struct dma_map_ops { void* (*alloc_coherent)(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp); void (*free_coherent)(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle); dma_addr_t (*map_page)(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs); void (*unmap_page)(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs); int (*map_sg)(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, struct dma_attrs *attrs); void (*unmap_sg)(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, struct dma_attrs *attrs); void (*sync_single_for_cpu)(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir); void (*sync_single_for_device)(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir); void (*sync_single_range_for_cpu)(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, enum dma_data_direction dir); void (*sync_single_range_for_device)(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, enum dma_data_direction dir); void (*sync_sg_for_cpu)(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir); void (*sync_sg_for_device)(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir); int (*mapping_error)(struct device *dev, dma_addr_t dma_addr); int (*dma_supported)(struct device *dev, u64 mask); int is_phys; }; #define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL << (n)) - 1)) static inline int dma_supported(struct device *dev, u64 mask) { /* XXX busdma takes care of this elsewhere. */ return (1); } static inline int dma_set_mask(struct device *dev, u64 dma_mask) { if (!dev->dma_mask || !dma_supported(dev, dma_mask)) return -EIO; *dev->dma_mask = dma_mask; return (0); } static inline int dma_set_coherent_mask(struct device *dev, u64 mask) { if (!dma_supported(dev, mask)) return -EIO; /* XXX Currently we don't support a seperate coherent mask. */ return 0; } static inline void * dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag) { vm_paddr_t high; size_t align; void *mem; if (dev->dma_mask) high = *dev->dma_mask; else high = BUS_SPACE_MAXADDR_32BIT; align = PAGE_SIZE << get_order(size); mem = (void *)kmem_alloc_contig(kmem_map, size, flag, 0, high, align, 0, VM_MEMATTR_DEFAULT); if (mem) *dma_handle = vtophys(mem); else *dma_handle = 0; return (mem); } static inline void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_handle) { kmem_free(kmem_map, (vm_offset_t)cpu_addr, size); } /* XXX This only works with no iommu. */ static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { return vtophys(ptr); } static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { } static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction dir, struct dma_attrs *attrs) { struct scatterlist *sg; int i; for_each_sg(sgl, sg, nents, i) sg_dma_address(sg) = sg_phys(sg); return (nents); } static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, struct dma_attrs *attrs) { } static inline dma_addr_t dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction) { return VM_PAGE_TO_PHYS(page) + offset; } static inline void dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, enum dma_data_direction direction) { } static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction) { } static inline void dma_sync_single(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { dma_sync_single_for_cpu(dev, addr, size, dir); } static inline void dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction) { } static inline void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction direction) { } static inline void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction direction) { } static inline void dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, int direction) { } static inline void dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, int direction) { } static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) { return (0); } + +static inline unsigned int dma_set_max_seg_size(struct device *dev, + unsigned int size) +{ + return (0); +} + #define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, NULL) #define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, NULL) #define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, NULL) #define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, NULL) #define DEFINE_DMA_UNMAP_ADDR(name) dma_addr_t name #define DEFINE_DMA_UNMAP_LEN(name) __u32 name #define dma_unmap_addr(p, name) ((p)->name) #define dma_unmap_addr_set(p, name, v) (((p)->name) = (v)) #define dma_unmap_len(p, name) ((p)->name) #define dma_unmap_len_set(p, name, v) (((p)->name) = (v)) extern int uma_align_cache; #define dma_get_cache_alignment() uma_align_cache #endif /* _LINUX_DMA_MAPPING_H_ */ Index: stable/9/sys/ofed/include/linux/dmapool.h =================================================================== --- stable/9/sys/ofed/include/linux/dmapool.h (revision 273863) +++ stable/9/sys/ofed/include/linux/dmapool.h (revision 273864) @@ -1,85 +1,86 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_DMAPOOL_H_ #define _LINUX_DMAPOOL_H_ #include #include #include #include #include struct dma_pool { uma_zone_t pool_zone; }; static inline struct dma_pool * dma_pool_create(char *name, struct device *dev, size_t size, size_t align, size_t boundary) { struct dma_pool *pool; pool = kmalloc(sizeof(*pool), GFP_KERNEL); align--; /* * XXX Eventually this could use a seperate allocf to honor boundary * and physical address requirements of the device. */ pool->pool_zone = uma_zcreate(name, size, NULL, NULL, NULL, NULL, align, UMA_ZONE_OFFPAGE|UMA_ZONE_HASH); return (pool); } static inline void dma_pool_destroy(struct dma_pool *pool) { uma_zdestroy(pool->pool_zone); kfree(pool); } static inline void * dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, dma_addr_t *handle) { void *vaddr; vaddr = uma_zalloc(pool->pool_zone, mem_flags); if (vaddr) *handle = vtophys(vaddr); return (vaddr); } static inline void dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t addr) { uma_zfree(pool->pool_zone, vaddr); } #endif /* _LINUX_DMAPOOL_H_ */ Index: stable/9/sys/ofed/include/linux/err.h =================================================================== --- stable/9/sys/ofed/include/linux/err.h (revision 273863) +++ stable/9/sys/ofed/include/linux/err.h (revision 273864) @@ -1,60 +1,72 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_ERR_H_ #define _LINUX_ERR_H_ #define MAX_ERRNO 4095 #define IS_ERR_VALUE(x) ((x) >= (unsigned long)-MAX_ERRNO) static inline void * ERR_PTR(long error) { return (void *)error; } static inline long PTR_ERR(const void *ptr) { return (long)ptr; } static inline long IS_ERR(const void *ptr) { return IS_ERR_VALUE((unsigned long)ptr); } static inline void * ERR_CAST(void *ptr) { return (void *)ptr; } + +static inline int +PTR_ERR_OR_ZERO(const void *ptr) +{ + if (IS_ERR(ptr)) + return PTR_ERR(ptr); + else + return 0; +} + +#define PTR_RET(p) PTR_ERR_OR_ZERO(p) #endif /* _LINUX_ERR_H_ */ Index: stable/9/sys/ofed/include/linux/errno.h =================================================================== --- stable/9/sys/ofed/include/linux/errno.h (revision 273863) +++ stable/9/sys/ofed/include/linux/errno.h (revision 273864) @@ -1,39 +1,42 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_ERRNO_H_ #define _LINUX_ERRNO_H_ #include -#define ECOMM ESTALE -#define ENODATA ECONNREFUSED -#define ENOIOCTLCMD ENOIOCTL /* XXX this is negative */ -#define ERESTARTSYS ERESTART /* XXX this is negative */ +#define ECOMM ESTALE +#define ENODATA ECONNREFUSED +#define ENOIOCTLCMD ENOIOCTL +#define ERESTARTSYS ERESTART +#define ENOTSUPP EOPNOTSUPP +#define ENONET EHOSTDOWN -#endif /* _LINUX_ERRNO_H_ */ +#endif /* _LINUX_ERRNO_H_ */ Index: stable/9/sys/ofed/include/linux/etherdevice.h =================================================================== --- stable/9/sys/ofed/include/linux/etherdevice.h (nonexistent) +++ stable/9/sys/ofed/include/linux/etherdevice.h (revision 273864) @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2014 Mellanox Technologies, Ltd. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef _LINUX_ETHERDEVICE +#define _LINUX_ETHERDEVICE + +#include + +/** + * is_zero_ether_addr - Determine if give Ethernet address is all zeros. + * @addr: Pointer to a six-byte array containing the Ethernet address + * + * Return true if the address is all zeroes. + */ +static inline bool is_zero_ether_addr(const u8 *addr) +{ + return !(addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]); +} + + + +/** + * is_multicast_ether_addr - Determine if the Ethernet address is a multicast. + * @addr: Pointer to a six-byte array containing the Ethernet address + * + * Return true if the address is a multicast address. + * By definition the broadcast address is also a multicast address. + */ +static inline bool is_multicast_ether_addr(const u8 *addr) +{ + return (0x01 & addr[0]); +} + +/** + * is_broadcast_ether_addr - Determine if the Ethernet address is broadcast + * @addr: Pointer to a six-byte array containing the Ethernet address + * + * Return true if the address is the broadcast address. + */ +static inline bool is_broadcast_ether_addr(const u8 *addr) +{ + return (addr[0] & addr[1] & addr[2] & addr[3] & addr[4] & addr[5]) == 0xff; +} + +/** + * is_valid_ether_addr - Determine if the given Ethernet address is valid + * @addr: Pointer to a six-byte array containing the Ethernet address + * + * Check that the Ethernet address (MAC) is not 00:00:00:00:00:00, is not + * a multicast address, and is not FF:FF:FF:FF:FF:FF. + * + * Return true if the address is valid. + **/ +static inline bool is_valid_ether_addr(const u8 *addr) +{ + /* FF:FF:FF:FF:FF:FF is a multicast address so we don't need to + ** explicitly check for it here. */ + return !is_multicast_ether_addr(addr) && !is_zero_ether_addr(addr); +} + + + +#endif /* _LINUX_ETHERDEVICE */ Property changes on: stable/9/sys/ofed/include/linux/etherdevice.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/include/linux/ethtool.h =================================================================== --- stable/9/sys/ofed/include/linux/ethtool.h (revision 273863) +++ stable/9/sys/ofed/include/linux/ethtool.h (revision 273864) @@ -1,31 +1,32 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_ETHTOOL_H_ #define _LINUX_ETHTOOL_H_ #endif /* _LINUX_ETHTOOL_H_ */ Index: stable/9/sys/ofed/include/linux/file.h =================================================================== --- stable/9/sys/ofed/include/linux/file.h (revision 273863) +++ stable/9/sys/ofed/include/linux/file.h (revision 273864) @@ -1,120 +1,134 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_FILE_H_ #define _LINUX_FILE_H_ #include #include #include #include #include #include struct linux_file; #undef file extern struct fileops linuxfileops; static inline struct linux_file * linux_fget(unsigned int fd) { struct file *file; file = fget_unlocked(curthread->td_proc->p_fd, fd); return (struct linux_file *)file->f_data; } static inline void fput(struct linux_file *filp) { if (filp->_file == NULL) { kfree(filp); return; } if (refcount_release(&filp->_file->f_count)) { _fdrop(filp->_file, curthread); kfree(filp); } } static inline void put_unused_fd(unsigned int fd) { struct file *file; file = fget_unlocked(curthread->td_proc->p_fd, fd); if (file == NULL) return; fdclose(curthread->td_proc->p_fd, file, fd, curthread); } static inline void fd_install(unsigned int fd, struct linux_file *filp) { struct file *file; file = fget_unlocked(curthread->td_proc->p_fd, fd); filp->_file = file; finit(file, filp->f_mode, DTYPE_DEV, filp, &linuxfileops); } static inline int get_unused_fd(void) { struct file *file; int error; int fd; error = falloc(curthread, &file, &fd, 0); if (error) return -error; return fd; } static inline struct linux_file * -_alloc_file(int mode, const struct file_operations *fops) +alloc_file(int mode, const struct file_operations *fops) { struct linux_file *filp; filp = kzalloc(sizeof(*filp), GFP_KERNEL); - if (filp == NULL) + if (filp == NULL) return (NULL); filp->f_op = fops; filp->f_mode = mode; return filp; } -#define alloc_file(mnt, root, mode, fops) _alloc_file((mode), (fops)) +struct fd { + struct linux_file *linux_file; +}; + +static inline void fdput(struct fd fd) +{ + fput(fd.linux_file); +} + +static inline struct fd fdget(unsigned int fd) +{ + struct linux_file *f = linux_fget(fd); + return (struct fd){f}; +} #define file linux_file #define fget linux_fget #endif /* _LINUX_FILE_H_ */ Index: stable/9/sys/ofed/include/linux/fs.h =================================================================== --- stable/9/sys/ofed/include/linux/fs.h (revision 273863) +++ stable/9/sys/ofed/include/linux/fs.h (revision 273864) @@ -1,183 +1,211 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_FS_H_ #define _LINUX_FS_H_ #include #include #include #include #include #include #include #include struct module; struct kiocb; struct iovec; struct dentry; struct page; struct file_lock; struct pipe_inode_info; struct vm_area_struct; struct poll_table_struct; struct files_struct; #define inode vnode #define i_cdev v_rdev #define S_IRUGO (S_IRUSR | S_IRGRP | S_IROTH) #define S_IWUGO (S_IWUSR | S_IWGRP | S_IWOTH) typedef struct files_struct *fl_owner_t; struct dentry { struct inode *d_inode; }; struct file_operations; struct linux_file { struct file *_file; const struct file_operations *f_op; void *private_data; int f_flags; int f_mode; /* Just starting mode. */ struct dentry *f_dentry; struct dentry f_dentry_store; struct selinfo f_selinfo; struct sigio *f_sigio; struct vnode *f_vnode; }; #define file linux_file #define fasync_struct sigio * #define fasync_helper(fd, filp, on, queue) \ ({ \ if ((on)) \ *(queue) = &(filp)->f_sigio; \ else \ *(queue) = NULL; \ 0; \ }) #define kill_fasync(queue, sig, pollstat) \ do { \ if (*(queue) != NULL) \ pgsigio(*(queue), (sig), 0); \ } while (0) typedef int (*filldir_t)(void *, const char *, int, loff_t, u64, unsigned); struct file_operations { struct module *owner; ssize_t (*read)(struct file *, char __user *, size_t, loff_t *); ssize_t (*write)(struct file *, const char __user *, size_t, loff_t *); unsigned int (*poll) (struct file *, struct poll_table_struct *); long (*unlocked_ioctl)(struct file *, unsigned int, unsigned long); int (*mmap)(struct file *, struct vm_area_struct *); int (*open)(struct inode *, struct file *); int (*release)(struct inode *, struct file *); int (*fasync)(int, struct file *, int); + +/* Although not supported in FreeBSD, to align with Linux code + * we are adding llseek() only when it is mapped to no_llseek which returns + * an illegal seek error + */ + loff_t (*llseek)(struct file *, loff_t, int); #if 0 /* We do not support these methods. Don't permit them to compile. */ loff_t (*llseek)(struct file *, loff_t, int); ssize_t (*aio_read)(struct kiocb *, const struct iovec *, unsigned long, loff_t); ssize_t (*aio_write)(struct kiocb *, const struct iovec *, unsigned long, loff_t); int (*readdir)(struct file *, void *, filldir_t); int (*ioctl)(struct inode *, struct file *, unsigned int, unsigned long); long (*compat_ioctl)(struct file *, unsigned int, unsigned long); int (*flush)(struct file *, fl_owner_t id); int (*fsync)(struct file *, struct dentry *, int datasync); int (*aio_fsync)(struct kiocb *, int datasync); int (*lock)(struct file *, int, struct file_lock *); ssize_t (*sendpage)(struct file *, struct page *, int, size_t, loff_t *, int); unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); int (*check_flags)(int); int (*flock)(struct file *, int, struct file_lock *); ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int); ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); int (*setlease)(struct file *, long, struct file_lock **); #endif }; #define fops_get(fops) (fops) #define FMODE_READ FREAD #define FMODE_WRITE FWRITE #define FMODE_EXEC FEXEC static inline int register_chrdev_region(dev_t dev, unsigned range, const char *name) { return 0; } static inline void unregister_chrdev_region(dev_t dev, unsigned range) { return; } +static inline int +alloc_chrdev_region(dev_t *dev, unsigned baseminor, unsigned count, + const char *name) +{ + + return 0; +} + +/* No current support for seek op in FreeBSD */ +static inline int +nonseekable_open(struct inode *inode, struct file *filp) +{ + return 0; +} + static inline dev_t iminor(struct inode *inode) { return dev2unit(inode->v_rdev); } static inline struct inode * igrab(struct inode *inode) { int error; error = vget(inode, 0, curthread); if (error) return (NULL); return (inode); } static inline void iput(struct inode *inode) { vrele(inode); } -#endif /* _LINUX_FS_H_ */ +static inline loff_t +no_llseek(struct file *file, loff_t offset, int whence) +{ + return -ESPIPE; +} + +#endif /* _LINUX_FS_H_ */ Index: stable/9/sys/ofed/include/linux/gfp.h =================================================================== --- stable/9/sys/ofed/include/linux/gfp.h (revision 273863) +++ stable/9/sys/ofed/include/linux/gfp.h (revision 273864) @@ -1,124 +1,129 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_GFP_H_ #define _LINUX_GFP_H_ #include #include #include #include #include #include #include #define __GFP_NOWARN 0 #define __GFP_HIGHMEM 0 #define __GFP_ZERO M_ZERO #define GFP_NOWAIT M_NOWAIT #define GFP_ATOMIC (M_NOWAIT | M_USE_RESERVE) #define GFP_KERNEL M_WAITOK #define GFP_USER M_WAITOK #define GFP_HIGHUSER M_WAITOK #define GFP_HIGHUSER_MOVABLE M_WAITOK #define GFP_IOFS M_NOWAIT static inline void * page_address(struct page *page) { if (page->object != kmem_object && page->object != kernel_object) return (NULL); return ((void *)(uintptr_t)(VM_MIN_KERNEL_ADDRESS + IDX_TO_OFF(page->pindex))); } static inline unsigned long _get_page(gfp_t mask) { return kmem_malloc(kmem_map, PAGE_SIZE, mask); } #define get_zeroed_page(mask) _get_page((mask) | M_ZERO) #define alloc_page(mask) virt_to_page(_get_page((mask))) #define __get_free_page(mask) _get_page((mask)) static inline void free_page(unsigned long page) { if (page == 0) return; kmem_free(kmem_map, page, PAGE_SIZE); } static inline void __free_page(struct page *m) { if (m->object != kmem_object) panic("__free_page: Freed page %p not allocated via wrappers.", m); kmem_free(kmem_map, (vm_offset_t)page_address(m), PAGE_SIZE); } static inline void __free_pages(void *p, unsigned int order) { size_t size; if (p == 0) return; size = PAGE_SIZE << order; kmem_free(kmem_map, (vm_offset_t)p, size); } /* * Alloc pages allocates directly from the buddy allocator on linux so * order specifies a power of two bucket of pages and the results * are expected to be aligned on the size as well. */ static inline struct page * alloc_pages(gfp_t gfp_mask, unsigned int order) { unsigned long page; size_t size; size = PAGE_SIZE << order; page = kmem_alloc_contig(kmem_map, size, gfp_mask, 0, -1, size, 0, VM_MEMATTR_DEFAULT); if (page == 0) return (NULL); return (virt_to_page(page)); } + +#define alloc_pages_node(node, mask, order) alloc_pages(mask, order) + +#define kmalloc_node(chunk, mask, node) kmalloc(chunk, mask) #endif /* _LINUX_GFP_H_ */ Index: stable/9/sys/ofed/include/linux/hardirq.h =================================================================== --- stable/9/sys/ofed/include/linux/hardirq.h (revision 273863) +++ stable/9/sys/ofed/include/linux/hardirq.h (revision 273864) @@ -1,39 +1,40 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_HARDIRQ_H_ #define _LINUX_HARDIRQ_H_ #include #include #include #include #define synchronize_irq(irq) _intr_drain((irq)) #endif /* _LINUX_HARDIRQ_H_ */ Index: stable/9/sys/ofed/include/linux/idr.h =================================================================== --- stable/9/sys/ofed/include/linux/idr.h (revision 273863) +++ stable/9/sys/ofed/include/linux/idr.h (revision 273864) @@ -1,70 +1,75 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_IDR_H_ #define _LINUX_IDR_H_ #include #define IDR_BITS 5 #define IDR_SIZE (1 << IDR_BITS) #define IDR_MASK (IDR_SIZE - 1) #define MAX_ID_SHIFT ((sizeof(int) * NBBY) - 1) #define MAX_ID_BIT (1U << MAX_ID_SHIFT) #define MAX_ID_MASK (MAX_ID_BIT - 1) #define MAX_LEVEL (MAX_ID_SHIFT + IDR_BITS - 1) / IDR_BITS + +#define MAX_IDR_SHIFT (sizeof(int)*8 - 1) +#define MAX_IDR_BIT (1U << MAX_IDR_SHIFT) +#define MAX_IDR_MASK (MAX_IDR_BIT - 1) struct idr_layer { unsigned long bitmap; struct idr_layer *ary[IDR_SIZE]; }; struct idr { struct mtx lock; struct idr_layer *top; struct idr_layer *free; int layers; }; #define DEFINE_IDR(name) \ struct idr name; \ SYSINIT(name##_idr_sysinit, SI_SUB_DRIVERS, SI_ORDER_FIRST, \ idr_init, &(name)); void *idr_find(struct idr *idp, int id); int idr_pre_get(struct idr *idp, gfp_t gfp_mask); int idr_get_new(struct idr *idp, void *ptr, int *id); int idr_get_new_above(struct idr *idp, void *ptr, int starting_id, int *id); void *idr_replace(struct idr *idp, void *ptr, int id); void idr_remove(struct idr *idp, int id); void idr_remove_all(struct idr *idp); void idr_destroy(struct idr *idp); void idr_init(struct idr *idp); #endif /* _LINUX_IDR_H_ */ Index: stable/9/sys/ofed/include/linux/if_arp.h =================================================================== --- stable/9/sys/ofed/include/linux/if_arp.h (revision 273863) +++ stable/9/sys/ofed/include/linux/if_arp.h (revision 273864) @@ -1,32 +1,33 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_IF_ARP_H_ #define _LINUX_IF_ARP_H_ #include #include #endif /* _LINUX_IF_ARP_H_ */ Index: stable/9/sys/ofed/include/linux/if_ether.h =================================================================== --- stable/9/sys/ofed/include/linux/if_ether.h (revision 273863) +++ stable/9/sys/ofed/include/linux/if_ether.h (revision 273864) @@ -1,37 +1,50 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_IF_ETHER_H_ #define _LINUX_IF_ETHER_H_ #include #include #define ETH_P_8021Q ETHERTYPE_VLAN + +#define ETH_HLEN ETHER_HDR_LEN /* Total octets in header. */ +#ifndef ETH_ALEN +#define ETH_ALEN ETHER_ADDR_LEN +#endif +#define ETH_FCS_LEN 4 /* Octets in the FCS */ +#define VLAN_HLEN 4 /* The additional bytes (on top of the Ethernet header) + * that VLAN requires. */ +/* + * defined Ethernet Protocol ID's. + */ +#define ETH_P_IP 0x0800 /* Internet Protocol packet */ #endif /* _LINUX_IF_ETHER_H_ */ Index: stable/9/sys/ofed/include/linux/if_vlan.h =================================================================== --- stable/9/sys/ofed/include/linux/if_vlan.h (revision 273863) +++ stable/9/sys/ofed/include/linux/if_vlan.h (revision 273864) @@ -1,35 +1,40 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_IF_VLAN_H_ #define _LINUX_IF_VLAN_H_ +#include +#include #include #include + +#define VLAN_N_VID 4096 #endif /* _LINUX_IF_VLAN_H_ */ Index: stable/9/sys/ofed/include/linux/in.h =================================================================== --- stable/9/sys/ofed/include/linux/in.h (revision 273863) +++ stable/9/sys/ofed/include/linux/in.h (revision 273864) @@ -1,39 +1,40 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_IN_H_ #define _LINUX_IN_H_ #include "opt_inet.h" #include #include #define ipv4_is_zeronet IN_ZERONET #define ipv4_is_loopback IN_LOOPBACK #endif /* _LINUX_IN_H_ */ Index: stable/9/sys/ofed/include/linux/in6.h =================================================================== --- stable/9/sys/ofed/include/linux/in6.h (revision 273863) +++ stable/9/sys/ofed/include/linux/in6.h (revision 273864) @@ -1,34 +1,37 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_IN6_H_ #define _LINUX_IN6_H_ +#ifndef KLD_MODULE #include "opt_inet6.h" +#endif #endif /* _LINUX_IN6_H_ */ Index: stable/9/sys/ofed/include/linux/inet.h =================================================================== --- stable/9/sys/ofed/include/linux/inet.h (revision 273863) +++ stable/9/sys/ofed/include/linux/inet.h (revision 273864) @@ -1,31 +1,32 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_INET_H_ #define _LINUX_INET_H_ #endif /* _LINUX_INET_H_ */ Index: stable/9/sys/ofed/include/linux/inetdevice.h =================================================================== --- stable/9/sys/ofed/include/linux/inetdevice.h (revision 273863) +++ stable/9/sys/ofed/include/linux/inetdevice.h (revision 273864) @@ -1,56 +1,57 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_INETDEVICE_H_ #define _LINUX_INETDEVICE_H_ #include static inline struct net_device * ip_dev_find(struct net *net, uint32_t addr) { struct sockaddr_in sin; struct ifaddr *ifa; struct ifnet *ifp; ifp = NULL; memset(&sin, 0, sizeof(sin)); sin.sin_addr.s_addr = addr; sin.sin_port = 0; sin.sin_len = sizeof(sin); sin.sin_family = AF_INET; ifa = ifa_ifwithaddr((struct sockaddr *)&sin); if (ifa) { ifp = ifa->ifa_ifp; if_ref(ifp); ifa_free(ifa); } return (ifp); } #endif /* _LINUX_INETDEVICE_H_ */ Index: stable/9/sys/ofed/include/linux/interrupt.h =================================================================== --- stable/9/sys/ofed/include/linux/interrupt.h (revision 273863) +++ stable/9/sys/ofed/include/linux/interrupt.h (revision 273864) @@ -1,139 +1,140 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_INTERRUPT_H_ #define _LINUX_INTERRUPT_H_ #include #include #include #include typedef irqreturn_t (*irq_handler_t)(int, void *); #define IRQ_RETVAL(x) ((x) != IRQ_NONE) #define IRQF_SHARED RF_SHAREABLE struct irq_ent { struct list_head links; struct device *dev; struct resource *res; void *arg; irqreturn_t (*handler)(int, void *); void *tag; int irq; }; static inline int _irq_rid(struct device *dev, int irq) { if (irq == dev->irq) return (0); return irq - dev->msix + 1; } static void _irq_handler(void *ent) { struct irq_ent *irqe; irqe = ent; irqe->handler(irqe->irq, irqe->arg); } static inline struct irq_ent * _irq_ent(struct device *dev, int irq) { struct irq_ent *irqe; list_for_each_entry(irqe, &dev->irqents, links) if (irqe->irq == irq) return (irqe); return (NULL); } static inline int request_irq(unsigned int irq, irq_handler_t handler, unsigned long flags, const char *name, void *arg) { struct resource *res; struct irq_ent *irqe; struct device *dev; int error; int rid; dev = _pci_find_irq_dev(irq); if (dev == NULL) return -ENXIO; rid = _irq_rid(dev, irq); res = bus_alloc_resource_any(dev->bsddev, SYS_RES_IRQ, &rid, flags | RF_ACTIVE); if (res == NULL) return (-ENXIO); irqe = kmalloc(sizeof(*irqe), GFP_KERNEL); irqe->dev = dev; irqe->res = res; irqe->arg = arg; irqe->handler = handler; irqe->irq = irq; error = bus_setup_intr(dev->bsddev, res, INTR_TYPE_NET | INTR_MPSAFE, NULL, _irq_handler, irqe, &irqe->tag); if (error) { bus_release_resource(dev->bsddev, SYS_RES_IRQ, rid, irqe->res); kfree(irqe); return (-error); } list_add(&irqe->links, &dev->irqents); return 0; } static inline void free_irq(unsigned int irq, void *device) { struct irq_ent *irqe; struct device *dev; int rid; dev = _pci_find_irq_dev(irq); if (dev == NULL) return; rid = _irq_rid(dev, irq); irqe = _irq_ent(dev, irq); if (irqe == NULL) return; bus_teardown_intr(dev->bsddev, irqe->res, irqe->tag); bus_release_resource(dev->bsddev, SYS_RES_IRQ, rid, irqe->res); list_del(&irqe->links); kfree(irqe); } #endif /* _LINUX_INTERRUPT_H_ */ Index: stable/9/sys/ofed/include/linux/io-mapping.h =================================================================== --- stable/9/sys/ofed/include/linux/io-mapping.h (revision 273863) +++ stable/9/sys/ofed/include/linux/io-mapping.h (revision 273864) @@ -1,77 +1,78 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_IO_MAPPING_H_ #define _LINUX_IO_MAPPING_H_ #include #include struct io_mapping; static inline struct io_mapping * io_mapping_create_wc(resource_size_t base, unsigned long size) { return ioremap_wc(base, size); } static inline void io_mapping_free(struct io_mapping *mapping) { iounmap(mapping); } static inline void * io_mapping_map_atomic_wc(struct io_mapping *mapping, unsigned long offset) { return (((char *)mapping) + offset); } static inline void io_mapping_unmap_atomic(void *vaddr) { } static inline void * io_mapping_map_wc(struct io_mapping *mapping, unsigned long offset) { return (((char *) mapping) + offset); } static inline void io_mapping_unmap(void *vaddr) { } #endif /* _LINUX_IO_MAPPING_H_ */ Index: stable/9/sys/ofed/include/linux/io.h =================================================================== --- stable/9/sys/ofed/include/linux/io.h (revision 273863) +++ stable/9/sys/ofed/include/linux/io.h (revision 273864) @@ -1,125 +1,126 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_IO_H_ #define _LINUX_IO_H_ #include static inline uint32_t __raw_readl(const volatile void *addr) { return *(const volatile uint32_t *)addr; } static inline void __raw_writel(uint32_t b, volatile void *addr) { *(volatile uint32_t *)addr = b; } static inline uint64_t __raw_readq(const volatile void *addr) { return *(const volatile uint64_t *)addr; } static inline void __raw_writeq(uint64_t b, volatile void *addr) { *(volatile uint64_t *)addr = b; } /* * XXX This is all x86 specific. It should be bus space access. */ #define mmiowb() #undef writel static inline void writel(uint32_t b, void *addr) { *(volatile uint32_t *)addr = b; } #undef writeq static inline void writeq(uint64_t b, void *addr) { *(volatile uint64_t *)addr = b; } #undef writeb static inline void writeb(uint8_t b, void *addr) { *(volatile uint8_t *)addr = b; } #undef writew static inline void writew(uint16_t b, void *addr) { *(volatile uint16_t *)addr = b; } void *_ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr); #define ioremap_nocache(addr, size) \ _ioremap_attr((addr), (size), VM_MEMATTR_UNCACHEABLE) #define ioremap_wc(addr, size) \ _ioremap_attr((addr), (size), VM_MEMATTR_WRITE_COMBINING) #define ioremap ioremap_nocache void iounmap(void *addr); #define memset_io(a, b, c) memset((a), (b), (c)) #define memcpy_fromio(a, b, c) memcpy((a), (b), (c)) #define memcpy_toio(a, b, c) memcpy((a), (b), (c)) static inline void __iowrite64_copy(void *to, void *from, size_t count) { #ifdef __LP64__ uint64_t *src; uint64_t *dst; int i; for (i = 0, src = from, dst = to; i < count; i++, src++, dst++) __raw_writeq(*src, dst); #else uint32_t *src; uint32_t *dst; int i; count *= 2; for (i = 0, src = from, dst = to; i < count; i++, src++, dst++) __raw_writel(*src, dst); #endif } #endif /* _LINUX_IO_H_ */ Index: stable/9/sys/ofed/include/linux/ioctl.h =================================================================== --- stable/9/sys/ofed/include/linux/ioctl.h (revision 273863) +++ stable/9/sys/ofed/include/linux/ioctl.h (revision 273864) @@ -1,34 +1,35 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_IOCTL_H_ #define _LINUX_IOCTL_H_ #include #endif /* _LINUX_IOCTL_H_ */ Index: stable/9/sys/ofed/include/linux/jiffies.h =================================================================== --- stable/9/sys/ofed/include/linux/jiffies.h (revision 273863) +++ stable/9/sys/ofed/include/linux/jiffies.h (revision 273864) @@ -1,56 +1,60 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_JIFFIES_H_ #define _LINUX_JIFFIES_H_ #include #include #include #include static inline int msecs_to_jiffies(int msec) { struct timeval tv; tv.tv_sec = msec / 1000; tv.tv_usec = (msec % 1000) * 1000; return (tvtohz(&tv)); } -#define jiffies ticks + +#define jiffies ticks +#define jiffies_to_msecs(x) (((int64_t)(x)) * 1000 / hz) + #define time_after(a, b) ((long)(b) - (long)(a) < 0) #define time_before(a, b) time_after(b,a) #define time_after_eq(a, b) ((long)(a) - (long)(b) >= 0) #define time_before_eq(a, b) time_after_eq(b, a) #define HZ hz #endif /* _LINUX_JIFFIES_H_ */ Index: stable/9/sys/ofed/include/linux/kdev_t.h =================================================================== --- stable/9/sys/ofed/include/linux/kdev_t.h (revision 273863) +++ stable/9/sys/ofed/include/linux/kdev_t.h (revision 273864) @@ -1,36 +1,37 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_KDEV_T_H_ #define _LINUX_KDEV_T_H_ #define MAJOR(dev) major((dev)) #define MINOR(dev) minor((dev)) #define MKDEV(ma, mi) makedev((ma), (mi)) #endif /* _LINUX_KDEV_T_H_ */ Index: stable/9/sys/ofed/include/linux/kernel.h =================================================================== --- stable/9/sys/ofed/include/linux/kernel.h (revision 273863) +++ stable/9/sys/ofed/include/linux/kernel.h (revision 273864) @@ -1,88 +1,160 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_KERNEL_H_ #define _LINUX_KERNEL_H_ #include #include #include #include #include +#include #include #include #include -#include #include #include #include #include -#include -#include -#include +#include #include +#define KERN_CONT "" #define KERN_EMERG "<0>" #define KERN_ALERT "<1>" #define KERN_CRIT "<2>" #define KERN_ERR "<3>" #define KERN_WARNING "<4>" #define KERN_NOTICE "<5>" #define KERN_INFO "<6>" #define KERN_DEBUG "<7>" #define BUG() panic("BUG") #define BUG_ON(condition) do { if (condition) BUG(); } while(0) #define WARN_ON BUG_ON #undef ALIGN #define ALIGN(x, y) roundup2((x), (y)) #define DIV_ROUND_UP howmany #define printk(X...) printf(X) #define pr_debug(fmt, ...) printk(KERN_DEBUG # fmt, ##__VA_ARGS__) #define udelay(t) DELAY(t) +#ifndef pr_fmt +#define pr_fmt(fmt) fmt +#endif + +/* + * Print a one-time message (analogous to WARN_ONCE() et al): + */ +#define printk_once(x...) ({ \ + static bool __print_once; \ + \ + if (!__print_once) { \ + __print_once = true; \ + printk(x); \ + } \ +}) + + + +#define pr_emerg(fmt, ...) \ + printk(KERN_EMERG pr_fmt(fmt), ##__VA_ARGS__) +#define pr_alert(fmt, ...) \ + printk(KERN_ALERT pr_fmt(fmt), ##__VA_ARGS__) +#define pr_crit(fmt, ...) \ + printk(KERN_CRIT pr_fmt(fmt), ##__VA_ARGS__) +#define pr_err(fmt, ...) \ + printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__) +#define pr_warning(fmt, ...) \ + printk(KERN_WARNING pr_fmt(fmt), ##__VA_ARGS__) +#define pr_warn pr_warning +#define pr_notice(fmt, ...) \ + printk(KERN_NOTICE pr_fmt(fmt), ##__VA_ARGS__) +#define pr_info(fmt, ...) \ + printk(KERN_INFO pr_fmt(fmt), ##__VA_ARGS__) +#define pr_info_once(fmt, ...) \ + printk_once(KERN_INFO pr_fmt(fmt), ##__VA_ARGS__) +#define pr_cont(fmt, ...) \ + printk(KERN_CONT fmt, ##__VA_ARGS__) + +/* pr_devel() should produce zero code unless DEBUG is defined */ +#ifdef DEBUG +#define pr_devel(fmt, ...) \ + printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__) +#else +#define pr_devel(fmt, ...) \ + ({ if (0) printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); 0; }) +#endif + +#ifndef WARN +#define WARN(condition, format...) ({ \ + int __ret_warn_on = !!(condition); \ + if (unlikely(__ret_warn_on)) \ + pr_warning(format); \ + unlikely(__ret_warn_on); \ +}) +#endif + #define container_of(ptr, type, member) \ ({ \ __typeof(((type *)0)->member) *_p = (ptr); \ (type *)((char *)_p - offsetof(type, member)); \ }) #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) #define simple_strtoul strtoul +#define simple_strtol strtol +#define kstrtol(a,b,c) ({*(c) = strtol(a,0,b);}) #define min(x, y) (x < y ? x : y) #define max(x, y) (x > y ? x : y) #define min_t(type, _x, _y) (type)(_x) < (type)(_y) ? (type)(_x) : (_y) #define max_t(type, _x, _y) (type)(_x) > (type)(_y) ? (type)(_x) : (_y) +/* + * This looks more complex than it should be. But we need to + * get the type for the ~ right in round_down (it needs to be + * as wide as the result!), and we want to evaluate the macro + * arguments just once each. + */ +#define __round_mask(x, y) ((__typeof__(x))((y)-1)) +#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1) +#define round_down(x, y) ((x) & ~__round_mask(x, y)) + #define num_possible_cpus() mp_ncpus + +typedef struct pm_message { + int event; +} pm_message_t; #endif /* _LINUX_KERNEL_H_ */ Index: stable/9/sys/ofed/include/linux/kmod.h =================================================================== --- stable/9/sys/ofed/include/linux/kmod.h (nonexistent) +++ stable/9/sys/ofed/include/linux/kmod.h (revision 273864) @@ -0,0 +1,51 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LINUX_KMOD_H_ +#define _LINUX_KMOD_H_ + +#include +#include +#include +#include +#include +#include + +#define request_module(...) \ +({\ + char modname[128]; \ + int fileid; \ + snprintf(modname, sizeof(modname), __VA_ARGS__); \ + kern_kldload(curthread, modname, &fileid); \ +}) + +#define request_module_nowait request_module + + +#endif /* _LINUX_KMOD_H_ */ Property changes on: stable/9/sys/ofed/include/linux/kmod.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/include/linux/kobject.h =================================================================== --- stable/9/sys/ofed/include/linux/kobject.h (revision 273863) +++ stable/9/sys/ofed/include/linux/kobject.h (revision 273864) @@ -1,153 +1,169 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_KOBJECT_H_ #define _LINUX_KOBJECT_H_ #include #include #include #include struct kobject; struct sysctl_oid; struct kobj_type { void (*release)(struct kobject *kobj); const struct sysfs_ops *sysfs_ops; struct attribute **default_attrs; }; extern struct kobj_type kfree_type; struct kobject { struct kobject *parent; char *name; struct kref kref; struct kobj_type *ktype; struct list_head entry; struct sysctl_oid *oidp; }; +extern struct kobject *mm_kobj; + static inline void kobject_init(struct kobject *kobj, struct kobj_type *ktype) { kref_init(&kobj->kref); INIT_LIST_HEAD(&kobj->entry); kobj->ktype = ktype; kobj->oidp = NULL; } static inline void kobject_put(struct kobject *kobj); void kobject_release(struct kref *kref); static inline void kobject_put(struct kobject *kobj) { if (kobj) kref_put(&kobj->kref, kobject_release); } static inline struct kobject * kobject_get(struct kobject *kobj) { if (kobj) kref_get(&kobj->kref); return kobj; } static inline int kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list args) { char *old; char *name; old = kobj->name; if (old && !fmt) return 0; name = kzalloc(MAXPATHLEN, GFP_KERNEL); if (!name) return -ENOMEM; vsnprintf(name, MAXPATHLEN, fmt, args); kobj->name = name; kfree(old); for (; *name != '\0'; name++) if (*name == '/') *name = '!'; return (0); } int kobject_add(struct kobject *kobj, struct kobject *parent, const char *fmt, ...); static inline struct kobject * kobject_create(void) { struct kobject *kobj; kobj = kzalloc(sizeof(*kobj), GFP_KERNEL); if (kobj == NULL) return (NULL); kobject_init(kobj, &kfree_type); return (kobj); } static inline struct kobject * kobject_create_and_add(const char *name, struct kobject *parent) { struct kobject *kobj; kobj = kobject_create(); if (kobj == NULL) return (NULL); if (kobject_add(kobj, parent, "%s", name) == 0) return (kobj); kobject_put(kobj); return (NULL); } static inline char * kobject_name(const struct kobject *kobj) { return kobj->name; } int kobject_set_name(struct kobject *kobj, const char *fmt, ...); int kobject_init_and_add(struct kobject *kobj, struct kobj_type *ktype, struct kobject *parent, const char *fmt, ...); + +/* sysfs.h calles for 'kobject' which is defined here, + * so we need to add the include only after the 'kobject' def. + */ +#include + +struct kobj_attribute { + struct attribute attr; + ssize_t (*show)(struct kobject *kobj, struct kobj_attribute *attr, + char *buf); + ssize_t (*store)(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count); +}; #endif /* _LINUX_KOBJECT_H_ */ Index: stable/9/sys/ofed/include/linux/kref.h =================================================================== --- stable/9/sys/ofed/include/linux/kref.h (revision 273863) +++ stable/9/sys/ofed/include/linux/kref.h (revision 273864) @@ -1,62 +1,63 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_KREF_H_ #define _LINUX_KREF_H_ #include struct kref { volatile u_int count; }; static inline void kref_init(struct kref *kref) { refcount_init(&kref->count, 1); } static inline void kref_get(struct kref *kref) { refcount_acquire(&kref->count); } static inline int kref_put(struct kref *kref, void (*rel)(struct kref *kref)) { if (refcount_release(&kref->count)) { rel(kref); return 1; } return 0; } -#endif /* _KREF_H_ */ +#endif /* _LINUX_KREF_H_ */ Index: stable/9/sys/ofed/include/linux/kthread.h =================================================================== --- stable/9/sys/ofed/include/linux/kthread.h (revision 273863) +++ stable/9/sys/ofed/include/linux/kthread.h (revision 273864) @@ -1,104 +1,105 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_KTHREAD_H_ #define _LINUX_KTHREAD_H_ #include #include #include #include #include #include #include #include static inline void _kthread_fn(void *arg) { struct task_struct *task; task = arg; task_struct_set(curthread, task); if (task->should_stop == 0) task->task_ret = task->task_fn(task->task_data); PROC_LOCK(task->task_thread->td_proc); task->should_stop = TASK_STOPPED; wakeup(task); PROC_UNLOCK(task->task_thread->td_proc); kthread_exit(); } static inline struct task_struct * _kthread_create(int (*threadfn)(void *data), void *data) { struct task_struct *task; task = kzalloc(sizeof(*task), GFP_KERNEL); task->task_fn = threadfn; task->task_data = data; return (task); } struct task_struct *kthread_create(int (*threadfn)(void *data), void *data, const char namefmt[], ...) __attribute__((format(printf, 3, 4))); #define kthread_run(fn, data, fmt, ...) \ ({ \ struct task_struct *_task; \ \ _task = _kthread_create((fn), (data)); \ if (kthread_add(_kthread_fn, _task, NULL, &_task->task_thread, \ 0, 0, fmt, ## __VA_ARGS__)) { \ kfree(_task); \ _task = NULL; \ } else \ task_struct_set(_task->task_thread, _task); \ _task; \ }) #define kthread_should_stop() current->should_stop static inline int kthread_stop(struct task_struct *task) { PROC_LOCK(task->task_thread->td_proc); task->should_stop = TASK_SHOULD_STOP; wake_up_process(task); while (task->should_stop != TASK_STOPPED) msleep(task, &task->task_thread->td_proc->p_mtx, PWAIT, "kstop", hz); PROC_UNLOCK(task->task_thread->td_proc); return task->task_ret; } #endif /* _LINUX_KTHREAD_H_ */ Index: stable/9/sys/ofed/include/linux/ktime.h =================================================================== --- stable/9/sys/ofed/include/linux/ktime.h (nonexistent) +++ stable/9/sys/ofed/include/linux/ktime.h (revision 273864) @@ -0,0 +1,291 @@ +/*- + * Copyright (c) 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LINUX_KTIME_H +#define _LINUX_KTIME_H + +#include +#include +#include + + +/* Get the monotonic time in timespec format: */ +#define ktime_get_ts getnanouptime + +#define NSEC_PER_USEC 1000L +#define NSEC_PER_SEC 1000000000L + +/* + * ktime_t: + * + * On 64-bit CPUs a single 64-bit variable is used to store the hrtimers + * internal representation of time values in scalar nanoseconds. The + * design plays out best on 64-bit CPUs, where most conversions are + * NOPs and most arithmetic ktime_t operations are plain arithmetic + * operations. + * + * On 32-bit CPUs an optimized representation of the timespec structure + * is used to avoid expensive conversions from and to timespecs. The + * endian-aware order of the tv struct members is chosen to allow + * mathematical operations on the tv64 member of the union too, which + * for certain operations produces better code. + * + * For architectures with efficient support for 64/32-bit conversions the + * plain scalar nanosecond based representation can be selected by the + * config switch CONFIG_KTIME_SCALAR. + */ +union ktime { + s64 tv64; +#if BITS_PER_LONG != 64 && !defined(CONFIG_KTIME_SCALAR) + struct { +# ifdef __BIG_ENDIAN + s32 sec, nsec; +# else + s32 nsec, sec; +# endif + } tv; +#endif +}; + +typedef union ktime ktime_t; /* Kill this */ + +#define KTIME_MAX ((s64)~((u64)1 << 63)) +#define KTIME_SEC_MAX (KTIME_MAX / NSEC_PER_SEC) + +/* + * ktime_t definitions when using the 64-bit scalar representation: + */ + +#if (BITS_PER_LONG == 64) || defined(CONFIG_KTIME_SCALAR) + +/** + * ktime_set - Set a ktime_t variable from a seconds/nanoseconds value + * @secs: seconds to set + * @nsecs: nanoseconds to set + * + * Return the ktime_t representation of the value + */ +static inline ktime_t ktime_set(const long secs, const unsigned long nsecs) +{ +#if (BITS_PER_LONG == 64) + if (unlikely(secs >= KTIME_SEC_MAX)) + return (ktime_t){ .tv64 = KTIME_MAX }; +#endif + return (ktime_t) { .tv64 = (s64)secs * NSEC_PER_SEC + (s64)nsecs }; +} + +/* Subtract two ktime_t variables. rem = lhs -rhs: */ +#define ktime_sub(lhs, rhs) \ + ({ (ktime_t){ .tv64 = (lhs).tv64 - (rhs).tv64 }; }) + +/* Add two ktime_t variables. res = lhs + rhs: */ +#define ktime_add(lhs, rhs) \ + ({ (ktime_t){ .tv64 = (lhs).tv64 + (rhs).tv64 }; }) + +/* + * Add a ktime_t variable and a scalar nanosecond value. + * res = kt + nsval: + */ +#define ktime_add_ns(kt, nsval) \ + ({ (ktime_t){ .tv64 = (kt).tv64 + (nsval) }; }) + +/* + * Subtract a scalar nanosecod from a ktime_t variable + * res = kt - nsval: + */ +#define ktime_sub_ns(kt, nsval) \ + ({ (ktime_t){ .tv64 = (kt).tv64 - (nsval) }; }) + +/* convert a timespec to ktime_t format: */ +static inline ktime_t timespec_to_ktime(struct timespec ts) +{ + return ktime_set(ts.tv_sec, ts.tv_nsec); +} + +/* convert a timeval to ktime_t format: */ +static inline ktime_t timeval_to_ktime(struct timeval tv) +{ + return ktime_set(tv.tv_sec, tv.tv_usec * NSEC_PER_USEC); +} + +/* Map the ktime_t to timespec conversion to ns_to_timespec function */ +#define ktime_to_timespec(kt) ns_to_timespec((kt).tv64) + +/* Map the ktime_t to timeval conversion to ns_to_timeval function */ +#define ktime_to_timeval(kt) ns_to_timeval((kt).tv64) + +/* Convert ktime_t to nanoseconds - NOP in the scalar storage format: */ +#define ktime_to_ns(kt) ((kt).tv64) + +#else /* !((BITS_PER_LONG == 64) || defined(CONFIG_KTIME_SCALAR)) */ + +/* + * Helper macros/inlines to get the ktime_t math right in the timespec + * representation. The macros are sometimes ugly - their actual use is + * pretty okay-ish, given the circumstances. We do all this for + * performance reasons. The pure scalar nsec_t based code was nice and + * simple, but created too many 64-bit / 32-bit conversions and divisions. + * + * Be especially aware that negative values are represented in a way + * that the tv.sec field is negative and the tv.nsec field is greater + * or equal to zero but less than nanoseconds per second. This is the + * same representation which is used by timespecs. + * + * tv.sec < 0 and 0 >= tv.nsec < NSEC_PER_SEC + */ + +/* Set a ktime_t variable to a value in sec/nsec representation: */ +static inline ktime_t ktime_set(const long secs, const unsigned long nsecs) +{ + return (ktime_t) { .tv = { .sec = secs, .nsec = nsecs } }; +} + +/** + * ktime_sub - subtract two ktime_t variables + * @lhs: minuend + * @rhs: subtrahend + * + * Returns the remainder of the subtraction + */ +static inline ktime_t ktime_sub(const ktime_t lhs, const ktime_t rhs) +{ + ktime_t res; + + res.tv64 = lhs.tv64 - rhs.tv64; + if (res.tv.nsec < 0) + res.tv.nsec += NSEC_PER_SEC; + + return res; +} + +/** + * ktime_add - add two ktime_t variables + * @add1: addend1 + * @add2: addend2 + * + * Returns the sum of @add1 and @add2. + */ +static inline ktime_t ktime_add(const ktime_t add1, const ktime_t add2) +{ + ktime_t res; + + res.tv64 = add1.tv64 + add2.tv64; + /* + * performance trick: the (u32) -NSEC gives 0x00000000Fxxxxxxx + * so we subtract NSEC_PER_SEC and add 1 to the upper 32 bit. + * + * it's equivalent to: + * tv.nsec -= NSEC_PER_SEC + * tv.sec ++; + */ + if (res.tv.nsec >= NSEC_PER_SEC) + res.tv64 += (u32)-NSEC_PER_SEC; + + return res; +} + +/** + * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable + * @kt: addend + * @nsec: the scalar nsec value to add + * + * Returns the sum of @kt and @nsec in ktime_t format + */ +extern ktime_t ktime_add_ns(const ktime_t kt, u64 nsec); + +/** + * ktime_sub_ns - Subtract a scalar nanoseconds value from a ktime_t variable + * @kt: minuend + * @nsec: the scalar nsec value to subtract + * + * Returns the subtraction of @nsec from @kt in ktime_t format + */ +extern ktime_t ktime_sub_ns(const ktime_t kt, u64 nsec); + +/** + * timespec_to_ktime - convert a timespec to ktime_t format + * @ts: the timespec variable to convert + * + * Returns a ktime_t variable with the converted timespec value + */ +static inline ktime_t timespec_to_ktime(const struct timespec ts) +{ + return (ktime_t) { .tv = { .sec = (s32)ts.tv_sec, + .nsec = (s32)ts.tv_nsec } }; +} + +/** + * timeval_to_ktime - convert a timeval to ktime_t format + * @tv: the timeval variable to convert + * + * Returns a ktime_t variable with the converted timeval value + */ +static inline ktime_t timeval_to_ktime(const struct timeval tv) +{ + return (ktime_t) { .tv = { .sec = (s32)tv.tv_sec, + .nsec = (s32)(tv.tv_usec * + NSEC_PER_USEC) } }; +} + +/** + * ktime_to_timespec - convert a ktime_t variable to timespec format + * @kt: the ktime_t variable to convert + * + * Returns the timespec representation of the ktime value + */ +static inline struct timespec ktime_to_timespec(const ktime_t kt) +{ + return (struct timespec) { .tv_sec = (time_t) kt.tv.sec, + .tv_nsec = (long) kt.tv.nsec }; +} + +/** + * ktime_to_timeval - convert a ktime_t variable to timeval format + * @kt: the ktime_t variable to convert + * + * Returns the timeval representation of the ktime value + */ +static inline struct timeval ktime_to_timeval(const ktime_t kt) +{ + return (struct timeval) { + .tv_sec = (time_t) kt.tv.sec, + .tv_usec = (suseconds_t) (kt.tv.nsec / NSEC_PER_USEC) }; +} + +/** + * ktime_to_ns - convert a ktime_t variable to scalar nanoseconds + * @kt: the ktime_t variable to convert + * + * Returns the scalar nanoseconds representation of @kt + */ +static inline s64 ktime_to_ns(const ktime_t kt) +{ + return (s64) kt.tv.sec * NSEC_PER_SEC + kt.tv.nsec; +} + +#endif /* !((BITS_PER_LONG == 64) || defined(CONFIG_KTIME_SCALAR)) */ + +#endif /* _LINUX_KTIME_H */ Property changes on: stable/9/sys/ofed/include/linux/ktime.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/include/linux/linux_compat.c =================================================================== --- stable/9/sys/ofed/include/linux/linux_compat.c (revision 273863) +++ stable/9/sys/ofed/include/linux/linux_compat.c (revision 273864) @@ -1,699 +1,718 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include MALLOC_DEFINE(M_KMALLOC, "linux", "Linux kmalloc compat"); #include /* Undo Linux compat changes. */ #undef RB_ROOT #undef file #undef cdev #define RB_ROOT(head) (head)->rbh_root #undef LIST_HEAD /* From sys/queue.h */ #define LIST_HEAD(name, type) \ struct name { \ struct type *lh_first; /* first element */ \ } struct kobject class_root; struct device linux_rootdev; struct class miscclass; struct list_head pci_drivers; struct list_head pci_devices; spinlock_t pci_lock; int panic_cmp(struct rb_node *one, struct rb_node *two) { panic("no cmp"); } RB_GENERATE(linux_root, rb_node, __entry, panic_cmp); int kobject_set_name(struct kobject *kobj, const char *fmt, ...) { va_list args; int error; va_start(args, fmt); error = kobject_set_name_vargs(kobj, fmt, args); va_end(args); return (error); } static inline int kobject_add_complete(struct kobject *kobj, struct kobject *parent) { struct kobj_type *t; int error; kobj->parent = kobject_get(parent); error = sysfs_create_dir(kobj); if (error == 0 && kobj->ktype && kobj->ktype->default_attrs) { struct attribute **attr; t = kobj->ktype; for (attr = t->default_attrs; *attr != NULL; attr++) { error = sysfs_create_file(kobj, *attr); if (error) break; } if (error) sysfs_remove_dir(kobj); } return (error); } int kobject_add(struct kobject *kobj, struct kobject *parent, const char *fmt, ...) { va_list args; int error; va_start(args, fmt); error = kobject_set_name_vargs(kobj, fmt, args); va_end(args); if (error) return (error); return kobject_add_complete(kobj, parent); } void kobject_release(struct kref *kref) { struct kobject *kobj; char *name; kobj = container_of(kref, struct kobject, kref); sysfs_remove_dir(kobj); if (kobj->parent) kobject_put(kobj->parent); kobj->parent = NULL; name = kobj->name; if (kobj->ktype && kobj->ktype->release) kobj->ktype->release(kobj); kfree(name); } static void kobject_kfree(struct kobject *kobj) { - kfree(kobj); } +static void +kobject_kfree_name(struct kobject *kobj) +{ + if (kobj) { + kfree(kobj->name); + } +} + struct kobj_type kfree_type = { .release = kobject_kfree }; struct device * device_create(struct class *class, struct device *parent, dev_t devt, void *drvdata, const char *fmt, ...) { struct device *dev; va_list args; dev = kzalloc(sizeof(*dev), M_WAITOK); dev->parent = parent; dev->class = class; dev->devt = devt; dev->driver_data = drvdata; va_start(args, fmt); kobject_set_name_vargs(&dev->kobj, fmt, args); va_end(args); device_register(dev); return (dev); } int kobject_init_and_add(struct kobject *kobj, struct kobj_type *ktype, struct kobject *parent, const char *fmt, ...) { va_list args; int error; kobject_init(kobj, ktype); kobj->ktype = ktype; kobj->parent = parent; kobj->name = NULL; va_start(args, fmt); error = kobject_set_name_vargs(kobj, fmt, args); va_end(args); if (error) return (error); return kobject_add_complete(kobj, parent); } static void linux_file_dtor(void *cdp) { struct linux_file *filp; filp = cdp; filp->f_op->release(filp->f_vnode, filp); vdrop(filp->f_vnode); kfree(filp); } static int linux_dev_open(struct cdev *dev, int oflags, int devtype, struct thread *td) { struct linux_cdev *ldev; struct linux_file *filp; struct file *file; int error; file = curthread->td_fpop; ldev = dev->si_drv1; if (ldev == NULL) return (ENODEV); filp = kzalloc(sizeof(*filp), GFP_KERNEL); filp->f_dentry = &filp->f_dentry_store; filp->f_op = ldev->ops; filp->f_flags = file->f_flag; vhold(file->f_vnode); filp->f_vnode = file->f_vnode; if (filp->f_op->open) { error = -filp->f_op->open(file->f_vnode, filp); if (error) { kfree(filp); return (error); } } error = devfs_set_cdevpriv(filp, linux_file_dtor); if (error) { filp->f_op->release(file->f_vnode, filp); kfree(filp); return (error); } return 0; } static int linux_dev_close(struct cdev *dev, int fflag, int devtype, struct thread *td) { struct linux_cdev *ldev; struct linux_file *filp; struct file *file; int error; file = curthread->td_fpop; ldev = dev->si_drv1; if (ldev == NULL) return (0); if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) return (error); filp->f_flags = file->f_flag; + devfs_clear_cdevpriv(); + return (0); } static int linux_dev_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td) { struct linux_cdev *ldev; struct linux_file *filp; struct file *file; int error; file = curthread->td_fpop; ldev = dev->si_drv1; if (ldev == NULL) return (0); if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) return (error); filp->f_flags = file->f_flag; /* * Linux does not have a generic ioctl copyin/copyout layer. All * linux ioctls must be converted to void ioctls which pass a * pointer to the address of the data. We want the actual user * address so we dereference here. */ data = *(void **)data; if (filp->f_op->unlocked_ioctl) error = -filp->f_op->unlocked_ioctl(filp, cmd, (u_long)data); else error = ENOTTY; return (error); } static int linux_dev_read(struct cdev *dev, struct uio *uio, int ioflag) { struct linux_cdev *ldev; struct linux_file *filp; struct file *file; ssize_t bytes; int error; file = curthread->td_fpop; ldev = dev->si_drv1; if (ldev == NULL) return (0); if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) return (error); filp->f_flags = file->f_flag; if (uio->uio_iovcnt != 1) panic("linux_dev_read: uio %p iovcnt %d", uio, uio->uio_iovcnt); if (filp->f_op->read) { bytes = filp->f_op->read(filp, uio->uio_iov->iov_base, uio->uio_iov->iov_len, &uio->uio_offset); if (bytes >= 0) { uio->uio_iov->iov_base += bytes; uio->uio_iov->iov_len -= bytes; uio->uio_resid -= bytes; } else error = -bytes; } else error = ENXIO; return (error); } static int linux_dev_write(struct cdev *dev, struct uio *uio, int ioflag) { struct linux_cdev *ldev; struct linux_file *filp; struct file *file; ssize_t bytes; int error; file = curthread->td_fpop; ldev = dev->si_drv1; if (ldev == NULL) return (0); if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) return (error); filp->f_flags = file->f_flag; if (uio->uio_iovcnt != 1) panic("linux_dev_write: uio %p iovcnt %d", uio, uio->uio_iovcnt); if (filp->f_op->write) { bytes = filp->f_op->write(filp, uio->uio_iov->iov_base, uio->uio_iov->iov_len, &uio->uio_offset); if (bytes >= 0) { uio->uio_iov->iov_base += bytes; uio->uio_iov->iov_len -= bytes; uio->uio_resid -= bytes; } else error = -bytes; } else error = ENXIO; return (error); } static int linux_dev_poll(struct cdev *dev, int events, struct thread *td) { struct linux_cdev *ldev; struct linux_file *filp; struct file *file; int revents; int error; file = curthread->td_fpop; ldev = dev->si_drv1; if (ldev == NULL) return (0); if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) return (error); filp->f_flags = file->f_flag; if (filp->f_op->poll) revents = filp->f_op->poll(filp, NULL) & events; else revents = 0; return (revents); } static int linux_dev_mmap(struct cdev *dev, vm_ooffset_t offset, vm_paddr_t *paddr, int nprot, vm_memattr_t *memattr) { /* XXX memattr not honored. */ *paddr = offset; return (0); } static int linux_dev_mmap_single(struct cdev *dev, vm_ooffset_t *offset, vm_size_t size, struct vm_object **object, int nprot) { struct linux_cdev *ldev; struct linux_file *filp; struct file *file; struct vm_area_struct vma; vm_paddr_t paddr; vm_page_t m; int error; file = curthread->td_fpop; ldev = dev->si_drv1; if (ldev == NULL) return (ENODEV); if (size != PAGE_SIZE) return (EINVAL); if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) return (error); filp->f_flags = file->f_flag; vma.vm_start = 0; vma.vm_end = PAGE_SIZE; vma.vm_pgoff = *offset / PAGE_SIZE; vma.vm_pfn = 0; vma.vm_page_prot = 0; if (filp->f_op->mmap) { error = -filp->f_op->mmap(filp, &vma); if (error == 0) { paddr = (vm_paddr_t)vma.vm_pfn << PAGE_SHIFT; *offset = paddr; m = PHYS_TO_VM_PAGE(paddr); *object = vm_pager_allocate(OBJT_DEVICE, dev, PAGE_SIZE, nprot, *offset, curthread->td_ucred); if (*object == NULL) return (EINVAL); if (vma.vm_page_prot != VM_MEMATTR_DEFAULT) pmap_page_set_memattr(m, vma.vm_page_prot); } } else error = ENODEV; return (error); } struct cdevsw linuxcdevsw = { .d_version = D_VERSION, .d_flags = D_TRACKCLOSE, .d_open = linux_dev_open, .d_close = linux_dev_close, .d_read = linux_dev_read, .d_write = linux_dev_write, .d_ioctl = linux_dev_ioctl, .d_mmap_single = linux_dev_mmap_single, .d_mmap = linux_dev_mmap, .d_poll = linux_dev_poll, }; static int linux_file_read(struct file *file, struct uio *uio, struct ucred *active_cred, int flags, struct thread *td) { struct linux_file *filp; ssize_t bytes; int error; error = 0; filp = (struct linux_file *)file->f_data; filp->f_flags = file->f_flag; if (uio->uio_iovcnt != 1) panic("linux_file_read: uio %p iovcnt %d", uio, uio->uio_iovcnt); if (filp->f_op->read) { bytes = filp->f_op->read(filp, uio->uio_iov->iov_base, uio->uio_iov->iov_len, &uio->uio_offset); if (bytes >= 0) { uio->uio_iov->iov_base += bytes; uio->uio_iov->iov_len -= bytes; uio->uio_resid -= bytes; } else error = -bytes; } else error = ENXIO; return (error); } static int linux_file_poll(struct file *file, int events, struct ucred *active_cred, struct thread *td) { struct linux_file *filp; int revents; filp = (struct linux_file *)file->f_data; filp->f_flags = file->f_flag; if (filp->f_op->poll) revents = filp->f_op->poll(filp, NULL) & events; else revents = 0; return (0); } static int linux_file_close(struct file *file, struct thread *td) { struct linux_file *filp; int error; filp = (struct linux_file *)file->f_data; filp->f_flags = file->f_flag; error = -filp->f_op->release(NULL, filp); funsetown(&filp->f_sigio); kfree(filp); return (error); } static int linux_file_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *cred, struct thread *td) { struct linux_file *filp; int error; filp = (struct linux_file *)fp->f_data; filp->f_flags = fp->f_flag; error = 0; switch (cmd) { case FIONBIO: break; case FIOASYNC: if (filp->f_op->fasync == NULL) break; error = filp->f_op->fasync(0, filp, fp->f_flag & FASYNC); break; case FIOSETOWN: error = fsetown(*(int *)data, &filp->f_sigio); if (error == 0) error = filp->f_op->fasync(0, filp, fp->f_flag & FASYNC); break; case FIOGETOWN: *(int *)data = fgetown(&filp->f_sigio); break; default: error = ENOTTY; break; } return (error); } struct fileops linuxfileops = { .fo_read = linux_file_read, .fo_poll = linux_file_poll, .fo_close = linux_file_close, .fo_ioctl = linux_file_ioctl, .fo_chmod = invfo_chmod, .fo_chown = invfo_chown, }; /* * Hash of vmmap addresses. This is infrequently accessed and does not * need to be particularly large. This is done because we must store the * caller's idea of the map size to properly unmap. */ struct vmmap { LIST_ENTRY(vmmap) vm_next; void *vm_addr; unsigned long vm_size; }; LIST_HEAD(vmmaphd, vmmap); #define VMMAP_HASH_SIZE 64 #define VMMAP_HASH_MASK (VMMAP_HASH_SIZE - 1) #define VM_HASH(addr) ((uintptr_t)(addr) >> PAGE_SHIFT) & VMMAP_HASH_MASK static struct vmmaphd vmmaphead[VMMAP_HASH_SIZE]; static struct mtx vmmaplock; static void vmmap_add(void *addr, unsigned long size) { struct vmmap *vmmap; vmmap = kmalloc(sizeof(*vmmap), GFP_KERNEL); mtx_lock(&vmmaplock); vmmap->vm_size = size; vmmap->vm_addr = addr; LIST_INSERT_HEAD(&vmmaphead[VM_HASH(addr)], vmmap, vm_next); mtx_unlock(&vmmaplock); } static struct vmmap * vmmap_remove(void *addr) { struct vmmap *vmmap; mtx_lock(&vmmaplock); LIST_FOREACH(vmmap, &vmmaphead[VM_HASH(addr)], vm_next) if (vmmap->vm_addr == addr) break; if (vmmap) LIST_REMOVE(vmmap, vm_next); mtx_unlock(&vmmaplock); return (vmmap); } void * _ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr) { void *addr; addr = pmap_mapdev_attr(phys_addr, size, attr); if (addr == NULL) return (NULL); vmmap_add(addr, size); return (addr); } void iounmap(void *addr) { struct vmmap *vmmap; vmmap = vmmap_remove(addr); if (vmmap == NULL) return; pmap_unmapdev((vm_offset_t)addr, vmmap->vm_size); kfree(vmmap); } void * vmap(struct page **pages, unsigned int count, unsigned long flags, int prot) { vm_offset_t off; size_t size; size = count * PAGE_SIZE; off = kmem_alloc_nofault(kernel_map, size); if (off == 0) return (NULL); vmmap_add((void *)off, size); pmap_qenter(off, pages, count); return ((void *)off); } void vunmap(void *addr) { struct vmmap *vmmap; vmmap = vmmap_remove(addr); if (vmmap == NULL) return; pmap_qremove((vm_offset_t)addr, vmmap->vm_size / PAGE_SIZE); kmem_free(kernel_map, (vm_offset_t)addr, vmmap->vm_size); kfree(vmmap); } static void linux_compat_init(void) { struct sysctl_oid *rootoid; int i; rootoid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(), OID_AUTO, "sys", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "sys"); kobject_init(&class_root, &class_ktype); kobject_set_name(&class_root, "class"); class_root.oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(rootoid), OID_AUTO, "class", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "class"); kobject_init(&linux_rootdev.kobj, &dev_ktype); kobject_set_name(&linux_rootdev.kobj, "device"); linux_rootdev.kobj.oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(rootoid), OID_AUTO, "device", CTLFLAG_RD, NULL, "device"); linux_rootdev.bsddev = root_bus; miscclass.name = "misc"; class_register(&miscclass); INIT_LIST_HEAD(&pci_drivers); INIT_LIST_HEAD(&pci_devices); spin_lock_init(&pci_lock); mtx_init(&vmmaplock, "IO Map lock", NULL, MTX_DEF); for (i = 0; i < VMMAP_HASH_SIZE; i++) LIST_INIT(&vmmaphead[i]); } SYSINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_init, NULL); + +static void +linux_compat_uninit(void) +{ + kobject_kfree_name(&class_root); + kobject_kfree_name(&linux_rootdev.kobj); + kobject_kfree_name(&miscclass.kobj); +} +SYSUNINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_uninit, NULL); Index: stable/9/sys/ofed/include/linux/linux_idr.c =================================================================== --- stable/9/sys/ofed/include/linux/linux_idr.c (revision 273863) +++ stable/9/sys/ofed/include/linux/linux_idr.c (revision 273864) @@ -1,447 +1,448 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include /* * IDR Implementation. * * This is quick and dirty and not as re-entrant as the linux version * however it should be fairly fast. It is basically a radix tree with * a builtin bitmap for allocation. */ static MALLOC_DEFINE(M_IDR, "idr", "Linux IDR compat"); static inline int idr_max(struct idr *idr) { return (1 << (idr->layers * IDR_BITS)) - 1; } static inline int idr_pos(int id, int layer) { return (id >> (IDR_BITS * layer)) & IDR_MASK; } void idr_init(struct idr *idr) { bzero(idr, sizeof(*idr)); mtx_init(&idr->lock, "idr", NULL, MTX_DEF); } /* Only frees cached pages. */ void idr_destroy(struct idr *idr) { struct idr_layer *il, *iln; mtx_lock(&idr->lock); for (il = idr->free; il != NULL; il = iln) { iln = il->ary[0]; free(il, M_IDR); } mtx_unlock(&idr->lock); } static void idr_remove_layer(struct idr_layer *il, int layer) { int i; if (il == NULL) return; if (layer == 0) { free(il, M_IDR); return; } for (i = 0; i < IDR_SIZE; i++) if (il->ary[i]) idr_remove_layer(il->ary[i], layer - 1); } void idr_remove_all(struct idr *idr) { mtx_lock(&idr->lock); idr_remove_layer(idr->top, idr->layers - 1); idr->top = NULL; idr->layers = 0; mtx_unlock(&idr->lock); } void idr_remove(struct idr *idr, int id) { struct idr_layer *il; int layer; int idx; id &= MAX_ID_MASK; mtx_lock(&idr->lock); il = idr->top; layer = idr->layers - 1; if (il == NULL || id > idr_max(idr)) { mtx_unlock(&idr->lock); return; } /* * Walk down the tree to this item setting bitmaps along the way * as we know at least one item will be free along this path. */ while (layer && il) { idx = idr_pos(id, layer); il->bitmap |= 1 << idx; il = il->ary[idx]; layer--; } idx = id & IDR_MASK; /* * At this point we've set free space bitmaps up the whole tree. * We could make this non-fatal and unwind but linux dumps a stack * and a warning so I don't think it's necessary. */ if (il == NULL || (il->bitmap & (1 << idx)) != 0) panic("idr_remove: Item %d not allocated (%p, %p)\n", id, idr, il); il->ary[idx] = NULL; il->bitmap |= 1 << idx; mtx_unlock(&idr->lock); return; } void * idr_replace(struct idr *idr, void *ptr, int id) { struct idr_layer *il; void *res; int layer; int idx; res = ERR_PTR(-EINVAL); id &= MAX_ID_MASK; mtx_lock(&idr->lock); il = idr->top; layer = idr->layers - 1; if (il == NULL || id > idr_max(idr)) goto out; while (layer && il) { il = il->ary[idr_pos(id, layer)]; layer--; } idx = id & IDR_MASK; /* * Replace still returns an error if the item was not allocated. */ if (il != NULL && (il->bitmap & (1 << idx)) != 0) { res = il->ary[idx]; il->ary[idx] = ptr; } out: mtx_unlock(&idr->lock); return (res); } void * idr_find(struct idr *idr, int id) { struct idr_layer *il; void *res; int layer; res = NULL; id &= MAX_ID_MASK; mtx_lock(&idr->lock); il = idr->top; layer = idr->layers - 1; if (il == NULL || id > idr_max(idr)) goto out; while (layer && il) { il = il->ary[idr_pos(id, layer)]; layer--; } if (il != NULL) res = il->ary[id & IDR_MASK]; out: mtx_unlock(&idr->lock); return (res); } int idr_pre_get(struct idr *idr, gfp_t gfp_mask) { struct idr_layer *il, *iln; struct idr_layer *head; int need; mtx_lock(&idr->lock); for (;;) { need = idr->layers + 1; for (il = idr->free; il != NULL; il = il->ary[0]) need--; mtx_unlock(&idr->lock); if (need == 0) break; for (head = NULL; need; need--) { iln = malloc(sizeof(*il), M_IDR, M_ZERO | gfp_mask); if (iln == NULL) break; bitmap_fill(&iln->bitmap, IDR_SIZE); if (head != NULL) { il->ary[0] = iln; il = iln; } else head = il = iln; } if (head == NULL) return (0); mtx_lock(&idr->lock); il->ary[0] = idr->free; idr->free = head; } return (1); } static inline struct idr_layer * idr_get(struct idr *idr) { struct idr_layer *il; il = idr->free; if (il) { idr->free = il->ary[0]; il->ary[0] = NULL; return (il); } il = malloc(sizeof(*il), M_IDR, M_ZERO | M_NOWAIT); bitmap_fill(&il->bitmap, IDR_SIZE); return (il); } /* * Could be implemented as get_new_above(idr, ptr, 0, idp) but written * first for simplicity sake. */ int idr_get_new(struct idr *idr, void *ptr, int *idp) { struct idr_layer *stack[MAX_LEVEL]; struct idr_layer *il; int error; int layer; int idx; int id; error = -EAGAIN; mtx_lock(&idr->lock); /* * Expand the tree until there is free space. */ if (idr->top == NULL || idr->top->bitmap == 0) { if (idr->layers == MAX_LEVEL + 1) { error = -ENOSPC; goto out; } il = idr_get(idr); if (il == NULL) goto out; il->ary[0] = idr->top; if (idr->top) il->bitmap &= ~1; idr->top = il; idr->layers++; } il = idr->top; id = 0; /* * Walk the tree following free bitmaps, record our path. */ for (layer = idr->layers - 1;; layer--) { stack[layer] = il; idx = ffsl(il->bitmap); if (idx == 0) panic("idr_get_new: Invalid leaf state (%p, %p)\n", idr, il); idx--; id |= idx << (layer * IDR_BITS); if (layer == 0) break; if (il->ary[idx] == NULL) { il->ary[idx] = idr_get(idr); if (il->ary[idx] == NULL) goto out; } il = il->ary[idx]; } /* * Allocate the leaf to the consumer. */ il->bitmap &= ~(1 << idx); il->ary[idx] = ptr; *idp = id; /* * Clear bitmaps potentially up to the root. */ while (il->bitmap == 0 && ++layer < idr->layers) { il = stack[layer]; il->bitmap &= ~(1 << idr_pos(id, layer)); } error = 0; out: mtx_unlock(&idr->lock); #ifdef INVARIANTS if (error == 0 && idr_find(idr, id) != ptr) { panic("idr_get_new: Failed for idr %p, id %d, ptr %p\n", idr, id, ptr); } #endif return (error); } int idr_get_new_above(struct idr *idr, void *ptr, int starting_id, int *idp) { struct idr_layer *stack[MAX_LEVEL]; struct idr_layer *il; int error; int layer; int idx, sidx; int id; error = -EAGAIN; mtx_lock(&idr->lock); /* * Compute the layers required to support starting_id and the mask * at the top layer. */ restart: idx = starting_id; layer = 0; while (idx & ~IDR_MASK) { layer++; idx >>= IDR_BITS; } if (layer == MAX_LEVEL + 1) { error = -ENOSPC; goto out; } /* * Expand the tree until there is free space at or beyond starting_id. */ while (idr->layers <= layer || idr->top->bitmap < (1 << idr_pos(starting_id, idr->layers - 1))) { if (idr->layers == MAX_LEVEL + 1) { error = -ENOSPC; goto out; } il = idr_get(idr); if (il == NULL) goto out; il->ary[0] = idr->top; if (idr->top && idr->top->bitmap == 0) il->bitmap &= ~1; idr->top = il; idr->layers++; } il = idr->top; id = 0; /* * Walk the tree following free bitmaps, record our path. */ for (layer = idr->layers - 1;; layer--) { stack[layer] = il; sidx = idr_pos(starting_id, layer); /* Returns index numbered from 0 or size if none exists. */ idx = find_next_bit(&il->bitmap, IDR_SIZE, sidx); if (idx == IDR_SIZE && sidx == 0) panic("idr_get_new: Invalid leaf state (%p, %p)\n", idr, il); /* * We may have walked a path where there was a free bit but * it was lower than what we wanted. Restart the search with * a larger starting id. id contains the progress we made so * far. Search the leaf one above this level. This may * restart as many as MAX_LEVEL times but that is expected * to be rare. */ if (idx == IDR_SIZE) { starting_id = id + (1 << (layer+1 * IDR_BITS)); goto restart; } if (idx > sidx) starting_id = 0; /* Search the whole subtree. */ id |= idx << (layer * IDR_BITS); if (layer == 0) break; if (il->ary[idx] == NULL) { il->ary[idx] = idr_get(idr); if (il->ary[idx] == NULL) goto out; } il = il->ary[idx]; } /* * Allocate the leaf to the consumer. */ il->bitmap &= ~(1 << idx); il->ary[idx] = ptr; *idp = id; /* * Clear bitmaps potentially up to the root. */ while (il->bitmap == 0 && ++layer < idr->layers) { il = stack[layer]; il->bitmap &= ~(1 << idr_pos(id, layer)); } error = 0; out: mtx_unlock(&idr->lock); #ifdef INVARIANTS if (error == 0 && idr_find(idr, id) != ptr) { panic("idr_get_new_above: Failed for idr %p, id %d, ptr %p\n", idr, id, ptr); } #endif return (error); } Index: stable/9/sys/ofed/include/linux/linux_radix.c =================================================================== --- stable/9/sys/ofed/include/linux/linux_radix.c (revision 273863) +++ stable/9/sys/ofed/include/linux/linux_radix.c (revision 273864) @@ -1,214 +1,215 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include static MALLOC_DEFINE(M_RADIX, "radix", "Linux radix compat"); static inline int radix_max(struct radix_tree_root *root) { return (1 << (root->height * RADIX_TREE_MAP_SHIFT)) - 1; } static inline int radix_pos(long id, int height) { return (id >> (RADIX_TREE_MAP_SHIFT * height)) & RADIX_TREE_MAP_MASK; } void * radix_tree_lookup(struct radix_tree_root *root, unsigned long index) { struct radix_tree_node *node; void *item; int height; item = NULL; node = root->rnode; height = root->height - 1; if (index > radix_max(root)) goto out; while (height && node) node = node->slots[radix_pos(index, height--)]; if (node) item = node->slots[radix_pos(index, 0)]; out: return (item); } void * radix_tree_delete(struct radix_tree_root *root, unsigned long index) { struct radix_tree_node *stack[RADIX_TREE_MAX_HEIGHT]; struct radix_tree_node *node; void *item; int height; int idx; item = NULL; node = root->rnode; height = root->height - 1; if (index > radix_max(root)) goto out; /* * Find the node and record the path in stack. */ while (height && node) { stack[height] = node; node = node->slots[radix_pos(index, height--)]; } idx = radix_pos(index, 0); if (node) item = node->slots[idx]; /* * If we removed something reduce the height of the tree. */ if (item) for (;;) { node->slots[idx] = NULL; node->count--; if (node->count > 0) break; free(node, M_RADIX); if (node == root->rnode) { root->rnode = NULL; root->height = 0; break; } height++; node = stack[height]; idx = radix_pos(index, height); } out: return (item); } int radix_tree_insert(struct radix_tree_root *root, unsigned long index, void *item) { struct radix_tree_node *node; struct radix_tree_node *temp[RADIX_TREE_MAX_HEIGHT - 1]; int height; int idx; /* bail out upon insertion of a NULL item */ if (item == NULL) return (-EINVAL); /* get root node, if any */ node = root->rnode; /* allocate root node, if any */ if (node == NULL) { node = malloc(sizeof(*node), M_RADIX, root->gfp_mask | M_ZERO); if (node == NULL) return (-ENOMEM); root->rnode = node; root->height++; } /* expand radix tree as needed */ while (radix_max(root) < index) { /* check if the radix tree is getting too big */ if (root->height == RADIX_TREE_MAX_HEIGHT) return (-E2BIG); /* * If the root radix level is not empty, we need to * allocate a new radix level: */ if (node->count != 0) { node = malloc(sizeof(*node), M_RADIX, root->gfp_mask | M_ZERO); if (node == NULL) return (-ENOMEM); node->slots[0] = root->rnode; node->count++; root->rnode = node; } root->height++; } /* get radix tree height index */ height = root->height - 1; /* walk down the tree until the first missing node, if any */ for ( ; height != 0; height--) { idx = radix_pos(index, height); if (node->slots[idx] == NULL) break; node = node->slots[idx]; } /* allocate the missing radix levels, if any */ for (idx = 0; idx != height; idx++) { temp[idx] = malloc(sizeof(*node), M_RADIX, root->gfp_mask | M_ZERO); if (temp[idx] == NULL) { while(idx--) free(temp[idx], M_RADIX); /* check if we should free the root node aswell */ if (root->rnode->count == 0) { free(root->rnode, M_RADIX); root->rnode = NULL; root->height = 0; } return (-ENOMEM); } } /* setup new radix levels, if any */ for ( ; height != 0; height--) { idx = radix_pos(index, height); node->slots[idx] = temp[height - 1]; node->count++; node = node->slots[idx]; } /* * Insert and adjust count if the item does not already exist. */ idx = radix_pos(index, 0); if (node->slots[idx]) return (-EEXIST); node->slots[idx] = item; node->count++; return (0); } Index: stable/9/sys/ofed/include/linux/list.h =================================================================== --- stable/9/sys/ofed/include/linux/list.h (revision 273863) +++ stable/9/sys/ofed/include/linux/list.h (revision 273864) @@ -1,332 +1,397 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_LIST_H_ #define _LINUX_LIST_H_ /* * Since LIST_HEAD conflicts with the linux definition we must include any * FreeBSD header which requires it here so it is resolved with the correct * definition prior to the undef. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define prefetch(x) struct list_head { struct list_head *next; struct list_head *prev; }; static inline void INIT_LIST_HEAD(struct list_head *list) { list->next = list->prev = list; } static inline int list_empty(const struct list_head *head) { return (head->next == head); } static inline void list_del(struct list_head *entry) { entry->next->prev = entry->prev; entry->prev->next = entry->next; } static inline void _list_add(struct list_head *new, struct list_head *prev, struct list_head *next) { next->prev = new; new->next = next; new->prev = prev; prev->next = new; } static inline void list_del_init(struct list_head *entry) { list_del(entry); INIT_LIST_HEAD(entry); } #define list_entry(ptr, type, field) container_of(ptr, type, field) +#define list_first_entry(ptr, type, member) \ + list_entry((ptr)->next, type, member) + #define list_for_each(p, head) \ for (p = (head)->next; p != (head); p = p->next) #define list_for_each_safe(p, n, head) \ for (p = (head)->next, n = p->next; p != (head); p = n, n = p->next) #define list_for_each_entry(p, h, field) \ for (p = list_entry((h)->next, typeof(*p), field); &p->field != (h); \ p = list_entry(p->field.next, typeof(*p), field)) #define list_for_each_entry_safe(p, n, h, field) \ for (p = list_entry((h)->next, typeof(*p), field), \ n = list_entry(p->field.next, typeof(*p), field); &p->field != (h);\ p = n, n = list_entry(n->field.next, typeof(*n), field)) #define list_for_each_entry_reverse(p, h, field) \ for (p = list_entry((h)->prev, typeof(*p), field); &p->field != (h); \ p = list_entry(p->field.prev, typeof(*p), field)) #define list_for_each_prev(p, h) for (p = (h)->prev; p != (h); p = p->prev) static inline void list_add(struct list_head *new, struct list_head *head) { _list_add(new, head, head->next); } static inline void list_add_tail(struct list_head *new, struct list_head *head) { _list_add(new, head->prev, head); } static inline void list_move(struct list_head *list, struct list_head *head) { list_del(list); list_add(list, head); } static inline void list_move_tail(struct list_head *entry, struct list_head *head) { list_del(entry); list_add_tail(entry, head); } static inline void _list_splice(const struct list_head *list, struct list_head *prev, struct list_head *next) { struct list_head *first; struct list_head *last; if (list_empty(list)) return; first = list->next; last = list->prev; first->prev = prev; prev->next = first; last->next = next; next->prev = last; } static inline void list_splice(const struct list_head *list, struct list_head *head) { _list_splice(list, head, head->next); } static inline void list_splice_tail(struct list_head *list, struct list_head *head) { _list_splice(list, head->prev, head); } static inline void list_splice_init(struct list_head *list, struct list_head *head) { _list_splice(list, head, head->next); INIT_LIST_HEAD(list); } static inline void list_splice_tail_init(struct list_head *list, struct list_head *head) { _list_splice(list, head->prev, head); INIT_LIST_HEAD(list); } #undef LIST_HEAD #define LIST_HEAD(name) struct list_head name = { &(name), &(name) } struct hlist_head { struct hlist_node *first; }; struct hlist_node { struct hlist_node *next, **pprev; }; #define HLIST_HEAD_INIT { } #define HLIST_HEAD(name) struct hlist_head name = HLIST_HEAD_INIT #define INIT_HLIST_HEAD(head) (head)->first = NULL #define INIT_HLIST_NODE(node) \ do { \ (node)->next = NULL; \ (node)->pprev = NULL; \ } while (0) static inline int hlist_unhashed(const struct hlist_node *h) { return !h->pprev; } static inline int hlist_empty(const struct hlist_head *h) { return !h->first; } static inline void hlist_del(struct hlist_node *n) { if (n->next) n->next->pprev = n->pprev; *n->pprev = n->next; } static inline void hlist_del_init(struct hlist_node *n) { if (hlist_unhashed(n)) return; hlist_del(n); INIT_HLIST_NODE(n); } static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h) { n->next = h->first; if (h->first) h->first->pprev = &n->next; h->first = n; n->pprev = &h->first; } static inline void hlist_add_before(struct hlist_node *n, struct hlist_node *next) { n->pprev = next->pprev; n->next = next; next->pprev = &n->next; *(n->pprev) = n; } static inline void hlist_add_after(struct hlist_node *n, struct hlist_node *next) { next->next = n->next; n->next = next; next->pprev = &n->next; if (next->next) next->next->pprev = &next->next; } static inline void hlist_move_list(struct hlist_head *old, struct hlist_head *new) { new->first = old->first; if (new->first) new->first->pprev = &new->first; old->first = NULL; } + +/** + * list_is_singular - tests whether a list has just one entry. + * @head: the list to test. + */ +static inline int list_is_singular(const struct list_head *head) +{ + return !list_empty(head) && (head->next == head->prev); +} + +static inline void __list_cut_position(struct list_head *list, + struct list_head *head, struct list_head *entry) +{ + struct list_head *new_first = entry->next; + list->next = head->next; + list->next->prev = list; + list->prev = entry; + entry->next = list; + head->next = new_first; + new_first->prev = head; +} + +/** + * list_cut_position - cut a list into two + * @list: a new list to add all removed entries + * @head: a list with entries + * @entry: an entry within head, could be the head itself + * and if so we won't cut the list + * + * This helper moves the initial part of @head, up to and + * including @entry, from @head to @list. You should + * pass on @entry an element you know is on @head. @list + * should be an empty list or a list you do not care about + * losing its data. + * + */ +static inline void list_cut_position(struct list_head *list, + struct list_head *head, struct list_head *entry) +{ + if (list_empty(head)) + return; + if (list_is_singular(head) && + (head->next != entry && head != entry)) + return; + if (entry == head) + INIT_LIST_HEAD(list); + else + __list_cut_position(list, head, entry); +} + +/** + * list_is_last - tests whether @list is the last entry in list @head + * @list: the entry to test + * @head: the head of the list + */ +static inline int list_is_last(const struct list_head *list, + const struct list_head *head) +{ + return list->next == head; +} #define hlist_entry(ptr, type, field) container_of(ptr, type, field) #define hlist_for_each(p, head) \ for (p = (head)->first; p; p = p->next) #define hlist_for_each_safe(p, n, head) \ for (p = (head)->first; p && ({ n = p->next; 1; }); p = n) #define hlist_for_each_entry(tp, p, head, field) \ for (p = (head)->first; \ p ? (tp = hlist_entry(p, typeof(*tp), field)): NULL; p = p->next) #define hlist_for_each_entry_continue(tp, p, field) \ for (p = (p)->next; \ p ? (tp = hlist_entry(p, typeof(*tp), field)): NULL; p = p->next) #define hlist_for_each_entry_from(tp, p, field) \ for (; p ? (tp = hlist_entry(p, typeof(*tp), field)): NULL; p = p->next) -#define hlist_for_each_entry_safe(tp, p, n, head, field) \ - for (p = (head)->first; p ? \ - (n = p->next) | (tp = hlist_entry(p, typeof(*tp), field)) : \ - NULL; p = n) +#define hlist_for_each_entry_safe(tpos, pos, n, head, member) \ + for (pos = (head)->first; \ + (pos) != 0 && ({ n = (pos)->next; \ + tpos = hlist_entry((pos), typeof(*(tpos)), member); 1;}); \ + pos = (n)) #endif /* _LINUX_LIST_H_ */ Index: stable/9/sys/ofed/include/linux/lockdep.h =================================================================== --- stable/9/sys/ofed/include/linux/lockdep.h (revision 273863) +++ stable/9/sys/ofed/include/linux/lockdep.h (revision 273864) @@ -1,37 +1,40 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_LOCKDEP_H_ #define _LINUX_LOCKDEP_H_ struct lock_class_key { }; #define lockdep_set_class(lock, key) -#endif /* _LINUX_LOCKDEP_H_ */ +#define lockdep_set_class_and_name(lock, key, name) + +#endif /* _LINUX_LOCKDEP_H_ */ Index: stable/9/sys/ofed/include/linux/log2.h =================================================================== --- stable/9/sys/ofed/include/linux/log2.h (revision 273863) +++ stable/9/sys/ofed/include/linux/log2.h (revision 273864) @@ -1,60 +1,170 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_LOG2_H_ #define _LINUX_LOG2_H_ #include #include static inline unsigned long roundup_pow_of_two(unsigned long x) { return (1UL << flsl(x - 1)); } static inline int is_power_of_2(unsigned long n) { return (n == roundup_pow_of_two(n)); } static inline unsigned long rounddown_pow_of_two(unsigned long x) { return (1UL << (flsl(x) - 1)); } -static inline unsigned long -ilog2(unsigned long x) + +/* + * deal with unrepresentable constant logarithms + */ +extern __attribute__((const, noreturn)) +int ____ilog2_NaN(void); + +/* + * non-constant log of base 2 calculators + * - the arch may override these in asm/bitops.h if they can be implemented + * more efficiently than using fls() and fls64() + * - the arch is not required to handle n==0 if implementing the fallback + */ +#ifndef CONFIG_ARCH_HAS_ILOG2_U32 +static inline __attribute__((const)) +int __ilog2_u32(u32 n) { - return (flsl(x) - 1); + return flsl(n) - 1; } +#endif + +#ifndef CONFIG_ARCH_HAS_ILOG2_U64 +static inline __attribute__((const)) +int __ilog2_u64(u64 n) +{ + return flsl(n) - 1; +} +#endif + + +/** + * ilog2 - log of base 2 of 32-bit or a 64-bit unsigned value + * @n - parameter + * + * constant-capable log of base 2 calculation + * - this can be used to initialise global variables from constant data, hence + * the massive ternary operator construction + * + * selects the appropriately-sized optimised version depending on sizeof(n) + */ +#define ilog2(n) \ +( \ + __builtin_constant_p(n) ? ( \ + (n) < 1 ? ____ilog2_NaN() : \ + (n) & (1ULL << 63) ? 63 : \ + (n) & (1ULL << 62) ? 62 : \ + (n) & (1ULL << 61) ? 61 : \ + (n) & (1ULL << 60) ? 60 : \ + (n) & (1ULL << 59) ? 59 : \ + (n) & (1ULL << 58) ? 58 : \ + (n) & (1ULL << 57) ? 57 : \ + (n) & (1ULL << 56) ? 56 : \ + (n) & (1ULL << 55) ? 55 : \ + (n) & (1ULL << 54) ? 54 : \ + (n) & (1ULL << 53) ? 53 : \ + (n) & (1ULL << 52) ? 52 : \ + (n) & (1ULL << 51) ? 51 : \ + (n) & (1ULL << 50) ? 50 : \ + (n) & (1ULL << 49) ? 49 : \ + (n) & (1ULL << 48) ? 48 : \ + (n) & (1ULL << 47) ? 47 : \ + (n) & (1ULL << 46) ? 46 : \ + (n) & (1ULL << 45) ? 45 : \ + (n) & (1ULL << 44) ? 44 : \ + (n) & (1ULL << 43) ? 43 : \ + (n) & (1ULL << 42) ? 42 : \ + (n) & (1ULL << 41) ? 41 : \ + (n) & (1ULL << 40) ? 40 : \ + (n) & (1ULL << 39) ? 39 : \ + (n) & (1ULL << 38) ? 38 : \ + (n) & (1ULL << 37) ? 37 : \ + (n) & (1ULL << 36) ? 36 : \ + (n) & (1ULL << 35) ? 35 : \ + (n) & (1ULL << 34) ? 34 : \ + (n) & (1ULL << 33) ? 33 : \ + (n) & (1ULL << 32) ? 32 : \ + (n) & (1ULL << 31) ? 31 : \ + (n) & (1ULL << 30) ? 30 : \ + (n) & (1ULL << 29) ? 29 : \ + (n) & (1ULL << 28) ? 28 : \ + (n) & (1ULL << 27) ? 27 : \ + (n) & (1ULL << 26) ? 26 : \ + (n) & (1ULL << 25) ? 25 : \ + (n) & (1ULL << 24) ? 24 : \ + (n) & (1ULL << 23) ? 23 : \ + (n) & (1ULL << 22) ? 22 : \ + (n) & (1ULL << 21) ? 21 : \ + (n) & (1ULL << 20) ? 20 : \ + (n) & (1ULL << 19) ? 19 : \ + (n) & (1ULL << 18) ? 18 : \ + (n) & (1ULL << 17) ? 17 : \ + (n) & (1ULL << 16) ? 16 : \ + (n) & (1ULL << 15) ? 15 : \ + (n) & (1ULL << 14) ? 14 : \ + (n) & (1ULL << 13) ? 13 : \ + (n) & (1ULL << 12) ? 12 : \ + (n) & (1ULL << 11) ? 11 : \ + (n) & (1ULL << 10) ? 10 : \ + (n) & (1ULL << 9) ? 9 : \ + (n) & (1ULL << 8) ? 8 : \ + (n) & (1ULL << 7) ? 7 : \ + (n) & (1ULL << 6) ? 6 : \ + (n) & (1ULL << 5) ? 5 : \ + (n) & (1ULL << 4) ? 4 : \ + (n) & (1ULL << 3) ? 3 : \ + (n) & (1ULL << 2) ? 2 : \ + (n) & (1ULL << 1) ? 1 : \ + (n) & (1ULL << 0) ? 0 : \ + ____ilog2_NaN() \ + ) : \ + (sizeof(n) <= 4) ? \ + __ilog2_u32(n) : \ + __ilog2_u64(n) \ + ) #endif /* _LINUX_LOG2_H_ */ Index: stable/9/sys/ofed/include/linux/math64.h =================================================================== --- stable/9/sys/ofed/include/linux/math64.h (nonexistent) +++ stable/9/sys/ofed/include/linux/math64.h (revision 273864) @@ -0,0 +1,133 @@ +/*- + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2014 Mellanox Technologies, Ltd. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LINUX_MATH64_H +#define _LINUX_MATH64_H + +#include +#include + +#if BITS_PER_LONG == 64 + +# define do_div(n, base) ({ \ + uint32_t __base = (base); \ + uint32_t __rem; \ + __rem = ((uint64_t)(n)) % __base; \ + (n) = ((uint64_t)(n)) / __base; \ + __rem; \ +}) + +/** +* div_u64_rem - unsigned 64bit divide with 32bit divisor with remainder +* +* This is commonly provided by 32bit archs to provide an optimized 64bit +* divide. +*/ +static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) +{ + *remainder = dividend % divisor; + return dividend / divisor; +} + + +#elif BITS_PER_LONG == 32 + +static uint32_t __div64_32(uint64_t *n, uint32_t base) +{ + uint64_t rem = *n; + uint64_t b = base; + uint64_t res, d = 1; + uint32_t high = rem >> 32; + + /* Reduce the thing a bit first */ + res = 0; + if (high >= base) { + high /= base; + res = (uint64_t) high << 32; + rem -= (uint64_t) (high*base) << 32; + } + + while ((int64_t)b > 0 && b < rem) { + b = b+b; + d = d+d; + } + + do { + if (rem >= b) { + rem -= b; + res += d; + } + b >>= 1; + d >>= 1; + } while (d); + + *n = res; + return rem; +} + +# define do_div(n, base) ({ \ + uint32_t __base = (base); \ + uint32_t __rem; \ + (void)(((typeof((n)) *)0) == ((uint64_t *)0)); \ + if (likely(((n) >> 32) == 0)) { \ + __rem = (uint32_t)(n) % __base; \ + (n) = (uint32_t)(n) / __base; \ + } else \ + __rem = __div64_32(&(n), __base); \ + __rem; \ +}) + +#ifndef div_u64_rem +static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) +{ + *remainder = do_div(dividend, divisor); + return dividend; +} +#endif + + +#endif /* BITS_PER_LONG */ + + + +/** + ** div_u64 - unsigned 64bit divide with 32bit divisor + ** + ** This is the most common 64bit divide and should be used if possible, + ** as many 32bit archs can optimize this variant better than a full 64bit + ** divide. + * */ +#ifndef div_u64 + +static inline u64 div_u64(u64 dividend, u32 divisor) +{ + u32 remainder; + return div_u64_rem(dividend, divisor, &remainder); +} +#endif + +#endif /* _LINUX_MATH64_H */ Property changes on: stable/9/sys/ofed/include/linux/math64.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/include/linux/miscdevice.h =================================================================== --- stable/9/sys/ofed/include/linux/miscdevice.h (revision 273863) +++ stable/9/sys/ofed/include/linux/miscdevice.h (revision 273864) @@ -1,72 +1,75 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_MISCDEVICE_H_ #define _LINUX_MISCDEVICE_H_ #define MISC_DYNAMIC_MINOR -1 #include #include struct miscdevice { const char *name; struct device *this_device; const struct file_operations *fops; struct cdev *cdev; int minor; + const char *nodename; + umode_t mode; }; extern struct class miscclass; static inline int misc_register(struct miscdevice *misc) { misc->this_device = device_create(&miscclass, &linux_rootdev, 0, misc, misc->name); misc->cdev = cdev_alloc(); if (misc->cdev == NULL) return -ENOMEM; misc->cdev->owner = THIS_MODULE; misc->cdev->ops = misc->fops; kobject_set_name(&misc->cdev->kobj, misc->name); if (cdev_add(misc->cdev, misc->this_device->devt, 1)) return -EINVAL; return (0); } static inline int misc_deregister(struct miscdevice *misc) { device_destroy(&miscclass, misc->this_device->devt); cdev_del(misc->cdev); return (0); } #endif /* _LINUX_MISCDEVICE_H_ */ Index: stable/9/sys/ofed/include/linux/mlx4/cmd.h =================================================================== --- stable/9/sys/ofed/include/linux/mlx4/cmd.h (revision 273863) +++ stable/9/sys/ofed/include/linux/mlx4/cmd.h (revision 273864) @@ -1,196 +1,258 @@ /* * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef MLX4_CMD_H #define MLX4_CMD_H #include +#include enum { /* initialization and general commands */ MLX4_CMD_SYS_EN = 0x1, MLX4_CMD_SYS_DIS = 0x2, MLX4_CMD_MAP_FA = 0xfff, MLX4_CMD_UNMAP_FA = 0xffe, MLX4_CMD_RUN_FW = 0xff6, MLX4_CMD_MOD_STAT_CFG = 0x34, MLX4_CMD_QUERY_DEV_CAP = 0x3, MLX4_CMD_QUERY_FW = 0x4, MLX4_CMD_ENABLE_LAM = 0xff8, MLX4_CMD_DISABLE_LAM = 0xff7, MLX4_CMD_QUERY_DDR = 0x5, MLX4_CMD_QUERY_ADAPTER = 0x6, MLX4_CMD_INIT_HCA = 0x7, MLX4_CMD_CLOSE_HCA = 0x8, MLX4_CMD_INIT_PORT = 0x9, MLX4_CMD_CLOSE_PORT = 0xa, MLX4_CMD_QUERY_HCA = 0xb, MLX4_CMD_QUERY_PORT = 0x43, MLX4_CMD_SENSE_PORT = 0x4d, MLX4_CMD_HW_HEALTH_CHECK = 0x50, MLX4_CMD_SET_PORT = 0xc, MLX4_CMD_SET_NODE = 0x5a, + MLX4_CMD_QUERY_FUNC = 0x56, MLX4_CMD_ACCESS_DDR = 0x2e, MLX4_CMD_MAP_ICM = 0xffa, MLX4_CMD_UNMAP_ICM = 0xff9, MLX4_CMD_MAP_ICM_AUX = 0xffc, MLX4_CMD_UNMAP_ICM_AUX = 0xffb, MLX4_CMD_SET_ICM_SIZE = 0xffd, + /*master notify fw on finish for slave's flr*/ + MLX4_CMD_INFORM_FLR_DONE = 0x5b, + MLX4_CMD_GET_OP_REQ = 0x59, /* TPT commands */ MLX4_CMD_SW2HW_MPT = 0xd, MLX4_CMD_QUERY_MPT = 0xe, MLX4_CMD_HW2SW_MPT = 0xf, MLX4_CMD_READ_MTT = 0x10, MLX4_CMD_WRITE_MTT = 0x11, MLX4_CMD_SYNC_TPT = 0x2f, /* EQ commands */ MLX4_CMD_MAP_EQ = 0x12, MLX4_CMD_SW2HW_EQ = 0x13, MLX4_CMD_HW2SW_EQ = 0x14, MLX4_CMD_QUERY_EQ = 0x15, /* CQ commands */ MLX4_CMD_SW2HW_CQ = 0x16, MLX4_CMD_HW2SW_CQ = 0x17, MLX4_CMD_QUERY_CQ = 0x18, MLX4_CMD_MODIFY_CQ = 0x2c, /* SRQ commands */ MLX4_CMD_SW2HW_SRQ = 0x35, MLX4_CMD_HW2SW_SRQ = 0x36, MLX4_CMD_QUERY_SRQ = 0x37, MLX4_CMD_ARM_SRQ = 0x40, /* QP/EE commands */ MLX4_CMD_RST2INIT_QP = 0x19, MLX4_CMD_INIT2RTR_QP = 0x1a, MLX4_CMD_RTR2RTS_QP = 0x1b, MLX4_CMD_RTS2RTS_QP = 0x1c, MLX4_CMD_SQERR2RTS_QP = 0x1d, MLX4_CMD_2ERR_QP = 0x1e, MLX4_CMD_RTS2SQD_QP = 0x1f, MLX4_CMD_SQD2SQD_QP = 0x38, MLX4_CMD_SQD2RTS_QP = 0x20, MLX4_CMD_2RST_QP = 0x21, MLX4_CMD_QUERY_QP = 0x22, MLX4_CMD_INIT2INIT_QP = 0x2d, MLX4_CMD_SUSPEND_QP = 0x32, MLX4_CMD_UNSUSPEND_QP = 0x33, + MLX4_CMD_UPDATE_QP = 0x61, /* special QP and management commands */ MLX4_CMD_CONF_SPECIAL_QP = 0x23, MLX4_CMD_MAD_IFC = 0x24, /* multicast commands */ MLX4_CMD_READ_MCG = 0x25, MLX4_CMD_WRITE_MCG = 0x26, MLX4_CMD_MGID_HASH = 0x27, /* miscellaneous commands */ MLX4_CMD_DIAG_RPRT = 0x30, MLX4_CMD_NOP = 0x31, + MLX4_CMD_ACCESS_MEM = 0x2e, + MLX4_CMD_SET_VEP = 0x52, + /* Ethernet specific commands */ + MLX4_CMD_SET_VLAN_FLTR = 0x47, + MLX4_CMD_SET_MCAST_FLTR = 0x48, + MLX4_CMD_DUMP_ETH_STATS = 0x49, + + /* Communication channel commands */ + MLX4_CMD_ARM_COMM_CHANNEL = 0x57, + MLX4_CMD_GEN_EQE = 0x58, + + /* virtual commands */ + MLX4_CMD_ALLOC_RES = 0xf00, + MLX4_CMD_FREE_RES = 0xf01, + MLX4_CMD_MCAST_ATTACH = 0xf05, + MLX4_CMD_UCAST_ATTACH = 0xf06, + MLX4_CMD_PROMISC = 0xf08, + MLX4_CMD_QUERY_FUNC_CAP = 0xf0a, + MLX4_CMD_QP_ATTACH = 0xf0b, + /* debug commands */ MLX4_CMD_QUERY_DEBUG_MSG = 0x2a, MLX4_CMD_SET_DEBUG_MSG = 0x2b, /* statistics commands */ MLX4_CMD_QUERY_IF_STAT = 0X54, MLX4_CMD_SET_IF_STAT = 0X55, + + /* register/delete flow steering network rules */ + MLX4_QP_FLOW_STEERING_ATTACH = 0x65, + MLX4_QP_FLOW_STEERING_DETACH = 0x66, + MLX4_FLOW_STEERING_IB_UC_QP_RANGE = 0x64, }; enum { - MLX4_CMD_TIME_CLASS_A = 10000, - MLX4_CMD_TIME_CLASS_B = 10000, - MLX4_CMD_TIME_CLASS_C = 10000, + MLX4_CMD_TIME_CLASS_A = 60000, + MLX4_CMD_TIME_CLASS_B = 60000, + MLX4_CMD_TIME_CLASS_C = 60000, }; enum { - MLX4_MAILBOX_SIZE = 4096 + MLX4_MAILBOX_SIZE = 4096, + MLX4_ACCESS_MEM_ALIGN = 256, }; enum { /* set port opcode modifiers */ - MLX4_SET_PORT_GENERAL = 0x0, - MLX4_SET_PORT_RQP_CALC = 0x1, - MLX4_SET_PORT_MAC_TABLE = 0x2, - MLX4_SET_PORT_VLAN_TABLE = 0x3, - MLX4_SET_PORT_PRIO_MAP = 0x4, - MLX4_SET_PORT_GID_TABLE = 0x5, + MLX4_SET_PORT_GENERAL = 0x0, + MLX4_SET_PORT_RQP_CALC = 0x1, + MLX4_SET_PORT_MAC_TABLE = 0x2, + MLX4_SET_PORT_VLAN_TABLE = 0x3, + MLX4_SET_PORT_PRIO_MAP = 0x4, + MLX4_SET_PORT_GID_TABLE = 0x5, + MLX4_SET_PORT_PRIO2TC = 0x8, + MLX4_SET_PORT_SCHEDULER = 0x9 }; +enum { + MLX4_CMD_WRAPPED, + MLX4_CMD_NATIVE +}; + struct mlx4_dev; struct mlx4_cmd_mailbox { void *buf; dma_addr_t dma; }; int __mlx4_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param, int out_is_imm, u32 in_modifier, u8 op_modifier, - u16 op, unsigned long timeout); + u16 op, unsigned long timeout, int native); /* Invoke a command with no output parameter */ static inline int mlx4_cmd(struct mlx4_dev *dev, u64 in_param, u32 in_modifier, - u8 op_modifier, u16 op, unsigned long timeout) + u8 op_modifier, u16 op, unsigned long timeout, + int native) { return __mlx4_cmd(dev, in_param, NULL, 0, in_modifier, - op_modifier, op, timeout); + op_modifier, op, timeout, native); } /* Invoke a command with an output mailbox */ static inline int mlx4_cmd_box(struct mlx4_dev *dev, u64 in_param, u64 out_param, u32 in_modifier, u8 op_modifier, u16 op, - unsigned long timeout) + unsigned long timeout, int native) { return __mlx4_cmd(dev, in_param, &out_param, 0, in_modifier, - op_modifier, op, timeout); + op_modifier, op, timeout, native); } /* * Invoke a command with an immediate output parameter (and copy the * output into the caller's out_param pointer after the command * executes). */ static inline int mlx4_cmd_imm(struct mlx4_dev *dev, u64 in_param, u64 *out_param, u32 in_modifier, u8 op_modifier, u16 op, - unsigned long timeout) + unsigned long timeout, int native) { return __mlx4_cmd(dev, in_param, out_param, 1, in_modifier, - op_modifier, op, timeout); + op_modifier, op, timeout, native); } struct mlx4_cmd_mailbox *mlx4_alloc_cmd_mailbox(struct mlx4_dev *dev); void mlx4_free_cmd_mailbox(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox); + +u32 mlx4_comm_get_version(void); +int mlx4_set_vf_mac(struct mlx4_dev *dev, int port, int vf, u8 *mac); +int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, u8 qos); +int mlx4_set_vf_spoofchk(struct mlx4_dev *dev, int port, int vf, bool setting); +int mlx4_set_vf_link_state(struct mlx4_dev *dev, int port, int vf, int link_state); +int mlx4_get_vf_link_state(struct mlx4_dev *dev, int port, int vf); +/* + * mlx4_get_slave_default_vlan - + * retrun true if VST ( default vlan) + * if VST will fill vlan & qos (if not NULL) + */ +bool mlx4_get_slave_default_vlan(struct mlx4_dev *dev, int port, int slave, u16 *vlan, u8 *qos); + +enum { + IFLA_VF_LINK_STATE_AUTO, /* link state of the uplink */ + IFLA_VF_LINK_STATE_ENABLE, /* link always up */ + IFLA_VF_LINK_STATE_DISABLE, /* link always down */ + __IFLA_VF_LINK_STATE_MAX, +}; + +#define MLX4_COMM_GET_IF_REV(cmd_chan_ver) (u8)((cmd_chan_ver) >> 8) #endif /* MLX4_CMD_H */ Index: stable/9/sys/ofed/include/linux/mlx4/cq.h =================================================================== --- stable/9/sys/ofed/include/linux/mlx4/cq.h (revision 273863) +++ stable/9/sys/ofed/include/linux/mlx4/cq.h (revision 273864) @@ -1,150 +1,181 @@ /* * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef MLX4_CQ_H #define MLX4_CQ_H #include #include #include struct mlx4_cqe { __be32 vlan_my_qpn; __be32 immed_rss_invalid; __be32 g_mlpath_rqpn; - __be16 sl_vid; - __be16 rlid; - __be16 status; - u8 ipv6_ext_mask; - u8 badfcs_enc; + union { + struct { + union { + struct { + __be16 sl_vid; + __be16 rlid; + }; + __be32 timestamp_16_47; + }; + __be16 status; + u8 ipv6_ext_mask; + u8 badfcs_enc; + }; + struct { + __be16 reserved1; + u8 smac[6]; + }; + }; __be32 byte_cnt; __be16 wqe_index; __be16 checksum; - u8 reserved[3]; + u8 reserved2[1]; + __be16 timestamp_0_15; u8 owner_sr_opcode; -}; +} __packed; struct mlx4_err_cqe { __be32 my_qpn; u32 reserved1[5]; __be16 wqe_index; u8 vendor_err_syndrome; u8 syndrome; u8 reserved2[3]; u8 owner_sr_opcode; }; +struct mlx4_ts_cqe { + __be32 vlan_my_qpn; + __be32 immed_rss_invalid; + __be32 g_mlpath_rqpn; + __be32 timestamp_hi; + __be16 status; + u8 ipv6_ext_mask; + u8 badfcs_enc; + __be32 byte_cnt; + __be16 wqe_index; + __be16 checksum; + u8 reserved; + __be16 timestamp_lo; + u8 owner_sr_opcode; +} __packed; + enum { MLX4_CQE_VLAN_PRESENT_MASK = 1 << 29, MLX4_CQE_QPN_MASK = 0xffffff, + MLX4_CQE_VID_MASK = 0xfff, }; enum { MLX4_CQE_OWNER_MASK = 0x80, MLX4_CQE_IS_SEND_MASK = 0x40, MLX4_CQE_OPCODE_MASK = 0x1f }; enum { MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR = 0x01, MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR = 0x02, MLX4_CQE_SYNDROME_LOCAL_PROT_ERR = 0x04, MLX4_CQE_SYNDROME_WR_FLUSH_ERR = 0x05, MLX4_CQE_SYNDROME_MW_BIND_ERR = 0x06, MLX4_CQE_SYNDROME_BAD_RESP_ERR = 0x10, MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR = 0x11, MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR = 0x12, MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR = 0x13, MLX4_CQE_SYNDROME_REMOTE_OP_ERR = 0x14, MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR = 0x15, MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR = 0x16, MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR = 0x22, }; enum { MLX4_CQE_STATUS_IPV4 = 1 << 6, MLX4_CQE_STATUS_IPV4F = 1 << 7, MLX4_CQE_STATUS_IPV6 = 1 << 8, MLX4_CQE_STATUS_IPV4OPT = 1 << 9, MLX4_CQE_STATUS_TCP = 1 << 10, MLX4_CQE_STATUS_UDP = 1 << 11, MLX4_CQE_STATUS_IPOK = 1 << 12, }; enum { MLX4_CQE_LLC = 1, MLX4_CQE_SNAP = 1 << 1, MLX4_CQE_BAD_FCS = 1 << 4, }; static inline void mlx4_cq_arm(struct mlx4_cq *cq, u32 cmd, void __iomem *uar_page, spinlock_t *doorbell_lock) { __be32 doorbell[2]; u32 sn; u32 ci; sn = cq->arm_sn & 3; ci = cq->cons_index & 0xffffff; *cq->arm_db = cpu_to_be32(sn << 28 | cmd | ci); /* * Make sure that the doorbell record in host memory is * written before ringing the doorbell via PCI MMIO. */ wmb(); doorbell[0] = cpu_to_be32(sn << 28 | cmd | cq->cqn); doorbell[1] = cpu_to_be32(ci); mlx4_write64(doorbell, uar_page + MLX4_CQ_DOORBELL, doorbell_lock); } static inline void mlx4_cq_set_ci(struct mlx4_cq *cq) { *cq->set_ci_db = cpu_to_be32(cq->cons_index & 0xffffff); } enum { MLX4_CQ_DB_REQ_NOT_SOL = 1 << 24, MLX4_CQ_DB_REQ_NOT = 2 << 24 }; int mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq, u16 count, u16 period); int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq, int entries, struct mlx4_mtt *mtt); - +int mlx4_cq_ignore_overrun(struct mlx4_dev *dev, struct mlx4_cq *cq); #endif /* MLX4_CQ_H */ Index: stable/9/sys/ofed/include/linux/mlx4/device.h =================================================================== --- stable/9/sys/ofed/include/linux/mlx4/device.h (revision 273863) +++ stable/9/sys/ofed/include/linux/mlx4/device.h (revision 273864) @@ -1,623 +1,1284 @@ /* * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef MLX4_DEVICE_H #define MLX4_DEVICE_H #include #include #include - +#include +#include +#include #include -#include +#include +#define MAX_MSIX_P_PORT 17 +#define MAX_MSIX 64 +#define MSIX_LEGACY_SZ 4 +#define MIN_MSIX_P_PORT 5 + +#define MLX4_ROCE_MAX_GIDS 128 +#define MLX4_ROCE_PF_GIDS 16 + +#define MLX4_NUM_UP 8 +#define MLX4_NUM_TC 8 +#define MLX4_MAX_100M_UNITS_VAL 255 /* + * work around: can't set values + * greater then this value when + * using 100 Mbps units. + */ +#define MLX4_RATELIMIT_100M_UNITS 3 /* 100 Mbps */ +#define MLX4_RATELIMIT_1G_UNITS 4 /* 1 Gbps */ +#define MLX4_RATELIMIT_DEFAULT 0x00ff + +#define CORE_CLOCK_MASK 0xffffffffffffULL + enum { MLX4_FLAG_MSI_X = 1 << 0, MLX4_FLAG_OLD_PORT_CMDS = 1 << 1, + MLX4_FLAG_MASTER = 1 << 2, + MLX4_FLAG_SLAVE = 1 << 3, + MLX4_FLAG_SRIOV = 1 << 4, + MLX4_FLAG_DEV_NUM_STR = 1 << 5, + MLX4_FLAG_OLD_REG_MAC = 1 << 6, }; enum { - MLX4_MAX_PORTS = 2 + MLX4_PORT_CAP_IS_SM = 1 << 1, + MLX4_PORT_CAP_DEV_MGMT_SUP = 1 << 19, }; enum { - MLX4_BOARD_ID_LEN = 64 + MLX4_MAX_PORTS = 2, + MLX4_MAX_PORT_PKEYS = 128 }; +/* base qkey for use in sriov tunnel-qp/proxy-qp communication. + * These qkeys must not be allowed for general use. This is a 64k range, + * and to test for violation, we use the mask (protect against future chg). + */ +#define MLX4_RESERVED_QKEY_BASE (0xFFFF0000) +#define MLX4_RESERVED_QKEY_MASK (0xFFFF0000) + enum { - MLX4_DEV_CAP_FLAG_RC = 1 << 0, - MLX4_DEV_CAP_FLAG_UC = 1 << 1, - MLX4_DEV_CAP_FLAG_UD = 1 << 2, - MLX4_DEV_CAP_FLAG_XRC = 1 << 3, - MLX4_DEV_CAP_FLAG_SRQ = 1 << 6, - MLX4_DEV_CAP_FLAG_IPOIB_CSUM = 1 << 7, - MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR = 1 << 8, - MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR = 1 << 9, - MLX4_DEV_CAP_FLAG_DPDP = 1 << 12, - MLX4_DEV_CAP_FLAG_RAW_ETY = 1 << 13, - MLX4_DEV_CAP_FLAG_BLH = 1 << 15, - MLX4_DEV_CAP_FLAG_MEM_WINDOW = 1 << 16, - MLX4_DEV_CAP_FLAG_APM = 1 << 17, - MLX4_DEV_CAP_FLAG_ATOMIC = 1 << 18, - MLX4_DEV_CAP_FLAG_RAW_MCAST = 1 << 19, - MLX4_DEV_CAP_FLAG_UD_AV_PORT = 1 << 20, - MLX4_DEV_CAP_FLAG_UD_MCAST = 1 << 21, - MLX4_DEV_CAP_FLAG_IBOE = 1 << 30, - MLX4_DEV_CAP_FLAG_FC_T11 = 1 << 31 + MLX4_BOARD_ID_LEN = 64, + MLX4_VSD_LEN = 208 }; enum { + MLX4_MAX_NUM_PF = 16, + MLX4_MAX_NUM_VF = 64, + MLX4_MFUNC_MAX = 80, + MLX4_MAX_EQ_NUM = 1024, + MLX4_MFUNC_EQ_NUM = 4, + MLX4_MFUNC_MAX_EQES = 8, + MLX4_MFUNC_EQE_MASK = (MLX4_MFUNC_MAX_EQES - 1) +}; + +/* Driver supports 3 diffrent device methods to manage traffic steering: + * -device managed - High level API for ib and eth flow steering. FW is + * managing flow steering tables. + * - B0 steering mode - Common low level API for ib and (if supported) eth. + * - A0 steering mode - Limited low level API for eth. In case of IB, + * B0 mode is in use. + */ +enum { + MLX4_STEERING_MODE_A0, + MLX4_STEERING_MODE_B0, + MLX4_STEERING_MODE_DEVICE_MANAGED +}; + +static inline const char *mlx4_steering_mode_str(int steering_mode) +{ + switch (steering_mode) { + case MLX4_STEERING_MODE_A0: + return "A0 steering"; + + case MLX4_STEERING_MODE_B0: + return "B0 steering"; + + case MLX4_STEERING_MODE_DEVICE_MANAGED: + return "Device managed flow steering"; + + default: + return "Unrecognize steering mode"; + } +} + +enum { + MLX4_DEV_CAP_FLAG_RC = 1LL << 0, + MLX4_DEV_CAP_FLAG_UC = 1LL << 1, + MLX4_DEV_CAP_FLAG_UD = 1LL << 2, + MLX4_DEV_CAP_FLAG_XRC = 1LL << 3, + MLX4_DEV_CAP_FLAG_SRQ = 1LL << 6, + MLX4_DEV_CAP_FLAG_IPOIB_CSUM = 1LL << 7, + MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR = 1LL << 8, + MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR = 1LL << 9, + MLX4_DEV_CAP_FLAG_DPDP = 1LL << 12, + MLX4_DEV_CAP_FLAG_BLH = 1LL << 15, + MLX4_DEV_CAP_FLAG_MEM_WINDOW = 1LL << 16, + MLX4_DEV_CAP_FLAG_APM = 1LL << 17, + MLX4_DEV_CAP_FLAG_ATOMIC = 1LL << 18, + MLX4_DEV_CAP_FLAG_RAW_MCAST = 1LL << 19, + MLX4_DEV_CAP_FLAG_UD_AV_PORT = 1LL << 20, + MLX4_DEV_CAP_FLAG_UD_MCAST = 1LL << 21, + MLX4_DEV_CAP_FLAG_IBOE = 1LL << 30, + MLX4_DEV_CAP_FLAG_UC_LOOPBACK = 1LL << 32, + MLX4_DEV_CAP_FLAG_FCS_KEEP = 1LL << 34, + MLX4_DEV_CAP_FLAG_WOL_PORT1 = 1LL << 37, + MLX4_DEV_CAP_FLAG_WOL_PORT2 = 1LL << 38, + MLX4_DEV_CAP_FLAG_UDP_RSS = 1LL << 40, + MLX4_DEV_CAP_FLAG_VEP_UC_STEER = 1LL << 41, + MLX4_DEV_CAP_FLAG_VEP_MC_STEER = 1LL << 42, + MLX4_DEV_CAP_FLAG_CROSS_CHANNEL = 1LL << 44, + MLX4_DEV_CAP_FLAG_COUNTERS = 1LL << 48, + MLX4_DEV_CAP_FLAG_COUNTERS_EXT = 1LL << 49, + MLX4_DEV_CAP_FLAG_SET_PORT_ETH_SCHED = 1LL << 53, + MLX4_DEV_CAP_FLAG_SENSE_SUPPORT = 1LL << 55, + MLX4_DEV_CAP_FLAG_FAST_DROP = 1LL << 57, + MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV = 1LL << 59, + MLX4_DEV_CAP_FLAG_64B_EQE = 1LL << 61, + MLX4_DEV_CAP_FLAG_64B_CQE = 1LL << 62 +}; + +enum { + MLX4_DEV_CAP_FLAG2_RSS = 1LL << 0, + MLX4_DEV_CAP_FLAG2_RSS_TOP = 1LL << 1, + MLX4_DEV_CAP_FLAG2_RSS_XOR = 1LL << 2, + MLX4_DEV_CAP_FLAG2_FS_EN = 1LL << 3, + MLX4_DEV_CAP_FLAG2_FSM = 1LL << 4, + MLX4_DEV_CAP_FLAG2_VLAN_CONTROL = 1LL << 5, + MLX4_DEV_CAP_FLAG2_UPDATE_QP = 1LL << 6, + MLX4_DEV_CAP_FLAG2_LB_SRC_CHK = 1LL << 7, + MLX4_DEV_CAP_FLAG2_DMFS_IPOIB = 1LL << 8, + MLX4_DEV_CAP_FLAG2_ETS_CFG = 1LL << 9, + MLX4_DEV_CAP_FLAG2_ETH_BACKPL_AN_REP = 1LL << 10, + MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN = 1LL << 11, + MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT = 1LL << 12, + MLX4_DEV_CAP_FLAG2_TS = 1LL << 13, + MLX4_DEV_CAP_FLAG2_DRIVER_VERSION_TO_FW = 1LL << 14 +}; + +/* bit enums for an 8-bit flags field indicating special use + * QPs which require special handling in qp_reserve_range. + * Currently, this only includes QPs used by the ETH interface, + * where we expect to use blueflame. These QPs must not have + * bits 6 and 7 set in their qp number. + * + * This enum may use only bits 0..7. + */ +enum { + MLX4_RESERVE_BF_QP = 1 << 7, +}; + +enum { + MLX4_DEV_CAP_CQ_FLAG_IO = 1 << 0 +}; + +enum { + MLX4_DEV_CAP_64B_EQE_ENABLED = 1LL << 0, + MLX4_DEV_CAP_64B_CQE_ENABLED = 1LL << 1 +}; + +enum { + MLX4_USER_DEV_CAP_64B_CQE = 1L << 0 +}; + +enum { + MLX4_FUNC_CAP_64B_EQE_CQE = 1L << 0 +}; + + +#define MLX4_ATTR_EXTENDED_PORT_INFO cpu_to_be16(0xff90) + +enum { + MLX4_BMME_FLAG_WIN_TYPE_2B = 1 << 1, MLX4_BMME_FLAG_LOCAL_INV = 1 << 6, MLX4_BMME_FLAG_REMOTE_INV = 1 << 7, MLX4_BMME_FLAG_TYPE_2_WIN = 1 << 9, MLX4_BMME_FLAG_RESERVED_LKEY = 1 << 10, MLX4_BMME_FLAG_FAST_REG_WR = 1 << 11, }; enum mlx4_event { MLX4_EVENT_TYPE_COMP = 0x00, MLX4_EVENT_TYPE_PATH_MIG = 0x01, MLX4_EVENT_TYPE_COMM_EST = 0x02, MLX4_EVENT_TYPE_SQ_DRAINED = 0x03, MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE = 0x13, MLX4_EVENT_TYPE_SRQ_LIMIT = 0x14, MLX4_EVENT_TYPE_CQ_ERROR = 0x04, MLX4_EVENT_TYPE_WQ_CATAS_ERROR = 0x05, MLX4_EVENT_TYPE_EEC_CATAS_ERROR = 0x06, MLX4_EVENT_TYPE_PATH_MIG_FAILED = 0x07, MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10, MLX4_EVENT_TYPE_WQ_ACCESS_ERROR = 0x11, MLX4_EVENT_TYPE_SRQ_CATAS_ERROR = 0x12, MLX4_EVENT_TYPE_LOCAL_CATAS_ERROR = 0x08, MLX4_EVENT_TYPE_PORT_CHANGE = 0x09, MLX4_EVENT_TYPE_EQ_OVERFLOW = 0x0f, MLX4_EVENT_TYPE_ECC_DETECT = 0x0e, - MLX4_EVENT_TYPE_CMD = 0x0a + MLX4_EVENT_TYPE_CMD = 0x0a, + MLX4_EVENT_TYPE_VEP_UPDATE = 0x19, + MLX4_EVENT_TYPE_COMM_CHANNEL = 0x18, + MLX4_EVENT_TYPE_OP_REQUIRED = 0x1a, + MLX4_EVENT_TYPE_FATAL_WARNING = 0x1b, + MLX4_EVENT_TYPE_FLR_EVENT = 0x1c, + MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT = 0x1d, + MLX4_EVENT_TYPE_RECOVERABLE_ERROR_EVENT = 0x3e, + MLX4_EVENT_TYPE_NONE = 0xff, }; enum { MLX4_PORT_CHANGE_SUBTYPE_DOWN = 1, MLX4_PORT_CHANGE_SUBTYPE_ACTIVE = 4 }; enum { + MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_BAD_CABLE = 1, + MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_UNSUPPORTED_CABLE = 2, +}; + +enum { + MLX4_FATAL_WARNING_SUBTYPE_WARMING = 0, +}; + +enum slave_port_state { + SLAVE_PORT_DOWN = 0, + SLAVE_PENDING_UP, + SLAVE_PORT_UP, +}; + +enum slave_port_gen_event { + SLAVE_PORT_GEN_EVENT_DOWN = 0, + SLAVE_PORT_GEN_EVENT_UP, + SLAVE_PORT_GEN_EVENT_NONE, +}; + +enum slave_port_state_event { + MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN, + MLX4_PORT_STATE_DEV_EVENT_PORT_UP, + MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID, + MLX4_PORT_STATE_IB_EVENT_GID_INVALID, +}; + +enum { MLX4_PERM_LOCAL_READ = 1 << 10, MLX4_PERM_LOCAL_WRITE = 1 << 11, MLX4_PERM_REMOTE_READ = 1 << 12, MLX4_PERM_REMOTE_WRITE = 1 << 13, - MLX4_PERM_ATOMIC = 1 << 14 + MLX4_PERM_ATOMIC = 1 << 14, + MLX4_PERM_BIND_MW = 1 << 15, }; enum { MLX4_OPCODE_NOP = 0x00, MLX4_OPCODE_SEND_INVAL = 0x01, MLX4_OPCODE_RDMA_WRITE = 0x08, MLX4_OPCODE_RDMA_WRITE_IMM = 0x09, MLX4_OPCODE_SEND = 0x0a, MLX4_OPCODE_SEND_IMM = 0x0b, MLX4_OPCODE_LSO = 0x0e, - MLX4_OPCODE_BIG_LSO = 0x2e, MLX4_OPCODE_RDMA_READ = 0x10, MLX4_OPCODE_ATOMIC_CS = 0x11, MLX4_OPCODE_ATOMIC_FA = 0x12, MLX4_OPCODE_MASKED_ATOMIC_CS = 0x14, MLX4_OPCODE_MASKED_ATOMIC_FA = 0x15, MLX4_OPCODE_BIND_MW = 0x18, MLX4_OPCODE_FMR = 0x19, MLX4_OPCODE_LOCAL_INVAL = 0x1b, MLX4_OPCODE_CONFIG_CMD = 0x1f, MLX4_RECV_OPCODE_RDMA_WRITE_IMM = 0x00, MLX4_RECV_OPCODE_SEND = 0x01, MLX4_RECV_OPCODE_SEND_IMM = 0x02, MLX4_RECV_OPCODE_SEND_INVAL = 0x03, MLX4_CQE_OPCODE_ERROR = 0x1e, MLX4_CQE_OPCODE_RESIZE = 0x16, }; enum { MLX4_STAT_RATE_OFFSET = 5 }; +enum mlx4_protocol { + MLX4_PROT_IB_IPV6 = 0, + MLX4_PROT_ETH, + MLX4_PROT_IB_IPV4, + MLX4_PROT_FCOE +}; + enum { MLX4_MTT_FLAG_PRESENT = 1 }; +enum { + MLX4_MAX_MTT_SHIFT = 31 +}; + enum mlx4_qp_region { MLX4_QP_REGION_FW = 0, MLX4_QP_REGION_ETH_ADDR, MLX4_QP_REGION_FC_ADDR, + MLX4_QP_REGION_FC_EXCH, MLX4_NUM_QP_REGION }; enum mlx4_port_type { MLX4_PORT_TYPE_NONE = 0, MLX4_PORT_TYPE_IB = 1, MLX4_PORT_TYPE_ETH = 2, - MLX4_PORT_TYPE_AUTO = 3 + MLX4_PORT_TYPE_AUTO = 3, + MLX4_PORT_TYPE_NA = 4 }; enum mlx4_special_vlan_idx { MLX4_NO_VLAN_IDX = 0, MLX4_VLAN_MISS_IDX, MLX4_VLAN_REGULAR }; -#define MLX4_LEAST_ATTACHED_VECTOR 0xffffffff +enum mlx4_steer_type { + MLX4_MC_STEER = 0, + MLX4_UC_STEER, + MLX4_NUM_STEERS +}; + enum { - MLX4_CUNTERS_DISABLED, - MLX4_CUNTERS_BASIC, - MLX4_CUNTERS_EXT + MLX4_NUM_FEXCH = 64 * 1024, }; enum { - MAX_FAST_REG_PAGES = 511, + MLX4_MAX_FAST_REG_PAGES = 511, }; +enum { + MLX4_DEV_PMC_SUBTYPE_GUID_INFO = 0x14, + MLX4_DEV_PMC_SUBTYPE_PORT_INFO = 0x15, + MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE = 0x16, +}; + +/* Port mgmt change event handling */ +enum { + MLX4_EQ_PORT_INFO_MSTR_SM_LID_CHANGE_MASK = 1 << 0, + MLX4_EQ_PORT_INFO_GID_PFX_CHANGE_MASK = 1 << 1, + MLX4_EQ_PORT_INFO_LID_CHANGE_MASK = 1 << 2, + MLX4_EQ_PORT_INFO_CLIENT_REREG_MASK = 1 << 3, + MLX4_EQ_PORT_INFO_MSTR_SM_SL_CHANGE_MASK = 1 << 4, +}; + +#define MSTR_SM_CHANGE_MASK (MLX4_EQ_PORT_INFO_MSTR_SM_SL_CHANGE_MASK | \ + MLX4_EQ_PORT_INFO_MSTR_SM_LID_CHANGE_MASK) + static inline u64 mlx4_fw_ver(u64 major, u64 minor, u64 subminor) { return (major << 32) | (minor << 16) | subminor; } +struct mlx4_phys_caps { + u32 gid_phys_table_len[MLX4_MAX_PORTS + 1]; + u32 pkey_phys_table_len[MLX4_MAX_PORTS + 1]; + u32 num_phys_eqs; + u32 base_sqpn; + u32 base_proxy_sqpn; + u32 base_tunnel_sqpn; +}; + struct mlx4_caps { u64 fw_ver; + u32 function; int num_ports; int vl_cap[MLX4_MAX_PORTS + 1]; int ib_mtu_cap[MLX4_MAX_PORTS + 1]; __be32 ib_port_def_cap[MLX4_MAX_PORTS + 1]; u64 def_mac[MLX4_MAX_PORTS + 1]; int eth_mtu_cap[MLX4_MAX_PORTS + 1]; int gid_table_len[MLX4_MAX_PORTS + 1]; int pkey_table_len[MLX4_MAX_PORTS + 1]; int trans_type[MLX4_MAX_PORTS + 1]; int vendor_oui[MLX4_MAX_PORTS + 1]; int wavelength[MLX4_MAX_PORTS + 1]; u64 trans_code[MLX4_MAX_PORTS + 1]; int local_ca_ack_delay; int num_uars; + u32 uar_page_size; int bf_reg_size; int bf_regs_per_page; int max_sq_sg; int max_rq_sg; int num_qps; int max_wqes; int max_sq_desc_sz; int max_rq_desc_sz; int max_qp_init_rdma; int max_qp_dest_rdma; - int sqp_start; + u32 *qp0_proxy; + u32 *qp1_proxy; + u32 *qp0_tunnel; + u32 *qp1_tunnel; int num_srqs; int max_srq_wqes; int max_srq_sge; int reserved_srqs; int num_cqs; int max_cqes; int reserved_cqs; int num_eqs; int reserved_eqs; int num_comp_vectors; + int comp_pool; int num_mpts; - int num_mtt_segs; - int mtts_per_seg; + int max_fmr_maps; + u64 num_mtts; int fmr_reserved_mtts; int reserved_mtts; int reserved_mrws; int reserved_uars; int num_mgms; int num_amgms; int reserved_mcgs; int num_qp_per_mgm; + int steering_mode; int num_pds; int reserved_pds; - int mtt_entry_sz; - int reserved_xrcds; int max_xrcds; + int reserved_xrcds; + int mtt_entry_sz; u32 max_msg_sz; u32 page_size_cap; u64 flags; + u64 flags2; u32 bmme_flags; u32 reserved_lkey; u16 stat_rate_support; - int udp_rss; - int loopback_support; - int wol; + u8 cq_timestamp; u8 port_width_cap[MLX4_MAX_PORTS + 1]; int max_gso_sz; + int max_rss_tbl_sz; int reserved_qps_cnt[MLX4_NUM_QP_REGION]; int reserved_qps; int reserved_qps_base[MLX4_NUM_QP_REGION]; int log_num_macs; int log_num_vlans; - int log_num_prios; enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1]; u8 supported_type[MLX4_MAX_PORTS + 1]; - enum mlx4_port_type port_mask[MLX4_MAX_PORTS + 1]; + u8 suggested_type[MLX4_MAX_PORTS + 1]; + u8 default_sense[MLX4_MAX_PORTS + 1]; + u32 port_mask[MLX4_MAX_PORTS + 1]; enum mlx4_port_type possible_type[MLX4_MAX_PORTS + 1]; - u8 counters_mode; + u32 max_counters; + u8 port_ib_mtu[MLX4_MAX_PORTS + 1]; + u16 sqp_demux; + u32 sync_qp; + u32 cq_flags; + u32 eqe_size; + u32 cqe_size; + u8 eqe_factor; + u32 userspace_caps; /* userspace must be aware to */ + u32 function_caps; /* functions must be aware to */ + u8 fast_drop; + u16 hca_core_clock; u32 max_basic_counters; - u32 max_ext_counters; - u32 mc_promisc_mode; + u32 max_extended_counters; + u8 def_counter_index[MLX4_MAX_PORTS + 1]; }; struct mlx4_buf_list { void *buf; dma_addr_t map; }; struct mlx4_buf { struct mlx4_buf_list direct; struct mlx4_buf_list *page_list; int nbufs; int npages; int page_shift; }; struct mlx4_mtt { - u32 first_seg; + u32 offset; int order; int page_shift; }; enum { MLX4_DB_PER_PAGE = PAGE_SIZE / 4 }; struct mlx4_db_pgdir { struct list_head list; DECLARE_BITMAP(order0, MLX4_DB_PER_PAGE); DECLARE_BITMAP(order1, MLX4_DB_PER_PAGE / 2); unsigned long *bits[2]; __be32 *db_page; dma_addr_t db_dma; }; struct mlx4_ib_user_db_page; struct mlx4_db { __be32 *db; union { struct mlx4_db_pgdir *pgdir; struct mlx4_ib_user_db_page *user_page; } u; dma_addr_t dma; int index; int order; }; struct mlx4_hwq_resources { struct mlx4_db db; struct mlx4_mtt mtt; struct mlx4_buf buf; }; struct mlx4_mr { struct mlx4_mtt mtt; u64 iova; u64 size; u32 key; u32 pd; u32 access; int enabled; }; +enum mlx4_mw_type { + MLX4_MW_TYPE_1 = 1, + MLX4_MW_TYPE_2 = 2, +}; + +struct mlx4_mw { + u32 key; + u32 pd; + enum mlx4_mw_type type; + int enabled; +}; + struct mlx4_fmr { struct mlx4_mr mr; struct mlx4_mpt_entry *mpt; __be64 *mtts; dma_addr_t dma_handle; int max_pages; int max_maps; int maps; u8 page_shift; }; struct mlx4_uar { unsigned long pfn; int index; struct list_head bf_list; unsigned free_bf_bmap; void __iomem *map; void __iomem *bf_map; }; struct mlx4_bf { unsigned long offset; int buf_size; struct mlx4_uar *uar; void __iomem *reg; }; struct mlx4_cq { void (*comp) (struct mlx4_cq *); void (*event) (struct mlx4_cq *, enum mlx4_event); struct mlx4_uar *uar; u32 cons_index; __be32 *set_ci_db; __be32 *arm_db; int arm_sn; int cqn; unsigned vector; atomic_t refcount; struct completion free; + int eqn; + u16 irq; }; struct mlx4_qp { void (*event) (struct mlx4_qp *, enum mlx4_event); int qpn; atomic_t refcount; struct completion free; }; struct mlx4_srq { void (*event) (struct mlx4_srq *, enum mlx4_event); int srqn; int max; int max_gs; int wqe_shift; atomic_t refcount; struct completion free; }; struct mlx4_av { __be32 port_pd; u8 reserved1; u8 g_slid; __be16 dlid; u8 reserved2; u8 gid_index; u8 stat_rate; u8 hop_limit; __be32 sl_tclass_flowlabel; u8 dgid[16]; }; struct mlx4_eth_av { __be32 port_pd; u8 reserved1; u8 smac_idx; u16 reserved2; u8 reserved3; u8 gid_index; u8 stat_rate; u8 hop_limit; __be32 sl_tclass_flowlabel; u8 dgid[16]; - u32 reserved4[2]; + u8 s_mac[6]; + u8 reserved4[2]; __be16 vlan; u8 mac[6]; }; union mlx4_ext_av { struct mlx4_av ib; struct mlx4_eth_av eth; }; -struct mlx4_counters { - __be32 counter_mode; - __be32 num_ifc; - u32 reserved[2]; - __be64 rx_frames; - __be64 rx_bytes; - __be64 tx_frames; - __be64 tx_bytes; +struct mlx4_if_stat_control { + u8 reserved1[3]; + /* Extended counters enabled */ + u8 cnt_mode; + /* Number of interfaces */ + __be32 num_of_if; + __be32 reserved[2]; }; -struct mlx4_counters_ext { - __be32 counter_mode; - __be32 num_ifc; - u32 reserved[2]; - __be64 rx_uni_frames; - __be64 rx_uni_bytes; - __be64 rx_mcast_frames; - __be64 rx_mcast_bytes; - __be64 rx_bcast_frames; - __be64 rx_bcast_bytes; - __be64 rx_nobuf_frames; - __be64 rx_nobuf_bytes; - __be64 rx_err_frames; - __be64 rx_err_bytes; - __be64 tx_uni_frames; - __be64 tx_uni_bytes; - __be64 tx_mcast_frames; - __be64 tx_mcast_bytes; - __be64 tx_bcast_frames; - __be64 tx_bcast_bytes; - __be64 tx_nobuf_frames; - __be64 tx_nobuf_bytes; - __be64 tx_err_frames; - __be64 tx_err_bytes; +struct mlx4_if_stat_basic { + struct mlx4_if_stat_control control; + struct { + __be64 IfRxFrames; + __be64 IfRxOctets; + __be64 IfTxFrames; + __be64 IfTxOctets; + } counters[]; }; +#define MLX4_IF_STAT_BSC_SZ(ports)(sizeof(struct mlx4_if_stat_extended) +\ + sizeof(((struct mlx4_if_stat_extended *)0)->\ + counters[0]) * ports) +struct mlx4_if_stat_extended { + struct mlx4_if_stat_control control; + struct { + __be64 IfRxUnicastFrames; + __be64 IfRxUnicastOctets; + __be64 IfRxMulticastFrames; + __be64 IfRxMulticastOctets; + __be64 IfRxBroadcastFrames; + __be64 IfRxBroadcastOctets; + __be64 IfRxNoBufferFrames; + __be64 IfRxNoBufferOctets; + __be64 IfRxErrorFrames; + __be64 IfRxErrorOctets; + __be32 reserved[39]; + __be64 IfTxUnicastFrames; + __be64 IfTxUnicastOctets; + __be64 IfTxMulticastFrames; + __be64 IfTxMulticastOctets; + __be64 IfTxBroadcastFrames; + __be64 IfTxBroadcastOctets; + __be64 IfTxDroppedFrames; + __be64 IfTxDroppedOctets; + __be64 IfTxRequestedFramesSent; + __be64 IfTxGeneratedFramesSent; + __be64 IfTxTsoOctets; + } __packed counters[]; +}; +#define MLX4_IF_STAT_EXT_SZ(ports) (sizeof(struct mlx4_if_stat_extended) +\ + sizeof(((struct mlx4_if_stat_extended *)\ + 0)->counters[0]) * ports) + +union mlx4_counter { + struct mlx4_if_stat_control control; + struct mlx4_if_stat_basic basic; + struct mlx4_if_stat_extended ext; +}; +#define MLX4_IF_STAT_SZ(ports) MLX4_IF_STAT_EXT_SZ(ports) + +struct mlx4_quotas { + int qp; + int cq; + int srq; + int mpt; + int mtt; + int counter; + int xrcd; +}; + struct mlx4_dev { struct pci_dev *pdev; unsigned long flags; + unsigned long num_slaves; struct mlx4_caps caps; + struct mlx4_phys_caps phys_caps; + struct mlx4_quotas quotas; struct radix_tree_root qp_table_tree; - struct radix_tree_root srq_table_tree; - u32 rev_id; + u8 rev_id; char board_id[MLX4_BOARD_ID_LEN]; + u16 vsd_vendor_id; + char vsd[MLX4_VSD_LEN]; + int num_vfs; + int numa_node; + int oper_log_mgm_entry_size; + u64 regid_promisc_array[MLX4_MAX_PORTS + 1]; + u64 regid_allmulti_array[MLX4_MAX_PORTS + 1]; }; +struct mlx4_clock_params { + u64 offset; + u8 bar; + u8 size; +}; + +struct mlx4_eqe { + u8 reserved1; + u8 type; + u8 reserved2; + u8 subtype; + union { + u32 raw[6]; + struct { + __be32 cqn; + } __packed comp; + struct { + u16 reserved1; + __be16 token; + u32 reserved2; + u8 reserved3[3]; + u8 status; + __be64 out_param; + } __packed cmd; + struct { + __be32 qpn; + } __packed qp; + struct { + __be32 srqn; + } __packed srq; + struct { + __be32 cqn; + u32 reserved1; + u8 reserved2[3]; + u8 syndrome; + } __packed cq_err; + struct { + u32 reserved1[2]; + __be32 port; + } __packed port_change; + struct { + #define COMM_CHANNEL_BIT_ARRAY_SIZE 4 + u32 reserved; + u32 bit_vec[COMM_CHANNEL_BIT_ARRAY_SIZE]; + } __packed comm_channel_arm; + struct { + u8 port; + u8 reserved[3]; + __be64 mac; + } __packed mac_update; + struct { + __be32 slave_id; + } __packed flr_event; + struct { + __be16 current_temperature; + __be16 warning_threshold; + } __packed warming; + struct { + u8 reserved[3]; + u8 port; + union { + struct { + __be16 mstr_sm_lid; + __be16 port_lid; + __be32 changed_attr; + u8 reserved[3]; + u8 mstr_sm_sl; + __be64 gid_prefix; + } __packed port_info; + struct { + __be32 block_ptr; + __be32 tbl_entries_mask; + } __packed tbl_change_info; + } params; + } __packed port_mgmt_change; + struct { + u8 reserved[3]; + u8 port; + u32 reserved1[5]; + } __packed bad_cable; + } event; + u8 slave_id; + u8 reserved3[2]; + u8 owner; +} __packed; + struct mlx4_init_port_param { int set_guid0; int set_node_guid; int set_si_guid; u16 mtu; int port_width_cap; u16 vl_cap; u16 max_gid; u16 max_pkey; u64 guid0; u64 node_guid; u64 si_guid; }; -static inline void mlx4_query_steer_cap(struct mlx4_dev *dev, int *log_mac, - int *log_vlan, int *log_prio) -{ - *log_mac = dev->caps.log_num_macs; - *log_vlan = dev->caps.log_num_vlans; - *log_prio = dev->caps.log_num_prios; -} - #define mlx4_foreach_port(port, dev, type) \ for ((port) = 1; (port) <= (dev)->caps.num_ports; (port)++) \ if ((type) == (dev)->caps.port_mask[(port)]) +#define mlx4_foreach_non_ib_transport_port(port, dev) \ + for ((port) = 1; (port) <= (dev)->caps.num_ports; (port)++) \ + if (((dev)->caps.port_mask[port] != MLX4_PORT_TYPE_IB)) + #define mlx4_foreach_ib_transport_port(port, dev) \ - for ((port) = 1; (port) <= (dev)->caps.num_ports; (port)++) \ + for ((port) = 1; (port) <= (dev)->caps.num_ports; (port)++) \ if (((dev)->caps.port_mask[port] == MLX4_PORT_TYPE_IB) || \ ((dev)->caps.flags & MLX4_DEV_CAP_FLAG_IBOE)) +#define MLX4_INVALID_SLAVE_ID 0xFF + +#define MLX4_SINK_COUNTER_INDEX 0xff + +void handle_port_mgmt_change_event(struct work_struct *work); + +static inline int mlx4_master_func_num(struct mlx4_dev *dev) +{ + return dev->caps.function; +} + +static inline int mlx4_is_master(struct mlx4_dev *dev) +{ + return dev->flags & MLX4_FLAG_MASTER; +} + +static inline int mlx4_num_reserved_sqps(struct mlx4_dev *dev) +{ + return dev->phys_caps.base_sqpn + 8 + + 16 * MLX4_MFUNC_MAX * !!mlx4_is_master(dev); +} + +static inline int mlx4_is_qp_reserved(struct mlx4_dev *dev, u32 qpn) +{ + return (qpn < dev->phys_caps.base_sqpn + 8 + + 16 * MLX4_MFUNC_MAX * !!mlx4_is_master(dev)); +} + +static inline int mlx4_is_guest_proxy(struct mlx4_dev *dev, int slave, u32 qpn) +{ + int guest_proxy_base = dev->phys_caps.base_proxy_sqpn + slave * 8; + + if (qpn >= guest_proxy_base && qpn < guest_proxy_base + 8) + return 1; + + return 0; +} + +static inline int mlx4_is_mfunc(struct mlx4_dev *dev) +{ + return dev->flags & (MLX4_FLAG_SLAVE | MLX4_FLAG_MASTER); +} + +static inline int mlx4_is_slave(struct mlx4_dev *dev) +{ + return dev->flags & MLX4_FLAG_SLAVE; +} + int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, struct mlx4_buf *buf); void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf); static inline void *mlx4_buf_offset(struct mlx4_buf *buf, int offset) { - if (buf->direct.buf != NULL) + if (BITS_PER_LONG == 64 || buf->nbufs == 1) return buf->direct.buf + offset; else return buf->page_list[offset >> PAGE_SHIFT].buf + (offset & (PAGE_SIZE - 1)); } int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn); void mlx4_pd_free(struct mlx4_dev *dev, u32 pdn); - int mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn); void mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn); int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar); void mlx4_uar_free(struct mlx4_dev *dev, struct mlx4_uar *uar); -int mlx4_bf_alloc(struct mlx4_dev *dev, struct mlx4_bf *bf); +int mlx4_bf_alloc(struct mlx4_dev *dev, struct mlx4_bf *bf, int node); void mlx4_bf_free(struct mlx4_dev *dev, struct mlx4_bf *bf); int mlx4_mtt_init(struct mlx4_dev *dev, int npages, int page_shift, struct mlx4_mtt *mtt); void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt); u64 mlx4_mtt_addr(struct mlx4_dev *dev, struct mlx4_mtt *mtt); -int mlx4_wol_read(struct mlx4_dev *dev, u64 *config, int port); -int mlx4_wol_write(struct mlx4_dev *dev, u64 config, int port); - -int mlx4_mr_reserve_range(struct mlx4_dev *dev, int cnt, int align, u32 *base_mridx); -void mlx4_mr_release_range(struct mlx4_dev *dev, u32 base_mridx, int cnt); -int mlx4_mr_alloc_reserved(struct mlx4_dev *dev, u32 mridx, u32 pd, - u64 iova, u64 size, u32 access, int npages, - int page_shift, struct mlx4_mr *mr); int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access, int npages, int page_shift, struct mlx4_mr *mr); -void mlx4_mr_free_reserved(struct mlx4_dev *dev, struct mlx4_mr *mr); -void mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr); +int mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr); int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr); +int mlx4_mw_alloc(struct mlx4_dev *dev, u32 pd, enum mlx4_mw_type type, + struct mlx4_mw *mw); +void mlx4_mw_free(struct mlx4_dev *dev, struct mlx4_mw *mw); +int mlx4_mw_enable(struct mlx4_dev *dev, struct mlx4_mw *mw); int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, int start_index, int npages, u64 *page_list); int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, struct mlx4_buf *buf); int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order); void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db); int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, int size, int max_direct); void mlx4_free_hwq_res(struct mlx4_dev *mdev, struct mlx4_hwq_resources *wqres, int size); int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt, struct mlx4_uar *uar, u64 db_rec, struct mlx4_cq *cq, - unsigned vector, int collapsed); + unsigned vector, int collapsed, int timestamp_en); void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq); -int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, int *base); +int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, + int *base, u8 flags); void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt); int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp); void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp); -int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, u32 cqn, u16 xrcd, +int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, u32 cqn, u16 xrcdn, struct mlx4_mtt *mtt, u64 db_rec, struct mlx4_srq *srq); void mlx4_srq_free(struct mlx4_dev *dev, struct mlx4_srq *srq); int mlx4_srq_arm(struct mlx4_dev *dev, struct mlx4_srq *srq, int limit_watermark); int mlx4_srq_query(struct mlx4_dev *dev, struct mlx4_srq *srq, int *limit_watermark); int mlx4_INIT_PORT(struct mlx4_dev *dev, int port); int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port); +int mlx4_unicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + int block_mcast_loopback, enum mlx4_protocol prot); +int mlx4_unicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], + enum mlx4_protocol prot); int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - int block_mcast_loopback, enum mlx4_mcast_prot prot); + u8 port, int block_mcast_loopback, + enum mlx4_protocol protocol, u64 *reg_id); int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], - enum mlx4_mcast_prot prot); + enum mlx4_protocol protocol, u64 reg_id); -int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *index); -void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, int index); +enum { + MLX4_DOMAIN_UVERBS = 0x1000, + MLX4_DOMAIN_ETHTOOL = 0x2000, + MLX4_DOMAIN_RFS = 0x3000, + MLX4_DOMAIN_NIC = 0x5000, +}; +enum mlx4_net_trans_rule_id { + MLX4_NET_TRANS_RULE_ID_ETH = 0, + MLX4_NET_TRANS_RULE_ID_IB, + MLX4_NET_TRANS_RULE_ID_IPV6, + MLX4_NET_TRANS_RULE_ID_IPV4, + MLX4_NET_TRANS_RULE_ID_TCP, + MLX4_NET_TRANS_RULE_ID_UDP, + MLX4_NET_TRANS_RULE_NUM, /* should be last */ +}; + +extern const u16 __sw_id_hw[]; + +static inline int map_hw_to_sw_id(u16 header_id) +{ + + int i; + for (i = 0; i < MLX4_NET_TRANS_RULE_NUM; i++) { + if (header_id == __sw_id_hw[i]) + return i; + } + return -EINVAL; +} + +enum mlx4_net_trans_promisc_mode { + MLX4_FS_REGULAR = 1, + MLX4_FS_ALL_DEFAULT, + MLX4_FS_MC_DEFAULT, + MLX4_FS_UC_SNIFFER, + MLX4_FS_MC_SNIFFER, + MLX4_FS_MODE_NUM, /* should be last */ +}; + +struct mlx4_spec_eth { + u8 dst_mac[6]; + u8 dst_mac_msk[6]; + u8 src_mac[6]; + u8 src_mac_msk[6]; + u8 ether_type_enable; + __be16 ether_type; + __be16 vlan_id_msk; + __be16 vlan_id; +}; + +struct mlx4_spec_tcp_udp { + __be16 dst_port; + __be16 dst_port_msk; + __be16 src_port; + __be16 src_port_msk; +}; + +struct mlx4_spec_ipv4 { + __be32 dst_ip; + __be32 dst_ip_msk; + __be32 src_ip; + __be32 src_ip_msk; +}; + +struct mlx4_spec_ib { + __be32 l3_qpn; + __be32 qpn_msk; + u8 dst_gid[16]; + u8 dst_gid_msk[16]; +}; + +struct mlx4_spec_list { + struct list_head list; + enum mlx4_net_trans_rule_id id; + union { + struct mlx4_spec_eth eth; + struct mlx4_spec_ib ib; + struct mlx4_spec_ipv4 ipv4; + struct mlx4_spec_tcp_udp tcp_udp; + }; +}; + +enum mlx4_net_trans_hw_rule_queue { + MLX4_NET_TRANS_Q_FIFO, + MLX4_NET_TRANS_Q_LIFO, +}; + +struct mlx4_net_trans_rule { + struct list_head list; + enum mlx4_net_trans_hw_rule_queue queue_mode; + bool exclusive; + bool allow_loopback; + enum mlx4_net_trans_promisc_mode promisc_mode; + u8 port; + u16 priority; + u32 qpn; +}; + +struct mlx4_net_trans_rule_hw_ctrl { + __be16 prio; + u8 type; + u8 flags; + u8 rsvd1; + u8 funcid; + u8 vep; + u8 port; + __be32 qpn; + __be32 rsvd2; +}; + +struct mlx4_net_trans_rule_hw_ib { + u8 size; + u8 rsvd1; + __be16 id; + u32 rsvd2; + __be32 l3_qpn; + __be32 qpn_mask; + u8 dst_gid[16]; + u8 dst_gid_msk[16]; +} __packed; + +struct mlx4_net_trans_rule_hw_eth { + u8 size; + u8 rsvd; + __be16 id; + u8 rsvd1[6]; + u8 dst_mac[6]; + u16 rsvd2; + u8 dst_mac_msk[6]; + u16 rsvd3; + u8 src_mac[6]; + u16 rsvd4; + u8 src_mac_msk[6]; + u8 rsvd5; + u8 ether_type_enable; + __be16 ether_type; + __be16 vlan_tag_msk; + __be16 vlan_tag; +} __packed; + +struct mlx4_net_trans_rule_hw_tcp_udp { + u8 size; + u8 rsvd; + __be16 id; + __be16 rsvd1[3]; + __be16 dst_port; + __be16 rsvd2; + __be16 dst_port_msk; + __be16 rsvd3; + __be16 src_port; + __be16 rsvd4; + __be16 src_port_msk; +} __packed; + +struct mlx4_net_trans_rule_hw_ipv4 { + u8 size; + u8 rsvd; + __be16 id; + __be32 rsvd1; + __be32 dst_ip; + __be32 dst_ip_msk; + __be32 src_ip; + __be32 src_ip_msk; +} __packed; + +struct _rule_hw { + union { + struct { + u8 size; + u8 rsvd; + __be16 id; + }; + struct mlx4_net_trans_rule_hw_eth eth; + struct mlx4_net_trans_rule_hw_ib ib; + struct mlx4_net_trans_rule_hw_ipv4 ipv4; + struct mlx4_net_trans_rule_hw_tcp_udp tcp_udp; + }; +}; + +int mlx4_flow_steer_promisc_add(struct mlx4_dev *dev, u8 port, u32 qpn, + enum mlx4_net_trans_promisc_mode mode); +int mlx4_flow_steer_promisc_remove(struct mlx4_dev *dev, u8 port, + enum mlx4_net_trans_promisc_mode mode); +int mlx4_multicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port); +int mlx4_multicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port); +int mlx4_unicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port); +int mlx4_unicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port); + +int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac); +void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, u64 mac); +int mlx4_get_base_qpn(struct mlx4_dev *dev, u8 port); +int __mlx4_replace_mac(struct mlx4_dev *dev, u8 port, int qpn, u64 new_mac); +void mlx4_set_stats_bitmap(struct mlx4_dev *dev, unsigned long *stats_bitmap); +int mlx4_SET_PORT_general(struct mlx4_dev *dev, u8 port, int mtu, + u8 pptx, u8 pfctx, u8 pprx, u8 pfcrx); +int mlx4_SET_PORT_qpn_calc(struct mlx4_dev *dev, u8 port, u32 base_qpn, + u8 promisc); +int mlx4_SET_PORT_PRIO2TC(struct mlx4_dev *dev, u8 port, u8 *prio2tc); +int mlx4_SET_PORT_SCHEDULER(struct mlx4_dev *dev, u8 port, u8 *tc_tx_bw, + u8 *pg, u16 *ratelimit); int mlx4_find_cached_vlan(struct mlx4_dev *dev, u8 port, u16 vid, int *idx); int mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index); -void mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, int index); +void mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, u16 vlan); -int mlx4_map_phys_fmr_fbo(struct mlx4_dev *dev, struct mlx4_fmr *fmr, - u64 *page_list, int npages, u64 iova, u32 fbo, - u32 len, u32 *lkey, u32 *rkey, int same_key); int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list, int npages, u64 iova, u32 *lkey, u32 *rkey); -int mlx4_fmr_alloc_reserved(struct mlx4_dev *dev, u32 mridx, u32 pd, - u32 access, int max_pages, int max_maps, - u8 page_shift, struct mlx4_fmr *fmr); int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages, int max_maps, u8 page_shift, struct mlx4_fmr *fmr); int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr); void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u32 *lkey, u32 *rkey); -int mlx4_fmr_free_reserved(struct mlx4_dev *dev, struct mlx4_fmr *fmr); int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr); int mlx4_SYNC_TPT(struct mlx4_dev *dev); int mlx4_query_diag_counters(struct mlx4_dev *mlx4_dev, int array_length, - u8 op_modifier, u32 in_offset[], u32 counter_out[]); + u8 op_modifier, u32 in_offset[], + u32 counter_out[]); + int mlx4_test_interrupts(struct mlx4_dev *dev); +int mlx4_assign_eq(struct mlx4_dev *dev, char* name, int * vector); +void mlx4_release_eq(struct mlx4_dev *dev, int vec); -void mlx4_get_fc_t11_settings(struct mlx4_dev *dev, int *enable_pre_t11, int *t11_supported); +int mlx4_wol_read(struct mlx4_dev *dev, u64 *config, int port); +int mlx4_wol_write(struct mlx4_dev *dev, u64 config, int port); -int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx); -void mlx4_counter_free(struct mlx4_dev *dev, u32 idx); +int mlx4_counter_alloc(struct mlx4_dev *dev, u8 port, u32 *idx); +void mlx4_counter_free(struct mlx4_dev *dev, u8 port, u32 idx); + +int mlx4_flow_attach(struct mlx4_dev *dev, + struct mlx4_net_trans_rule *rule, u64 *reg_id); +int mlx4_flow_detach(struct mlx4_dev *dev, u64 reg_id); +int map_sw_to_hw_steering_mode(struct mlx4_dev *dev, + enum mlx4_net_trans_promisc_mode flow_type); +int map_sw_to_hw_steering_id(struct mlx4_dev *dev, + enum mlx4_net_trans_rule_id id); +int hw_rule_sz(struct mlx4_dev *dev, enum mlx4_net_trans_rule_id id); + +void mlx4_sync_pkey_table(struct mlx4_dev *dev, int slave, int port, + int i, int val); + +int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey); + +int mlx4_is_slave_active(struct mlx4_dev *dev, int slave); +int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, u8 port); +int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, u8 port); +int mlx4_gen_slaves_port_mgt_ev(struct mlx4_dev *dev, u8 port, int attr, u16 lid, u8 sl); +int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port, u8 port_subtype_change); +enum slave_port_state mlx4_get_slave_port_state(struct mlx4_dev *dev, int slave, u8 port); +int set_and_calc_slave_port_state(struct mlx4_dev *dev, int slave, u8 port, int event, enum slave_port_gen_event *gen_event); + +void mlx4_put_slave_node_guid(struct mlx4_dev *dev, int slave, __be64 guid); +__be64 mlx4_get_slave_node_guid(struct mlx4_dev *dev, int slave); +int mlx4_get_slave_from_roce_gid(struct mlx4_dev *dev, int port, u8 *gid, int *slave_id); +int mlx4_get_roce_gid_from_slave(struct mlx4_dev *dev, int port, int slave_id, u8 *gid); + +int mlx4_FLOW_STEERING_IB_UC_QP_RANGE(struct mlx4_dev *dev, u32 min_range_qpn, u32 max_range_qpn); + +int mlx4_read_clock(struct mlx4_dev *dev); +int mlx4_get_internal_clock_params(struct mlx4_dev *dev, + struct mlx4_clock_params *params); #endif /* MLX4_DEVICE_H */ Index: stable/9/sys/ofed/include/linux/mlx4/driver.h =================================================================== --- stable/9/sys/ofed/include/linux/mlx4/driver.h (revision 273863) +++ stable/9/sys/ofed/include/linux/mlx4/driver.h (revision 273864) @@ -1,82 +1,136 @@ /* * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef MLX4_DRIVER_H #define MLX4_DRIVER_H -#include +#include struct mlx4_dev; +#define MLX4_MAC_MASK 0xffffffffffffULL +#define MLX4_BE_SHORT_MASK cpu_to_be16(0xffff) +#define MLX4_BE_WORD_MASK cpu_to_be32(0xffffffff) + enum mlx4_dev_event { MLX4_DEV_EVENT_CATASTROPHIC_ERROR, MLX4_DEV_EVENT_PORT_UP, MLX4_DEV_EVENT_PORT_DOWN, MLX4_DEV_EVENT_PORT_REINIT, + MLX4_DEV_EVENT_PORT_MGMT_CHANGE, + MLX4_DEV_EVENT_SLAVE_INIT, + MLX4_DEV_EVENT_SLAVE_SHUTDOWN, }; -enum mlx4_query_reply { - MLX4_QUERY_NOT_MINE = -1, - MLX4_QUERY_MINE_NOPORT = 0 +struct mlx4_interface { + void * (*add) (struct mlx4_dev *dev); + void (*remove)(struct mlx4_dev *dev, void *context); + void (*event) (struct mlx4_dev *dev, void *context, + enum mlx4_dev_event event, unsigned long param); + void * (*get_dev)(struct mlx4_dev *dev, void *context, u8 port); + struct list_head list; + enum mlx4_protocol protocol; }; -enum mlx4_prot { - MLX4_PROT_IB, - MLX4_PROT_EN, +enum { + MLX4_MAX_DEVICES = 32, + MLX4_DEVS_TBL_SIZE = MLX4_MAX_DEVICES + 1, + MLX4_DBDF2VAL_STR_SIZE = 512, + MLX4_STR_NAME_SIZE = 64, + MLX4_MAX_BDF_VALS = 2, + MLX4_ENDOF_TBL = -1LL }; -enum mlx4_mcast_prot { - MLX4_MCAST_PROT_IB = 0, - MLX4_MCAST_PROT_EN = 1, +struct mlx4_dbdf2val { + u64 dbdf; + int val[MLX4_MAX_BDF_VALS]; }; -struct mlx4_interface { - void * (*add) (struct mlx4_dev *dev); - void (*remove)(struct mlx4_dev *dev, void *context); - void (*event) (struct mlx4_dev *dev, void *context, - enum mlx4_dev_event event, int port); - void * (*get_prot_dev) (struct mlx4_dev *dev, void *context, u8 port); - enum mlx4_prot protocol; +struct mlx4_range { + int min; + int max; +}; - enum mlx4_query_reply (*query) (void *context, void *); - struct list_head list; +/* + * mlx4_dbdf2val_lst struct holds all the data needed to convert + * dbdf-to-value-list string into dbdf-to-value table. + * dbdf-to-value-list string is a comma separated list of dbdf-to-value strings. + * the format of dbdf-to-value string is: "[mmmm:]bb:dd.f-v1[;v2]" + * mmmm - Domain number (optional) + * bb - Bus number + * dd - device number + * f - Function number + * v1 - First value related to the domain-bus-device-function. + * v2 - Second value related to the domain-bus-device-function (optional). + * bb, dd - Two hexadecimal digits without preceding 0x. + * mmmm - Four hexadecimal digits without preceding 0x. + * f - One hexadecimal without preceding 0x. + * v1,v2 - Number with normal convention (e.g 100, 0xd3). + * dbdf-to-value-list string format: + * "[mmmm:]bb:dd.f-v1[;v2],[mmmm:]bb:dd.f-v1[;v2],..." + * + */ +struct mlx4_dbdf2val_lst { + char name[MLX4_STR_NAME_SIZE]; /* String name */ + char str[MLX4_DBDF2VAL_STR_SIZE]; /* dbdf2val list str */ + struct mlx4_dbdf2val tbl[MLX4_DEVS_TBL_SIZE];/* dbdf to value table */ + int num_vals; /* # of vals per dbdf */ + int def_val[MLX4_MAX_BDF_VALS]; /* Default values */ + struct mlx4_range range; /* Valid values range */ }; +int mlx4_fill_dbdf2val_tbl(struct mlx4_dbdf2val_lst *dbdf2val_lst); +int mlx4_get_val(struct mlx4_dbdf2val *tbl, struct pci_dev *pdev, int idx, + int *val); + int mlx4_register_interface(struct mlx4_interface *intf); void mlx4_unregister_interface(struct mlx4_interface *intf); -void *mlx4_get_prot_dev(struct mlx4_dev *dev, enum mlx4_prot proto, int port); -struct mlx4_dev *mlx4_query_interface(void *, int *port); -void mlx4_set_iboe_counter(struct mlx4_dev *dev, int index, u8 port); -int mlx4_get_iboe_counter(struct mlx4_dev *dev, u8 port); +void *mlx4_get_protocol_dev(struct mlx4_dev *dev, enum mlx4_protocol proto, + int port); + +#ifndef ETH_ALEN +#define ETH_ALEN 6 +#endif +static inline u64 mlx4_mac_to_u64(u8 *addr) +{ + u64 mac = 0; + int i; + + for (i = 0; i < ETH_ALEN; i++) { + mac <<= 8; + mac |= addr[i]; + } + return mac; +} #endif /* MLX4_DRIVER_H */ Index: stable/9/sys/ofed/include/linux/mlx4/qp.h =================================================================== --- stable/9/sys/ofed/include/linux/mlx4/qp.h (revision 273863) +++ stable/9/sys/ofed/include/linux/mlx4/qp.h (revision 273864) @@ -1,346 +1,445 @@ /* * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef MLX4_QP_H #define MLX4_QP_H #include #include #define MLX4_INVALID_LKEY 0x100 +enum ib_m_qp_attr_mask { + IB_M_EXT_CLASS_1 = 1 << 28, + IB_M_EXT_CLASS_2 = 1 << 29, + IB_M_EXT_CLASS_3 = 1 << 30, + + IB_M_QP_MOD_VEND_MASK = (IB_M_EXT_CLASS_1 | IB_M_EXT_CLASS_2 | + IB_M_EXT_CLASS_3) +}; + enum mlx4_qp_optpar { MLX4_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0, MLX4_QP_OPTPAR_RRE = 1 << 1, MLX4_QP_OPTPAR_RAE = 1 << 2, MLX4_QP_OPTPAR_RWE = 1 << 3, MLX4_QP_OPTPAR_PKEY_INDEX = 1 << 4, MLX4_QP_OPTPAR_Q_KEY = 1 << 5, MLX4_QP_OPTPAR_RNR_TIMEOUT = 1 << 6, MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH = 1 << 7, MLX4_QP_OPTPAR_SRA_MAX = 1 << 8, MLX4_QP_OPTPAR_RRA_MAX = 1 << 9, MLX4_QP_OPTPAR_PM_STATE = 1 << 10, MLX4_QP_OPTPAR_RETRY_COUNT = 1 << 12, MLX4_QP_OPTPAR_RNR_RETRY = 1 << 13, MLX4_QP_OPTPAR_ACK_TIMEOUT = 1 << 14, MLX4_QP_OPTPAR_SCHED_QUEUE = 1 << 16, MLX4_QP_OPTPAR_COUNTER_INDEX = 1 << 20 }; enum mlx4_qp_state { MLX4_QP_STATE_RST = 0, MLX4_QP_STATE_INIT = 1, MLX4_QP_STATE_RTR = 2, MLX4_QP_STATE_RTS = 3, MLX4_QP_STATE_SQER = 4, MLX4_QP_STATE_SQD = 5, MLX4_QP_STATE_ERR = 6, MLX4_QP_STATE_SQ_DRAINING = 7, MLX4_QP_NUM_STATE }; enum { MLX4_QP_ST_RC = 0x0, MLX4_QP_ST_UC = 0x1, MLX4_QP_ST_RD = 0x2, MLX4_QP_ST_UD = 0x3, MLX4_QP_ST_XRC = 0x6, MLX4_QP_ST_MLX = 0x7 }; enum { MLX4_QP_PM_MIGRATED = 0x3, MLX4_QP_PM_ARMED = 0x0, MLX4_QP_PM_REARM = 0x1 }; enum { /* params1 */ MLX4_QP_BIT_SRE = 1 << 15, MLX4_QP_BIT_SWE = 1 << 14, MLX4_QP_BIT_SAE = 1 << 13, /* params2 */ MLX4_QP_BIT_RRE = 1 << 15, MLX4_QP_BIT_RWE = 1 << 14, MLX4_QP_BIT_RAE = 1 << 13, MLX4_QP_BIT_RIC = 1 << 4, + MLX4_QP_BIT_COLL_SYNC_RQ = 1 << 2, + MLX4_QP_BIT_COLL_SYNC_SQ = 1 << 1, + MLX4_QP_BIT_COLL_MASTER = 1 << 0 }; +enum { + MLX4_RSS_HASH_XOR = 0, + MLX4_RSS_HASH_TOP = 1, + + MLX4_RSS_UDP_IPV6 = 1 << 0, + MLX4_RSS_UDP_IPV4 = 1 << 1, + MLX4_RSS_TCP_IPV6 = 1 << 2, + MLX4_RSS_IPV6 = 1 << 3, + MLX4_RSS_TCP_IPV4 = 1 << 4, + MLX4_RSS_IPV4 = 1 << 5, + + /* offset of mlx4_rss_context within mlx4_qp_context.pri_path */ + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH = 0x24, + /* offset of being RSS indirection QP within mlx4_qp_context.flags */ + MLX4_RSS_QPC_FLAG_OFFSET = 13, +}; + +struct mlx4_rss_context { + __be32 base_qpn; + __be32 default_qpn; + u16 reserved; + u8 hash_fn; + u8 flags; + __be32 rss_key[10]; + __be32 base_qpn_udp; +}; + struct mlx4_qp_path { u8 fl; - u8 reserved1[2]; + u8 vlan_control; + u8 disable_pkey_check; u8 pkey_index; u8 counter_index; u8 grh_mylmc; __be16 rlid; u8 ackto; u8 mgid_index; u8 static_rate; u8 hop_limit; __be32 tclass_flowlabel; u8 rgid[16]; u8 sched_queue; u8 vlan_index; - u8 reserved3[2]; + u8 feup; + u8 fvl_rx; u8 reserved4[2]; u8 dmac[6]; }; +enum { /* fl */ + MLX4_FL_CV = 1 << 6, + MLX4_FL_ETH_HIDE_CQE_VLAN = 1 << 2, + MLX4_FL_ETH_SRC_CHECK_MC_LB = 1 << 1, + MLX4_FL_ETH_SRC_CHECK_UC_LB = 1 << 0, +}; +enum { /* vlan_control */ + MLX4_VLAN_CTRL_ETH_SRC_CHECK_IF_COUNTER = 1 << 7, + MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED = 1 << 6, + MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED = 1 << 2, + MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED = 1 << 1,/* 802.1p priorty tag*/ + MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED = 1 << 0 +}; + +enum { /* feup */ + MLX4_FEUP_FORCE_ETH_UP = 1 << 6, /* force Eth UP */ + MLX4_FSM_FORCE_ETH_SRC_MAC = 1 << 5, /* force Source MAC */ + MLX4_FVL_FORCE_ETH_VLAN = 1 << 3 /* force Eth vlan */ +}; + +enum { /* fvl_rx */ + MLX4_FVL_RX_FORCE_ETH_VLAN = 1 << 0 /* enforce Eth rx vlan */ +}; + struct mlx4_qp_context { __be32 flags; __be32 pd; u8 mtu_msgmax; u8 rq_size_stride; u8 sq_size_stride; u8 rlkey; __be32 usr_page; __be32 local_qpn; __be32 remote_qpn; struct mlx4_qp_path pri_path; struct mlx4_qp_path alt_path; __be32 params1; u32 reserved1; __be32 next_send_psn; __be32 cqn_send; u32 reserved2[2]; __be32 last_acked_psn; __be32 ssn; __be32 params2; __be32 rnr_nextrecvpsn; __be32 xrcd; __be32 cqn_recv; __be64 db_rec_addr; __be32 qkey; __be32 srqn; __be32 msn; __be16 rq_wqe_counter; __be16 sq_wqe_counter; u32 reserved3[2]; __be32 param3; __be32 nummmcpeers_basemkey; u8 log_page_size; u8 reserved4[2]; u8 mtt_base_addr_h; __be32 mtt_base_addr_l; - u8 VE; - u8 reserved5; - __be16 VFT_id_prio; - u8 reserved6; - u8 exch_size; - __be16 exch_base; - u8 VFT_hop_cnt; - u8 my_fc_id_idx; - __be16 reserved7; - u32 reserved8[7]; + u32 reserved5[10]; }; +struct mlx4_update_qp_context { + __be64 qp_mask; + __be64 primary_addr_path_mask; + __be64 secondary_addr_path_mask; + u64 reserved1; + struct mlx4_qp_context qp_context; + u64 reserved2[58]; +}; + +enum { + MLX4_UPD_QP_MASK_PM_STATE = 32, + MLX4_UPD_QP_MASK_VSD = 33, +}; + +enum { + MLX4_UPD_QP_PATH_MASK_PKEY_INDEX = 0 + 32, + MLX4_UPD_QP_PATH_MASK_FSM = 1 + 32, + MLX4_UPD_QP_PATH_MASK_MAC_INDEX = 2 + 32, + MLX4_UPD_QP_PATH_MASK_FVL = 3 + 32, + MLX4_UPD_QP_PATH_MASK_CV = 4 + 32, + MLX4_UPD_QP_PATH_MASK_VLAN_INDEX = 5 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_HIDE_CQE_VLAN = 6 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_UNTAGGED = 7 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_1P = 8 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_TX_BLOCK_TAGGED = 9 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_UNTAGGED = 10 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_1P = 11 + 32, + MLX4_UPD_QP_PATH_MASK_ETH_RX_BLOCK_TAGGED = 12 + 32, + MLX4_UPD_QP_PATH_MASK_FEUP = 13 + 32, + MLX4_UPD_QP_PATH_MASK_SCHED_QUEUE = 14 + 32, + MLX4_UPD_QP_PATH_MASK_IF_COUNTER_INDEX = 15 + 32, + MLX4_UPD_QP_PATH_MASK_FVL_RX = 16 + 32, +}; + +enum { /* param3 */ + MLX4_STRIP_VLAN = 1 << 30 +}; + + /* Which firmware version adds support for NEC (NoErrorCompletion) bit */ #define MLX4_FW_VER_WQE_CTRL_NEC mlx4_fw_ver(2, 2, 232) enum { MLX4_WQE_CTRL_NEC = 1 << 29, MLX4_WQE_CTRL_FENCE = 1 << 6, MLX4_WQE_CTRL_CQ_UPDATE = 3 << 2, MLX4_WQE_CTRL_SOLICITED = 1 << 1, MLX4_WQE_CTRL_IP_CSUM = 1 << 4, MLX4_WQE_CTRL_TCP_UDP_CSUM = 1 << 5, MLX4_WQE_CTRL_INS_VLAN = 1 << 6, MLX4_WQE_CTRL_STRONG_ORDER = 1 << 7, MLX4_WQE_CTRL_FORCE_LOOPBACK = 1 << 0, }; struct mlx4_wqe_ctrl_seg { __be32 owner_opcode; __be16 vlan_tag; u8 ins_vlan; u8 fence_size; /* * High 24 bits are SRC remote buffer; low 8 bits are flags: * [7] SO (strong ordering) * [5] TCP/UDP checksum * [4] IP checksum * [3:2] C (generate completion queue entry) * [1] SE (solicited event) + * [0] FL (force loopback) */ - __be32 srcrb_flags; + union { + __be32 srcrb_flags; + __be16 srcrb_flags16[2]; + }; /* * imm is immediate data for send/RDMA write w/ immediate; * also invalidation key for send with invalidate; input * modifier for WQEs on CCQs. */ __be32 imm; }; enum { MLX4_WQE_MLX_VL15 = 1 << 17, - MLX4_WQE_MLX_SLR = 1 << 16, - MLX4_WQE_MLX_ICRC = 1 << 4 + MLX4_WQE_MLX_SLR = 1 << 16 }; struct mlx4_wqe_mlx_seg { u8 owner; u8 reserved1[2]; u8 opcode; - u8 reserved2[3]; + __be16 sched_prio; + u8 reserved2; u8 size; /* * [17] VL15 * [16] SLR * [15:12] static rate * [11:8] SL * [4] ICRC * [3:2] C * [0] FL (force loopback) */ __be32 flags; __be16 rlid; u16 reserved3; }; struct mlx4_wqe_datagram_seg { __be32 av[8]; __be32 dqpn; __be32 qkey; __be16 vlan; u8 mac[6]; }; struct mlx4_wqe_lso_seg { __be32 mss_hdr_size; __be32 header[0]; }; +enum mlx4_wqe_bind_seg_flags2 { + MLX4_WQE_BIND_TYPE_2 = (1<<31), + MLX4_WQE_BIND_ZERO_BASED = (1<<30), +}; + struct mlx4_wqe_bind_seg { __be32 flags1; __be32 flags2; __be32 new_rkey; __be32 lkey; __be64 addr; __be64 length; }; enum { MLX4_WQE_FMR_PERM_LOCAL_READ = 1 << 27, MLX4_WQE_FMR_PERM_LOCAL_WRITE = 1 << 28, - MLX4_WQE_FMR_PERM_REMOTE_READ = 1 << 29, - MLX4_WQE_FMR_PERM_REMOTE_WRITE = 1 << 30, - MLX4_WQE_FMR_PERM_ATOMIC = 1 << 31 + MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ = 1 << 29, + MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE = 1 << 30, + MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC = 1 << 31 }; struct mlx4_wqe_fmr_seg { __be32 flags; __be32 mem_key; __be64 buf_list; __be64 start_addr; __be64 reg_len; __be32 offset; __be32 page_size; u32 reserved[2]; }; struct mlx4_wqe_fmr_ext_seg { u8 flags; u8 reserved; __be16 app_mask; __be16 wire_app_tag; __be16 mem_app_tag; __be32 wire_ref_tag_base; __be32 mem_ref_tag_base; }; struct mlx4_wqe_local_inval_seg { - __be32 flags; - u32 reserved1; + u64 reserved1; __be32 mem_key; - u32 reserved2[2]; - __be32 guest_id; - __be64 pa; + u32 reserved2; + u64 reserved3[2]; }; struct mlx4_wqe_raddr_seg { __be64 raddr; __be32 rkey; u32 reserved; }; struct mlx4_wqe_atomic_seg { __be64 swap_add; __be64 compare; }; struct mlx4_wqe_masked_atomic_seg { __be64 swap_add; __be64 compare; __be64 swap_add_mask; __be64 compare_mask; }; struct mlx4_wqe_data_seg { __be32 byte_count; __be32 lkey; __be64 addr; }; enum { MLX4_INLINE_ALIGN = 64, MLX4_INLINE_SEG = 1 << 31, }; struct mlx4_wqe_inline_seg { __be32 byte_count; }; int mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt, enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state, struct mlx4_qp_context *context, enum mlx4_qp_optpar optpar, int sqd_event, struct mlx4_qp *qp); int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp, struct mlx4_qp_context *context); int mlx4_qp_to_ready(struct mlx4_dev *dev, struct mlx4_mtt *mtt, struct mlx4_qp_context *context, struct mlx4_qp *qp, enum mlx4_qp_state *qp_state); static inline struct mlx4_qp *__mlx4_qp_lookup(struct mlx4_dev *dev, u32 qpn) { return radix_tree_lookup(&dev->qp_table_tree, qpn & (dev->caps.num_qps - 1)); } -struct mlx4_qp *mlx4_qp_lookup_lock(struct mlx4_dev *dev, u32 qpn); void mlx4_qp_remove(struct mlx4_dev *dev, struct mlx4_qp *qp); -int mlx4_qp_get_region(struct mlx4_dev *dev, enum mlx4_qp_region region, - int *base_qpn, int *cnt); #endif /* MLX4_QP_H */ Index: stable/9/sys/ofed/include/linux/mlx4/srq.h =================================================================== --- stable/9/sys/ofed/include/linux/mlx4/srq.h (revision 273863) +++ stable/9/sys/ofed/include/linux/mlx4/srq.h (revision 273864) @@ -1,54 +1,44 @@ /* * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef MLX4_SRQ_H #define MLX4_SRQ_H -#include -#include - struct mlx4_wqe_srq_next_seg { u16 reserved1; __be16 next_wqe_index; u32 reserved2[3]; }; -void mlx4_srq_invalidate(struct mlx4_dev *dev, struct mlx4_srq *srq); -void mlx4_srq_remove(struct mlx4_dev *dev, struct mlx4_srq *srq); - -static inline struct mlx4_srq *__mlx4_srq_lookup(struct mlx4_dev *dev, u32 srqn) -{ - return radix_tree_lookup(&dev->srq_table_tree, - srqn & (dev->caps.num_srqs - 1)); -} +struct mlx4_srq *mlx4_srq_lookup(struct mlx4_dev *dev, u32 srqn); #endif /* MLX4_SRQ_H */ Index: stable/9/sys/ofed/include/linux/mm.h =================================================================== --- stable/9/sys/ofed/include/linux/mm.h (revision 273863) +++ stable/9/sys/ofed/include/linux/mm.h (revision 273864) @@ -1,84 +1,85 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_MM_H_ #define _LINUX_MM_H_ #include #include #include #define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE) struct vm_area_struct { vm_offset_t vm_start; vm_offset_t vm_end; vm_offset_t vm_pgoff; vm_paddr_t vm_pfn; /* PFN For mmap. */ vm_memattr_t vm_page_prot; }; /* * Compute log2 of the power of two rounded up count of pages * needed for size bytes. */ static inline int get_order(unsigned long size) { int order; size = (size - 1) >> PAGE_SHIFT; order = 0; while (size) { order++; size >>= 1; } return (order); } static inline void * lowmem_page_address(struct page *page) { return page_address(page); } /* * This only works via mmap ops. */ static inline int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, vm_memattr_t prot) { vma->vm_page_prot = prot; vma->vm_pfn = pfn; return (0); } #endif /* _LINUX_MM_H_ */ Index: stable/9/sys/ofed/include/linux/module.h =================================================================== --- stable/9/sys/ofed/include/linux/module.h (revision 273863) +++ stable/9/sys/ofed/include/linux/module.h (revision 273864) @@ -1,92 +1,99 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_MODULE_H_ #define _LINUX_MODULE_H_ #include #include #include #include #include #define MODULE_AUTHOR(name) #define MODULE_DESCRIPTION(name) #define MODULE_LICENSE(name) -#define MODULE_VERSION(name) +#ifndef MODULE_VERSION +#define MODULE_VERSION(name) +#endif + #define THIS_MODULE ((struct module *)0) #define EXPORT_SYMBOL(name) #define EXPORT_SYMBOL_GPL(name) /* OFED pre-module initialization */ #define SI_SUB_OFED_PREINIT (SI_SUB_KTHREAD_INIT - 2) /* OFED default module initialization */ #define SI_SUB_OFED_MODINIT (SI_SUB_KTHREAD_INIT - 1) #include static inline void _module_run(void *arg) { void (*fn)(void); #ifdef OFED_DEBUG_INIT char name[1024]; caddr_t pc; long offset; pc = (caddr_t)arg; if (linker_search_symbol_name(pc, name, sizeof(name), &offset) != 0) printf("Running ??? (%p)\n", pc); else printf("Running %s (%p)\n", name, pc); #endif fn = arg; DROP_GIANT(); fn(); PICKUP_GIANT(); } #define module_init(fn) \ SYSINIT(fn, SI_SUB_OFED_MODINIT, SI_ORDER_FIRST, _module_run, (fn)) +#define module_exit(fn) \ + SYSUNINIT(fn, SI_SUB_OFED_MODINIT, SI_ORDER_SECOND, _module_run, (fn)) + /* - * XXX This is a freebsdism designed to work around not having a module - * load order resolver built in. + * The following two macros are a workaround for not having a module + * load and unload order resolver: */ #define module_init_order(fn, order) \ SYSINIT(fn, SI_SUB_OFED_MODINIT, (order), _module_run, (fn)) -#define module_exit(fn) \ - SYSUNINIT(fn, SI_SUB_OFED_MODINIT, SI_ORDER_FIRST, _module_run, (fn)) +#define module_exit_order(fn, order) \ + SYSUNINIT(fn, SI_SUB_OFED_MODINIT, (order), _module_run, (fn)) #define module_get(module) #define module_put(module) #define try_module_get(module) 1 #endif /* _LINUX_MODULE_H_ */ Index: stable/9/sys/ofed/include/linux/moduleparam.h =================================================================== --- stable/9/sys/ofed/include/linux/moduleparam.h (revision 273863) +++ stable/9/sys/ofed/include/linux/moduleparam.h (revision 273864) @@ -1,226 +1,233 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef _LINUX_MODULEPARAM_H_ #define _LINUX_MODULEPARAM_H_ #include /* * These are presently not hooked up to anything. In linux the parameters * can be set when modules are loaded. On FreeBSD these could be mapped * to kenv in the future. */ struct kernel_param; typedef int (*param_set_fn)(const char *val, struct kernel_param *kp); typedef int (*param_get_fn)(char *buffer, struct kernel_param *kp); struct kernel_param { const char *name; u16 perm; u16 flags; param_set_fn set; param_get_fn get; union { void *arg; struct kparam_string *str; struct kparam_array *arr; } un; }; #define KPARAM_ISBOOL 2 struct kparam_string { unsigned int maxlen; char *string; }; struct kparam_array { unsigned int max; unsigned int *num; param_set_fn set; param_get_fn get; unsigned int elemsize; void *elem; }; static inline void param_sysinit(struct kernel_param *param) { } #define module_param_call(name, set, get, arg, perm) \ static struct kernel_param __param_##name = \ { #name, perm, 0, set, get, { arg } }; \ SYSINIT(name##_param_sysinit, SI_SUB_DRIVERS, SI_ORDER_FIRST, \ param_sysinit, &__param_##name); +#define module_param_string(name, string, len, perm) + #define module_param_named(name, var, type, mode) \ module_param_call(name, param_set_##type, param_get_##type, &var, mode) #define module_param(var, type, mode) \ module_param_named(var, var, type, mode) + +#define module_param_array(var, type, addr_argc, mode) \ + module_param_named(var, var, type, mode) #define MODULE_PARM_DESC(name, desc) static inline int param_set_byte(const char *val, struct kernel_param *kp) { return 0; } static inline int param_get_byte(char *buffer, struct kernel_param *kp) { return 0; } static inline int param_set_short(const char *val, struct kernel_param *kp) { return 0; } static inline int param_get_short(char *buffer, struct kernel_param *kp) { return 0; } static inline int param_set_ushort(const char *val, struct kernel_param *kp) { return 0; } static inline int param_get_ushort(char *buffer, struct kernel_param *kp) { return 0; } static inline int param_set_int(const char *val, struct kernel_param *kp) { return 0; } static inline int param_get_int(char *buffer, struct kernel_param *kp) { return 0; } static inline int param_set_uint(const char *val, struct kernel_param *kp) { return 0; } static inline int param_get_uint(char *buffer, struct kernel_param *kp) { return 0; } static inline int param_set_long(const char *val, struct kernel_param *kp) { return 0; } static inline int param_get_long(char *buffer, struct kernel_param *kp) { return 0; } static inline int param_set_ulong(const char *val, struct kernel_param *kp) { return 0; } static inline int param_get_ulong(char *buffer, struct kernel_param *kp) { return 0; } static inline int param_set_charp(const char *val, struct kernel_param *kp) { return 0; } static inline int param_get_charp(char *buffer, struct kernel_param *kp) { return 0; } static inline int param_set_bool(const char *val, struct kernel_param *kp) { return 0; } static inline int param_get_bool(char *buffer, struct kernel_param *kp) { return 0; } #endif /* _LINUX_MODULEPARAM_H_ */ Index: stable/9/sys/ofed/include/linux/mount.h =================================================================== --- stable/9/sys/ofed/include/linux/mount.h (revision 273863) +++ stable/9/sys/ofed/include/linux/mount.h (revision 273864) @@ -1,33 +1,34 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_MOUNT_H_ #define _LINUX_MOUNT_H_ #endif /* _LINUX_MOUNT_H_ */ Index: stable/9/sys/ofed/include/linux/mutex.h =================================================================== --- stable/9/sys/ofed/include/linux/mutex.h (revision 273863) +++ stable/9/sys/ofed/include/linux/mutex.h (revision 273864) @@ -1,61 +1,62 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_MUTEX_H_ #define _LINUX_MUTEX_H_ #include #include #include #include typedef struct mutex { struct sx sx; } mutex_t; #define mutex_lock(_m) sx_xlock(&(_m)->sx) #define mutex_lock_nested(_m, _s) mutex_lock(_m) #define mutex_lock_interruptible(_m) ({ mutex_lock((_m)); 0; }) #define mutex_unlock(_m) sx_xunlock(&(_m)->sx) #define mutex_trylock(_m) !!sx_try_xlock(&(_m)->sx) #define DEFINE_MUTEX(lock) \ mutex_t lock; \ SX_SYSINIT_FLAGS(lock, &(lock).sx, "lnxmtx", SX_NOWITNESS) static inline void linux_mutex_init(mutex_t *m) { memset(&m->sx, 0, sizeof(m->sx)); sx_init_flags(&m->sx, "lnxmtx", SX_NOWITNESS); } #define mutex_init linux_mutex_init #endif /* _LINUX_MUTEX_H_ */ Index: stable/9/sys/ofed/include/linux/net.h =================================================================== --- stable/9/sys/ofed/include/linux/net.h (revision 273863) +++ stable/9/sys/ofed/include/linux/net.h (revision 273864) @@ -1,73 +1,74 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_NET_H_ #define _LINUX_NET_H_ #include #include #include static inline int sock_create_kern(int family, int type, int proto, struct socket **res) { return -socreate(family, res, type, proto, curthread->td_ucred, curthread); } static inline int sock_getname(struct socket *so, struct sockaddr *addr, int *sockaddr_len, int peer) { - struct sockaddr **nam; + struct sockaddr *nam; int error; nam = NULL; if (peer) { if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) return (-ENOTCONN); - error = (*so->so_proto->pr_usrreqs->pru_peeraddr)(so, nam); + error = (*so->so_proto->pr_usrreqs->pru_peeraddr)(so, &nam); } else - error = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, nam); + error = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, &nam); if (error) return (-error); - *addr = **nam; + *addr = *nam; *sockaddr_len = addr->sa_len; - free(*nam, M_SONAME); + free(nam, M_SONAME); return (0); } static inline void sock_release(struct socket *so) { soclose(so); } #endif /* _LINUX_NET_H_ */ Index: stable/9/sys/ofed/include/linux/netdevice.h =================================================================== --- stable/9/sys/ofed/include/linux/netdevice.h (revision 273863) +++ stable/9/sys/ofed/include/linux/netdevice.h (revision 273864) @@ -1,159 +1,204 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_NETDEVICE_H_ #define _LINUX_NETDEVICE_H_ #include #include #include #include #include #include #include #include #include #include #include #include struct net { }; extern struct net init_net; #define MAX_ADDR_LEN 20 #define net_device ifnet #define dev_get_by_index(n, idx) ifnet_byindex_ref((idx)) #define dev_hold(d) if_ref((d)) #define dev_put(d) if_rele((d)) #define netif_running(dev) !!((dev)->if_drv_flags & IFF_DRV_RUNNING) #define netif_oper_up(dev) !!((dev)->if_flags & IFF_UP) #define netif_carrier_ok(dev) netif_running(dev) static inline void * netdev_priv(const struct net_device *dev) { return (dev->if_softc); } static inline void _handle_ifnet_link_event(void *arg, struct ifnet *ifp, int linkstate) { struct notifier_block *nb; nb = arg; if (linkstate == LINK_STATE_UP) nb->notifier_call(nb, NETDEV_UP, ifp); else nb->notifier_call(nb, NETDEV_DOWN, ifp); } static inline void _handle_ifnet_arrival_event(void *arg, struct ifnet *ifp) { struct notifier_block *nb; nb = arg; nb->notifier_call(nb, NETDEV_REGISTER, ifp); } static inline void _handle_ifnet_departure_event(void *arg, struct ifnet *ifp) { struct notifier_block *nb; nb = arg; nb->notifier_call(nb, NETDEV_UNREGISTER, ifp); } +static inline void +_handle_iflladdr_event(void *arg, struct ifnet *ifp) +{ + struct notifier_block *nb; + + nb = arg; + nb->notifier_call(nb, NETDEV_CHANGEADDR, ifp); +} + +static inline void +_handle_ifaddr_event(void *arg, struct ifnet *ifp) +{ + struct notifier_block *nb; + + nb = arg; + nb->notifier_call(nb, NETDEV_CHANGEIFADDR, ifp); +} + static inline int register_netdevice_notifier(struct notifier_block *nb) { nb->tags[NETDEV_UP] = EVENTHANDLER_REGISTER( ifnet_link_event, _handle_ifnet_link_event, nb, 0); nb->tags[NETDEV_REGISTER] = EVENTHANDLER_REGISTER( ifnet_arrival_event, _handle_ifnet_arrival_event, nb, 0); nb->tags[NETDEV_UNREGISTER] = EVENTHANDLER_REGISTER( ifnet_departure_event, _handle_ifnet_departure_event, nb, 0); + nb->tags[NETDEV_CHANGEADDR] = EVENTHANDLER_REGISTER( + iflladdr_event, _handle_iflladdr_event, nb, 0); + return (0); } static inline int +register_inetaddr_notifier(struct notifier_block *nb) +{ + + nb->tags[NETDEV_CHANGEIFADDR] = EVENTHANDLER_REGISTER( + ifaddr_event, _handle_ifaddr_event, nb, 0); + return (0); +} + +static inline int unregister_netdevice_notifier(struct notifier_block *nb) { EVENTHANDLER_DEREGISTER(ifnet_link_event, nb->tags[NETDEV_UP]); EVENTHANDLER_DEREGISTER(ifnet_arrival_event, nb->tags[NETDEV_REGISTER]); EVENTHANDLER_DEREGISTER(ifnet_departure_event, nb->tags[NETDEV_UNREGISTER]); + EVENTHANDLER_DEREGISTER(iflladdr_event, + nb->tags[NETDEV_CHANGEADDR]); + return (0); } + +static inline int +unregister_inetaddr_notifier(struct notifier_block *nb) +{ + + EVENTHANDLER_DEREGISTER(ifaddr_event, + nb->tags[NETDEV_CHANGEIFADDR]); + + return (0); +} + #define rtnl_lock() #define rtnl_unlock() static inline int dev_mc_delete(struct net_device *dev, void *addr, int alen, int all) { struct sockaddr_dl sdl; if (alen > sizeof(sdl.sdl_data)) return (-EINVAL); memset(&sdl, 0, sizeof(sdl)); sdl.sdl_len = sizeof(sdl); sdl.sdl_family = AF_LINK; sdl.sdl_alen = alen; memcpy(&sdl.sdl_data, addr, alen); return -if_delmulti(dev, (struct sockaddr *)&sdl); } static inline int dev_mc_add(struct net_device *dev, void *addr, int alen, int newonly) { struct sockaddr_dl sdl; if (alen > sizeof(sdl.sdl_data)) return (-EINVAL); memset(&sdl, 0, sizeof(sdl)); sdl.sdl_len = sizeof(sdl); sdl.sdl_family = AF_LINK; sdl.sdl_alen = alen; memcpy(&sdl.sdl_data, addr, alen); return -if_addmulti(dev, (struct sockaddr *)&sdl, NULL); } #endif /* _LINUX_NETDEVICE_H_ */ Index: stable/9/sys/ofed/include/linux/notifier.h =================================================================== --- stable/9/sys/ofed/include/linux/notifier.h (revision 273863) +++ stable/9/sys/ofed/include/linux/notifier.h (revision 273864) @@ -1,54 +1,57 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_NOTIFIER_H_ #define _LINUX_NOTIFIER_H_ #include /* * Max number of FreeBSD events to map to Linux events per notify type. */ #define NOTIFY_DONE 0 -#define _NOTIFY_COUNT 5 +#define _NOTIFY_COUNT 7 struct notifier_block { int (*notifier_call)(struct notifier_block *, unsigned long, void *); struct notifier_block *next; int priority; eventhandler_tag tags[_NOTIFY_COUNT]; }; /* Values must be less than NOTIFY_COUNT */ #define NETDEV_UP 0x0001 #define NETDEV_DOWN 0x0002 #define NETDEV_REGISTER 0x0003 #define NETDEV_UNREGISTER 0x0004 +#define NETDEV_CHANGEADDR 0x0005 +#define NETDEV_CHANGEIFADDR 0x0006 #endif /* _LINUX_NOTIFIER_H_ */ Index: stable/9/sys/ofed/include/linux/page.h =================================================================== --- stable/9/sys/ofed/include/linux/page.h (revision 273863) +++ stable/9/sys/ofed/include/linux/page.h (revision 273864) @@ -1,49 +1,50 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_PAGE_H_ #define _LINUX_PAGE_H_ #include #include #include #include #define page vm_page #define virt_to_page(x) PHYS_TO_VM_PAGE(vtophys((x))) #define clear_page(page) memset((page), 0, PAGE_SIZE) #define pgprot_noncached(prot) VM_MEMATTR_UNCACHEABLE #define pgprot_writecombine(prot) VM_MEMATTR_WRITE_COMBINING #undef PAGE_MASK #define PAGE_MASK (~(PAGE_SIZE-1)) #endif /* _LINUX_PAGE_H_ */ Index: stable/9/sys/ofed/include/linux/pci.h =================================================================== --- stable/9/sys/ofed/include/linux/pci.h (revision 273863) +++ stable/9/sys/ofed/include/linux/pci.h (revision 273864) @@ -1,582 +1,830 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_PCI_H_ #define _LINUX_PCI_H_ #define CONFIG_PCI_MSI #include #include #include #include #include #include #include #include #include -#include #include #include #include #include #include #include #include struct pci_device_id { uint32_t vendor; uint32_t device; uint32_t subvendor; uint32_t subdevice; uint32_t class_mask; uintptr_t driver_data; }; #define MODULE_DEVICE_TABLE(bus, table) #define PCI_ANY_ID (-1) #define PCI_VENDOR_ID_MELLANOX 0x15b3 #define PCI_VENDOR_ID_TOPSPIN 0x1867 #define PCI_DEVICE_ID_MELLANOX_TAVOR 0x5a44 #define PCI_DEVICE_ID_MELLANOX_TAVOR_BRIDGE 0x5a46 #define PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT 0x6278 #define PCI_DEVICE_ID_MELLANOX_ARBEL 0x6282 #define PCI_DEVICE_ID_MELLANOX_SINAI_OLD 0x5e8c #define PCI_DEVICE_ID_MELLANOX_SINAI 0x6274 +#define PCI_DEVFN(slot, func) ((((slot) & 0x1f) << 3) | ((func) & 0x07)) +#define PCI_SLOT(devfn) (((devfn) >> 3) & 0x1f) +#define PCI_FUNC(devfn) ((devfn) & 0x07) #define PCI_VDEVICE(_vendor, _device) \ .vendor = PCI_VENDOR_ID_##_vendor, .device = (_device), \ .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID #define PCI_DEVICE(_vendor, _device) \ .vendor = (_vendor), .device = (_device), \ .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID #define to_pci_dev(n) container_of(n, struct pci_dev, dev) -#define PCI_VENDOR_ID PCIR_DEVVENDOR -#define PCI_COMMAND PCIR_COMMAND -#define PCI_EXP_DEVCTL PCIER_DEVICE_CTL -#define PCI_EXP_LNKCTL PCIER_LINK_CTL +#define PCI_VENDOR_ID PCIR_DEVVENDOR +#define PCI_COMMAND PCIR_COMMAND +#define PCI_EXP_DEVCTL PCIER_DEVICE_CTL /* Device Control */ +#define PCI_EXP_LNKCTL PCIER_LINK_CTL /* Link Control */ +#define PCI_EXP_FLAGS_TYPE PCIEM_FLAGS_TYPE /* Device/Port type */ +#define PCI_EXP_DEVCAP PCIER_DEVICE_CAP /* Device capabilities */ +#define PCI_EXP_DEVSTA PCIER_DEVICE_STA /* Device Status */ +#define PCI_EXP_LNKCAP PCIER_LINK_CAP /* Link Capabilities */ +#define PCI_EXP_LNKSTA PCIER_LINK_STA /* Link Status */ +#define PCI_EXP_SLTCAP PCIER_SLOT_CAP /* Slot Capabilities */ +#define PCI_EXP_SLTCTL PCIER_SLOT_CTL /* Slot Control */ +#define PCI_EXP_SLTSTA PCIER_SLOT_STA /* Slot Status */ +#define PCI_EXP_RTCTL PCIER_ROOT_CTL /* Root Control */ +#define PCI_EXP_RTCAP PCIER_ROOT_CAP /* Root Capabilities */ +#define PCI_EXP_RTSTA PCIER_ROOT_STA /* Root Status */ +#define PCI_EXP_DEVCAP2 PCIER_DEVICE_CAP2 /* Device Capabilities 2 */ +#define PCI_EXP_DEVCTL2 PCIER_DEVICE_CTL2 /* Device Control 2 */ +#define PCI_EXP_LNKCAP2 PCIER_LINK_CAP2 /* Link Capabilities 2 */ +#define PCI_EXP_LNKCTL2 PCIER_LINK_CTL2 /* Link Control 2 */ +#define PCI_EXP_LNKSTA2 PCIER_LINK_STA2 /* Link Status 2 */ +#define PCI_EXP_FLAGS PCIER_FLAGS /* Capabilities register */ +#define PCI_EXP_FLAGS_VERS PCIEM_FLAGS_VERSION /* Capability version */ +#define PCI_EXP_TYPE_ROOT_PORT PCIEM_TYPE_ROOT_PORT /* Root Port */ +#define PCI_EXP_TYPE_ENDPOINT PCIEM_TYPE_ENDPOINT /* Express Endpoint */ +#define PCI_EXP_TYPE_LEG_END PCIEM_TYPE_LEGACY_ENDPOINT /* Legacy Endpoint */ +#define PCI_EXP_TYPE_DOWNSTREAM PCIEM_TYPE_DOWNSTREAM_PORT /* Downstream Port */ +#define PCI_EXP_FLAGS_SLOT PCIEM_FLAGS_SLOT /* Slot implemented */ +#define PCI_EXP_TYPE_RC_EC PCIEM_TYPE_ROOT_EC /* Root Complex Event Collector */ + #define IORESOURCE_MEM SYS_RES_MEMORY #define IORESOURCE_IO SYS_RES_IOPORT #define IORESOURCE_IRQ SYS_RES_IRQ struct pci_dev; + struct pci_driver { struct list_head links; char *name; - struct pci_device_id *id_table; + const struct pci_device_id *id_table; int (*probe)(struct pci_dev *dev, const struct pci_device_id *id); void (*remove)(struct pci_dev *dev); + int (*suspend) (struct pci_dev *dev, pm_message_t state); /* Device suspended */ + int (*resume) (struct pci_dev *dev); /* Device woken up */ driver_t driver; devclass_t bsdclass; + const struct pci_error_handlers *err_handler; }; extern struct list_head pci_drivers; extern struct list_head pci_devices; extern spinlock_t pci_lock; #define __devexit_p(x) x struct pci_dev { struct device dev; struct list_head links; struct pci_driver *pdrv; uint64_t dma_mask; uint16_t device; uint16_t vendor; unsigned int irq; + unsigned int devfn; + u8 revision; + struct pci_devinfo *bus; /* bus this device is on, equivalent to linux struct pci_bus */ }; static inline struct resource_list_entry * _pci_get_rle(struct pci_dev *pdev, int type, int rid) { struct pci_devinfo *dinfo; struct resource_list *rl; dinfo = device_get_ivars(pdev->dev.bsddev); rl = &dinfo->resources; return resource_list_find(rl, type, rid); } static inline struct resource_list_entry * _pci_get_bar(struct pci_dev *pdev, int bar) { struct resource_list_entry *rle; bar = PCIR_BAR(bar); if ((rle = _pci_get_rle(pdev, SYS_RES_MEMORY, bar)) == NULL) rle = _pci_get_rle(pdev, SYS_RES_IOPORT, bar); return (rle); } static inline struct device * _pci_find_irq_dev(unsigned int irq) { struct pci_dev *pdev; spin_lock(&pci_lock); list_for_each_entry(pdev, &pci_devices, links) { if (irq == pdev->dev.irq) break; if (irq >= pdev->dev.msix && irq < pdev->dev.msix_max) break; } spin_unlock(&pci_lock); if (pdev) return &pdev->dev; return (NULL); } static inline unsigned long pci_resource_start(struct pci_dev *pdev, int bar) { struct resource_list_entry *rle; if ((rle = _pci_get_bar(pdev, bar)) == NULL) return (0); return rle->start; } static inline unsigned long pci_resource_len(struct pci_dev *pdev, int bar) { struct resource_list_entry *rle; if ((rle = _pci_get_bar(pdev, bar)) == NULL) return (0); return rle->count; } /* * All drivers just seem to want to inspect the type not flags. */ static inline int pci_resource_flags(struct pci_dev *pdev, int bar) { struct resource_list_entry *rle; if ((rle = _pci_get_bar(pdev, bar)) == NULL) return (0); return rle->type; } static inline const char * pci_name(struct pci_dev *d) { return device_get_desc(d->dev.bsddev); } static inline void * pci_get_drvdata(struct pci_dev *pdev) { return dev_get_drvdata(&pdev->dev); } static inline void pci_set_drvdata(struct pci_dev *pdev, void *data) { dev_set_drvdata(&pdev->dev, data); } static inline int pci_enable_device(struct pci_dev *pdev) { pci_enable_io(pdev->dev.bsddev, SYS_RES_IOPORT); pci_enable_io(pdev->dev.bsddev, SYS_RES_MEMORY); return (0); } static inline void pci_disable_device(struct pci_dev *pdev) { } static inline int pci_set_master(struct pci_dev *pdev) { pci_enable_busmaster(pdev->dev.bsddev); return (0); } static inline int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name) { int rid; int type; type = pci_resource_flags(pdev, bar); if (type == 0) return (-ENODEV); rid = PCIR_BAR(bar); if (bus_alloc_resource_any(pdev->dev.bsddev, type, &rid, RF_ACTIVE) == NULL) return (-EINVAL); return (0); } static inline void pci_release_region(struct pci_dev *pdev, int bar) { struct resource_list_entry *rle; if ((rle = _pci_get_bar(pdev, bar)) == NULL) return; bus_release_resource(pdev->dev.bsddev, rle->type, rle->rid, rle->res); } static inline void pci_release_regions(struct pci_dev *pdev) { int i; for (i = 0; i <= PCIR_MAX_BAR_0; i++) pci_release_region(pdev, i); } static inline int pci_request_regions(struct pci_dev *pdev, const char *res_name) { int error; int i; for (i = 0; i <= PCIR_MAX_BAR_0; i++) { error = pci_request_region(pdev, i, res_name); if (error && error != -ENODEV) { pci_release_regions(pdev); return (error); } } return (0); } static inline void pci_disable_msix(struct pci_dev *pdev) { pci_release_msi(pdev->dev.bsddev); } #define PCI_CAP_ID_EXP PCIY_EXPRESS #define PCI_CAP_ID_PCIX PCIY_PCIX + static inline int pci_find_capability(struct pci_dev *pdev, int capid) { int reg; if (pci_find_cap(pdev->dev.bsddev, capid, ®)) return (0); return (reg); } + + + +/** + * pci_pcie_cap - get the saved PCIe capability offset + * @dev: PCI device + * + * PCIe capability offset is calculated at PCI device initialization + * time and saved in the data structure. This function returns saved + * PCIe capability offset. Using this instead of pci_find_capability() + * reduces unnecessary search in the PCI configuration space. If you + * need to calculate PCIe capability offset from raw device for some + * reasons, please use pci_find_capability() instead. + */ +static inline int pci_pcie_cap(struct pci_dev *dev) +{ + return pci_find_capability(dev, PCI_CAP_ID_EXP); +} + + static inline int pci_read_config_byte(struct pci_dev *pdev, int where, u8 *val) { *val = (u8)pci_read_config(pdev->dev.bsddev, where, 1); return (0); } static inline int pci_read_config_word(struct pci_dev *pdev, int where, u16 *val) { *val = (u16)pci_read_config(pdev->dev.bsddev, where, 2); return (0); } static inline int pci_read_config_dword(struct pci_dev *pdev, int where, u32 *val) { *val = (u32)pci_read_config(pdev->dev.bsddev, where, 4); return (0); } static inline int pci_write_config_byte(struct pci_dev *pdev, int where, u8 val) { pci_write_config(pdev->dev.bsddev, where, val, 1); return (0); } static inline int pci_write_config_word(struct pci_dev *pdev, int where, u16 val) { pci_write_config(pdev->dev.bsddev, where, val, 2); return (0); } static inline int pci_write_config_dword(struct pci_dev *pdev, int where, u32 val) { pci_write_config(pdev->dev.bsddev, where, val, 4); return (0); } static struct pci_driver * -linux_pci_find(device_t dev, struct pci_device_id **idp) +linux_pci_find(device_t dev, const struct pci_device_id **idp) { - struct pci_device_id *id; + const struct pci_device_id *id; struct pci_driver *pdrv; uint16_t vendor; uint16_t device; vendor = pci_get_vendor(dev); device = pci_get_device(dev); spin_lock(&pci_lock); list_for_each_entry(pdrv, &pci_drivers, links) { for (id = pdrv->id_table; id->vendor != 0; id++) { if (vendor == id->vendor && device == id->device) { *idp = id; spin_unlock(&pci_lock); return (pdrv); } } } spin_unlock(&pci_lock); return (NULL); } static inline int linux_pci_probe(device_t dev) { - struct pci_device_id *id; + const struct pci_device_id *id; struct pci_driver *pdrv; if ((pdrv = linux_pci_find(dev, &id)) == NULL) return (ENXIO); if (device_get_driver(dev) != &pdrv->driver) return (ENXIO); device_set_desc(dev, pdrv->name); return (0); } static inline int linux_pci_attach(device_t dev) { struct resource_list_entry *rle; struct pci_dev *pdev; struct pci_driver *pdrv; - struct pci_device_id *id; + const struct pci_device_id *id; int error; pdrv = linux_pci_find(dev, &id); pdev = device_get_softc(dev); pdev->dev.parent = &linux_rootdev; pdev->dev.bsddev = dev; INIT_LIST_HEAD(&pdev->dev.irqents); pdev->device = id->device; pdev->vendor = id->vendor; pdev->dev.dma_mask = &pdev->dma_mask; pdev->pdrv = pdrv; kobject_init(&pdev->dev.kobj, &dev_ktype); kobject_set_name(&pdev->dev.kobj, device_get_nameunit(dev)); kobject_add(&pdev->dev.kobj, &linux_rootdev.kobj, kobject_name(&pdev->dev.kobj)); rle = _pci_get_rle(pdev, SYS_RES_IRQ, 0); if (rle) pdev->dev.irq = rle->start; else pdev->dev.irq = 0; pdev->irq = pdev->dev.irq; mtx_unlock(&Giant); spin_lock(&pci_lock); list_add(&pdev->links, &pci_devices); spin_unlock(&pci_lock); error = pdrv->probe(pdev, id); mtx_lock(&Giant); if (error) { spin_lock(&pci_lock); list_del(&pdev->links); spin_unlock(&pci_lock); put_device(&pdev->dev); return (-error); } return (0); } static inline int linux_pci_detach(device_t dev) { struct pci_dev *pdev; pdev = device_get_softc(dev); mtx_unlock(&Giant); pdev->pdrv->remove(pdev); mtx_lock(&Giant); spin_lock(&pci_lock); list_del(&pdev->links); spin_unlock(&pci_lock); put_device(&pdev->dev); return (0); } static device_method_t pci_methods[] = { DEVMETHOD(device_probe, linux_pci_probe), DEVMETHOD(device_attach, linux_pci_attach), DEVMETHOD(device_detach, linux_pci_detach), {0, 0} }; static inline int pci_register_driver(struct pci_driver *pdrv) { devclass_t bus; int error; spin_lock(&pci_lock); list_add(&pdrv->links, &pci_drivers); spin_unlock(&pci_lock); bus = devclass_find("pci"); pdrv->driver.name = pdrv->name; pdrv->driver.methods = pci_methods; pdrv->driver.size = sizeof(struct pci_dev); mtx_lock(&Giant); error = devclass_add_driver(bus, &pdrv->driver, BUS_PASS_DEFAULT, &pdrv->bsdclass); mtx_unlock(&Giant); if (error) return (-error); return (0); } static inline void pci_unregister_driver(struct pci_driver *pdrv) { devclass_t bus; list_del(&pdrv->links); bus = devclass_find("pci"); mtx_lock(&Giant); devclass_delete_driver(bus, &pdrv->driver); mtx_unlock(&Giant); } struct msix_entry { int entry; int vector; }; /* * Enable msix, positive errors indicate actual number of available * vectors. Negative errors are failures. */ static inline int pci_enable_msix(struct pci_dev *pdev, struct msix_entry *entries, int nreq) { struct resource_list_entry *rle; int error; int avail; int i; avail = pci_msix_count(pdev->dev.bsddev); if (avail < nreq) { if (avail == 0) return -EINVAL; return avail; } avail = nreq; if ((error = -pci_alloc_msix(pdev->dev.bsddev, &avail)) != 0) return error; rle = _pci_get_rle(pdev, SYS_RES_IRQ, 1); pdev->dev.msix = rle->start; pdev->dev.msix_max = rle->start + avail; for (i = 0; i < nreq; i++) entries[i].vector = pdev->dev.msix + i; return (0); } +static inline int pci_channel_offline(struct pci_dev *pdev) +{ + return false; +} + +static inline int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn) +{ + return -ENODEV; +} +static inline void pci_disable_sriov(struct pci_dev *dev) +{ +} + +/** + * DEFINE_PCI_DEVICE_TABLE - macro used to describe a pci device table + * @_table: device table name + * + * This macro is used to create a struct pci_device_id array (a device table) + * in a generic manner. + */ +#define DEFINE_PCI_DEVICE_TABLE(_table) \ + const struct pci_device_id _table[] __devinitdata + + /* XXX This should not be necessary. */ #define pcix_set_mmrbc(d, v) 0 #define pcix_get_max_mmrbc(d) 0 #define pcie_set_readrq(d, v) 0 #define PCI_DMA_BIDIRECTIONAL 0 #define PCI_DMA_TODEVICE 1 #define PCI_DMA_FROMDEVICE 2 #define PCI_DMA_NONE 3 #define pci_pool dma_pool #define pci_pool_destroy dma_pool_destroy #define pci_pool_alloc dma_pool_alloc #define pci_pool_free dma_pool_free #define pci_pool_create(_name, _pdev, _size, _align, _alloc) \ dma_pool_create(_name, &(_pdev)->dev, _size, _align, _alloc) #define pci_free_consistent(_hwdev, _size, _vaddr, _dma_handle) \ dma_free_coherent((_hwdev) == NULL ? NULL : &(_hwdev)->dev, \ _size, _vaddr, _dma_handle) #define pci_map_sg(_hwdev, _sg, _nents, _dir) \ dma_map_sg((_hwdev) == NULL ? NULL : &(_hwdev->dev), \ _sg, _nents, (enum dma_data_direction)_dir) #define pci_map_single(_hwdev, _ptr, _size, _dir) \ dma_map_single((_hwdev) == NULL ? NULL : &(_hwdev->dev), \ (_ptr), (_size), (enum dma_data_direction)_dir) #define pci_unmap_single(_hwdev, _addr, _size, _dir) \ dma_unmap_single((_hwdev) == NULL ? NULL : &(_hwdev)->dev, \ _addr, _size, (enum dma_data_direction)_dir) #define pci_unmap_sg(_hwdev, _sg, _nents, _dir) \ dma_unmap_sg((_hwdev) == NULL ? NULL : &(_hwdev)->dev, \ _sg, _nents, (enum dma_data_direction)_dir) #define pci_map_page(_hwdev, _page, _offset, _size, _dir) \ dma_map_page((_hwdev) == NULL ? NULL : &(_hwdev)->dev, _page,\ _offset, _size, (enum dma_data_direction)_dir) #define pci_unmap_page(_hwdev, _dma_address, _size, _dir) \ dma_unmap_page((_hwdev) == NULL ? NULL : &(_hwdev)->dev, \ _dma_address, _size, (enum dma_data_direction)_dir) #define pci_set_dma_mask(_pdev, mask) dma_set_mask(&(_pdev)->dev, (mask)) #define pci_dma_mapping_error(_pdev, _dma_addr) \ dma_mapping_error(&(_pdev)->dev, _dma_addr) #define pci_set_consistent_dma_mask(_pdev, _mask) \ dma_set_coherent_mask(&(_pdev)->dev, (_mask)) #define DECLARE_PCI_UNMAP_ADDR(x) DEFINE_DMA_UNMAP_ADDR(x); #define DECLARE_PCI_UNMAP_LEN(x) DEFINE_DMA_UNMAP_LEN(x); #define pci_unmap_addr dma_unmap_addr #define pci_unmap_addr_set dma_unmap_addr_set #define pci_unmap_len dma_unmap_len #define pci_unmap_len_set dma_unmap_len_set + +typedef unsigned int __bitwise pci_channel_state_t; +typedef unsigned int __bitwise pci_ers_result_t; + +enum pci_channel_state { + /* I/O channel is in normal state */ + pci_channel_io_normal = (__force pci_channel_state_t) 1, + + /* I/O to channel is blocked */ + pci_channel_io_frozen = (__force pci_channel_state_t) 2, + + /* PCI card is dead */ + pci_channel_io_perm_failure = (__force pci_channel_state_t) 3, +}; + +enum pci_ers_result { + /* no result/none/not supported in device driver */ + PCI_ERS_RESULT_NONE = (__force pci_ers_result_t) 1, + + /* Device driver can recover without slot reset */ + PCI_ERS_RESULT_CAN_RECOVER = (__force pci_ers_result_t) 2, + + /* Device driver wants slot to be reset. */ + PCI_ERS_RESULT_NEED_RESET = (__force pci_ers_result_t) 3, + + /* Device has completely failed, is unrecoverable */ + PCI_ERS_RESULT_DISCONNECT = (__force pci_ers_result_t) 4, + + /* Device driver is fully recovered and operational */ + PCI_ERS_RESULT_RECOVERED = (__force pci_ers_result_t) 5, +}; + + +/* PCI bus error event callbacks */ +struct pci_error_handlers { + /* PCI bus error detected on this device */ + pci_ers_result_t (*error_detected)(struct pci_dev *dev, + enum pci_channel_state error); + + /* MMIO has been re-enabled, but not DMA */ + pci_ers_result_t (*mmio_enabled)(struct pci_dev *dev); + + /* PCI Express link has been reset */ + pci_ers_result_t (*link_reset)(struct pci_dev *dev); + + /* PCI slot has been reset */ + pci_ers_result_t (*slot_reset)(struct pci_dev *dev); + + /* Device driver may resume normal operations */ + void (*resume)(struct pci_dev *dev); +}; + +/* freeBSD does not support SRIOV - yet */ +static inline struct pci_dev *pci_physfn(struct pci_dev *dev) +{ + return dev; +} + +static inline bool pci_is_pcie(struct pci_dev *dev) +{ + return !!pci_pcie_cap(dev); +} + +static inline u16 pcie_flags_reg(struct pci_dev *dev) +{ + int pos; + u16 reg16; + + pos = pci_find_capability(dev, PCI_CAP_ID_EXP); + if (!pos) + return 0; + + pci_read_config_word(dev, pos + PCI_EXP_FLAGS, ®16); + + return reg16; +} + + +static inline int pci_pcie_type(struct pci_dev *dev) +{ + return (pcie_flags_reg(dev) & PCI_EXP_FLAGS_TYPE) >> 4; +} + +static inline int pcie_cap_version(struct pci_dev *dev) +{ + return pcie_flags_reg(dev) & PCI_EXP_FLAGS_VERS; +} + +static inline bool pcie_cap_has_lnkctl(struct pci_dev *dev) +{ + int type = pci_pcie_type(dev); + + return pcie_cap_version(dev) > 1 || + type == PCI_EXP_TYPE_ROOT_PORT || + type == PCI_EXP_TYPE_ENDPOINT || + type == PCI_EXP_TYPE_LEG_END; +} + +static inline bool pcie_cap_has_devctl(const struct pci_dev *dev) +{ + return true; +} + +static inline bool pcie_cap_has_sltctl(struct pci_dev *dev) +{ + int type = pci_pcie_type(dev); + + return pcie_cap_version(dev) > 1 || + type == PCI_EXP_TYPE_ROOT_PORT || + (type == PCI_EXP_TYPE_DOWNSTREAM && + pcie_flags_reg(dev) & PCI_EXP_FLAGS_SLOT); +} + +static inline bool pcie_cap_has_rtctl(struct pci_dev *dev) +{ + int type = pci_pcie_type(dev); + + return pcie_cap_version(dev) > 1 || + type == PCI_EXP_TYPE_ROOT_PORT || + type == PCI_EXP_TYPE_RC_EC; +} + +static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos) +{ + if (!pci_is_pcie(dev)) + return false; + + switch (pos) { + case PCI_EXP_FLAGS_TYPE: + return true; + case PCI_EXP_DEVCAP: + case PCI_EXP_DEVCTL: + case PCI_EXP_DEVSTA: + return pcie_cap_has_devctl(dev); + case PCI_EXP_LNKCAP: + case PCI_EXP_LNKCTL: + case PCI_EXP_LNKSTA: + return pcie_cap_has_lnkctl(dev); + case PCI_EXP_SLTCAP: + case PCI_EXP_SLTCTL: + case PCI_EXP_SLTSTA: + return pcie_cap_has_sltctl(dev); + case PCI_EXP_RTCTL: + case PCI_EXP_RTCAP: + case PCI_EXP_RTSTA: + return pcie_cap_has_rtctl(dev); + case PCI_EXP_DEVCAP2: + case PCI_EXP_DEVCTL2: + case PCI_EXP_LNKCAP2: + case PCI_EXP_LNKCTL2: + case PCI_EXP_LNKSTA2: + return pcie_cap_version(dev) > 1; + default: + return false; + } +} + + +static inline int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val) +{ + if (pos & 1) + return -EINVAL; + + if (!pcie_capability_reg_implemented(dev, pos)) + return 0; + + return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val); +} #endif /* _LINUX_PCI_H_ */ Index: stable/9/sys/ofed/include/linux/poll.h =================================================================== --- stable/9/sys/ofed/include/linux/poll.h (revision 273863) +++ stable/9/sys/ofed/include/linux/poll.h (revision 273864) @@ -1,44 +1,45 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_POLL_H_ #define _LINUX_POLL_H_ #include #include typedef struct poll_table_struct { } poll_table; static inline void poll_wait(struct file *filp, wait_queue_head_t *wait_address, poll_table *p) { selrecord(curthread, &filp->f_selinfo); } #endif /* _LINUX_POLL_H_ */ Index: stable/9/sys/ofed/include/linux/radix-tree.h =================================================================== --- stable/9/sys/ofed/include/linux/radix-tree.h (revision 273863) +++ stable/9/sys/ofed/include/linux/radix-tree.h (revision 273864) @@ -1,60 +1,61 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_RADIX_TREE_H_ #define _LINUX_RADIX_TREE_H_ #define RADIX_TREE_MAP_SHIFT 6 #define RADIX_TREE_MAP_SIZE (1 << RADIX_TREE_MAP_SHIFT) #define RADIX_TREE_MAP_MASK (RADIX_TREE_MAP_SIZE - 1) #define RADIX_TREE_MAX_HEIGHT \ DIV_ROUND_UP((sizeof(long) * NBBY), RADIX_TREE_MAP_SHIFT) struct radix_tree_node { void *slots[RADIX_TREE_MAP_SIZE]; int count; }; struct radix_tree_root { struct radix_tree_node *rnode; gfp_t gfp_mask; int height; }; #define RADIX_TREE_INIT(mask) \ { .rnode = NULL, .gfp_mask = mask, .height = 0 }; #define INIT_RADIX_TREE(root, mask) \ { (root)->rnode = NULL; (root)->gfp_mask = mask; (root)->height = 0; } #define RADIX_TREE(name, mask) \ struct radix_tree_root name = RADIX_TREE_INIT(mask) void *radix_tree_lookup(struct radix_tree_root *, unsigned long); void *radix_tree_delete(struct radix_tree_root *, unsigned long); int radix_tree_insert(struct radix_tree_root *, unsigned long, void *); #endif /* _LINUX_RADIX_TREE_H_ */ Index: stable/9/sys/ofed/include/linux/random.h =================================================================== --- stable/9/sys/ofed/include/linux/random.h (revision 273863) +++ stable/9/sys/ofed/include/linux/random.h (revision 273864) @@ -1,40 +1,41 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_RANDOM_H_ #define _LINUX_RANDOM_H_ #include static inline void get_random_bytes(void *buf, int nbytes) { read_random(buf, nbytes); } #endif /* _LINUX_RANDOM_H_ */ Index: stable/9/sys/ofed/include/linux/rbtree.h =================================================================== --- stable/9/sys/ofed/include/linux/rbtree.h (revision 273863) +++ stable/9/sys/ofed/include/linux/rbtree.h (revision 273864) @@ -1,111 +1,112 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_RBTREE_H_ #define _LINUX_RBTREE_H_ #include #include struct rb_node { RB_ENTRY(rb_node) __entry; }; #define rb_left __entry.rbe_left #define rb_right __entry.rbe_right /* * We provide a false structure that has the same bit pattern as tree.h * presents so it matches the member names expected by linux. */ struct rb_root { struct rb_node *rb_node; }; /* * In linux all of the comparisons are done by the caller. */ int panic_cmp(struct rb_node *one, struct rb_node *two); RB_HEAD(linux_root, rb_node); RB_PROTOTYPE(linux_root, rb_node, __entry, panic_cmp); #define rb_parent(r) RB_PARENT(r, __entry) #define rb_color(r) RB_COLOR(r, __entry) #define rb_is_red(r) (rb_color(r) == RB_RED) #define rb_is_black(r) (rb_color(r) == RB_BLACK) #define rb_set_parent(r, p) rb_parent((r)) = (p) #define rb_set_color(r, c) rb_color((r)) = (c) #define rb_entry(ptr, type, member) container_of(ptr, type, member) #define RB_EMPTY_ROOT(root) RB_EMPTY((struct linux_root *)root) #define RB_EMPTY_NODE(node) (rb_parent(node) == node) #define RB_CLEAR_NODE(node) (rb_set_parent(node, node)) #define rb_insert_color(node, root) \ linux_root_RB_INSERT_COLOR((struct linux_root *)(root), (node)) #define rb_erase(node, root) \ linux_root_RB_REMOVE((struct linux_root *)(root), (node)) #define rb_next(node) RB_NEXT(linux_root, NULL, (node)) #define rb_prev(node) RB_PREV(linux_root, NULL, (node)) #define rb_first(root) RB_MIN(linux_root, (struct linux_root *)(root)) #define rb_last(root) RB_MAX(linux_root, (struct linux_root *)(root)) static inline void rb_link_node(struct rb_node *node, struct rb_node *parent, struct rb_node **rb_link) { rb_set_parent(node, parent); rb_set_color(node, RB_RED); node->__entry.rbe_left = node->__entry.rbe_right = NULL; *rb_link = node; } static inline void rb_replace_node(struct rb_node *victim, struct rb_node *new, struct rb_root *root) { struct rb_node *p; p = rb_parent(victim); if (p) { if (p->rb_left == victim) p->rb_left = new; else p->rb_right = new; } else root->rb_node = new; if (victim->rb_left) rb_set_parent(victim->rb_left, new); if (victim->rb_right) rb_set_parent(victim->rb_right, new); *new = *victim; } #undef RB_ROOT #define RB_ROOT (struct rb_root) { NULL } #endif /* _LINUX_RBTREE_H_ */ Index: stable/9/sys/ofed/include/linux/rwlock.h =================================================================== --- stable/9/sys/ofed/include/linux/rwlock.h (revision 273863) +++ stable/9/sys/ofed/include/linux/rwlock.h (revision 273864) @@ -1,63 +1,64 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_RWLOCK_H_ #define _LINUX_RWLOCK_H_ #include #include typedef struct { struct rwlock rw; } rwlock_t; #define read_lock(_l) rw_rlock(&(_l)->rw) #define write_lock(_l) rw_wlock(&(_l)->rw) #define read_unlock(_l) rw_runlock(&(_l)->rw) #define write_unlock(_l) rw_wunlock(&(_l)->rw) #define read_lock_irq(lock) read_lock((lock)) #define read_unlock_irq(lock) read_unlock((lock)) #define write_lock_irq(lock) write_lock((lock)) #define write_unlock_irq(lock) write_unlock((lock)) #define read_lock_irqsave(lock, flags) \ do {(flags) = 0; read_lock(lock); } while (0) #define write_lock_irqsave(lock, flags) \ do {(flags) = 0; write_lock(lock); } while (0) #define read_unlock_irqrestore(lock, flags) \ do { read_unlock(lock); } while (0) #define write_unlock_irqrestore(lock, flags) \ do { write_unlock(lock); } while (0) static inline void rwlock_init(rwlock_t *lock) { memset(&lock->rw, 0, sizeof(lock->rw)); rw_init_flags(&lock->rw, "lnxrw", RW_NOWITNESS); } #endif /* _LINUX_RWLOCK_H_ */ Index: stable/9/sys/ofed/include/linux/rwsem.h =================================================================== --- stable/9/sys/ofed/include/linux/rwsem.h (revision 273863) +++ stable/9/sys/ofed/include/linux/rwsem.h (revision 273864) @@ -1,56 +1,57 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_RWSEM_H_ #define _LINUX_RWSEM_H_ #include #include #include struct rw_semaphore { struct sx sx; }; #define down_write(_rw) sx_xlock(&(_rw)->sx) #define up_write(_rw) sx_xunlock(&(_rw)->sx) #define down_read(_rw) sx_slock(&(_rw)->sx) #define up_read(_rw) sx_sunlock(&(_rw)->sx) #define down_read_trylock(_rw) !!sx_try_slock(&(_rw)->sx) #define down_write_trylock(_rw) !!sx_try_xlock(&(_rw)->sx) #define downgrade_write(_rw) sx_downgrade(&(_rw)->sx) #define down_read_nested(_rw, _sc) down_read(_rw) static inline void init_rwsem(struct rw_semaphore *rw) { memset(&rw->sx, 0, sizeof(rw->sx)); sx_init_flags(&rw->sx, "lnxrwsem", SX_NOWITNESS); } #endif /* _LINUX_RWSEM_H_ */ Index: stable/9/sys/ofed/include/linux/scatterlist.h =================================================================== --- stable/9/sys/ofed/include/linux/scatterlist.h (revision 273863) +++ stable/9/sys/ofed/include/linux/scatterlist.h (revision 273864) @@ -1,98 +1,105 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef _LINUX_SCATTERLIST_H_ #define _LINUX_SCATTERLIST_H_ -#include #include struct scatterlist { union { struct page *page; struct scatterlist *sg; } sl_un; dma_addr_t address; unsigned long offset; uint32_t length; uint32_t flags; +}; + +struct sg_table { + struct scatterlist *sgl; /* the list */ + unsigned int nents; /* number of mapped entries */ + unsigned int orig_nents; /* original size of list */ }; #define sg_dma_address(sg) (sg)->address #define sg_dma_len(sg) (sg)->length #define sg_page(sg) (sg)->sl_un.page #define sg_scatternext(sg) (sg)->sl_un.sg #define SG_END 0x01 #define SG_CHAIN 0x02 static inline void sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, unsigned int offset) { sg_page(sg) = page; sg_dma_len(sg) = len; sg->offset = offset; if (offset > PAGE_SIZE) panic("sg_set_page: Invalid offset %d\n", offset); } static inline void sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) { sg_set_page(sg, virt_to_page(buf), buflen, ((uintptr_t)buf) & ~PAGE_MASK); } static inline void sg_init_table(struct scatterlist *sg, unsigned int nents) { bzero(sg, sizeof(*sg) * nents); sg[nents - 1].flags = SG_END; } static inline struct scatterlist * sg_next(struct scatterlist *sg) { if (sg->flags & SG_END) return (NULL); sg++; if (sg->flags & SG_CHAIN) sg = sg_scatternext(sg); return (sg); } static inline vm_paddr_t sg_phys(struct scatterlist *sg) { return sg_page(sg)->phys_addr + sg->offset; } #define for_each_sg(sglist, sg, sgmax, _itr) \ for (_itr = 0, sg = (sglist); _itr < (sgmax); _itr++, sg = sg_next(sg)) #endif /* _LINUX_SCATTERLIST_H_ */ Index: stable/9/sys/ofed/include/linux/sched.h =================================================================== --- stable/9/sys/ofed/include/linux/sched.h (revision 273863) +++ stable/9/sys/ofed/include/linux/sched.h (revision 273864) @@ -1,109 +1,110 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_SCHED_H_ #define _LINUX_SCHED_H_ #include #include #include #include #include #define MAX_SCHEDULE_TIMEOUT LONG_MAX #define TASK_RUNNING 0 #define TASK_INTERRUPTIBLE 1 #define TASK_UNINTERRUPTIBLE 2 #define TASK_DEAD 64 #define TASK_WAKEKILL 128 #define TASK_WAKING 256 #define TASK_SHOULD_STOP 1 #define TASK_STOPPED 2 /* * A task_struct is only provided for those tasks created with kthread. * Using these routines with threads not started via kthread will cause * panics because no task_struct is allocated and td_retval[1] is * overwritten by syscalls which kernel threads will not make use of. */ struct task_struct { struct thread *task_thread; int (*task_fn)(void *data); void *task_data; int task_ret; int state; int should_stop; }; #define current ((struct task_struct *)curthread->td_retval[1]) #define task_struct_get(x) (struct task_struct *)(x)->td_retval[1] #define task_struct_set(x, y) (x)->td_retval[1] = (register_t)(y) #define set_current_state(x) \ atomic_store_rel_int((volatile int *)¤t->state, (x)) #define __set_current_state(x) current->state = (x) #define schedule() \ do { \ void *c; \ \ if (cold) \ break; \ c = curthread; \ sleepq_lock(c); \ if (current->state == TASK_INTERRUPTIBLE || \ current->state == TASK_UNINTERRUPTIBLE) { \ sleepq_add(c, NULL, "task", SLEEPQ_SLEEP, 0); \ sleepq_wait(c, 0); \ } else { \ sleepq_release(c); \ sched_relinquish(curthread); \ } \ } while (0) #define wake_up_process(x) \ do { \ int wakeup_swapper; \ void *c; \ \ c = (x)->task_thread; \ sleepq_lock(c); \ (x)->state = TASK_RUNNING; \ wakeup_swapper = sleepq_signal(c, SLEEPQ_SLEEP, 0, 0); \ sleepq_release(c); \ if (wakeup_swapper) \ kick_proc0(); \ } while (0) #define cond_resched() if (!cold) sched_relinquish(curthread) #define sched_yield() sched_relinquish(curthread) #endif /* _LINUX_SCHED_H_ */ Index: stable/9/sys/ofed/include/linux/semaphore.h =================================================================== --- stable/9/sys/ofed/include/linux/semaphore.h (revision 273863) +++ stable/9/sys/ofed/include/linux/semaphore.h (revision 273864) @@ -1,66 +1,67 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_SEMAPHORE_H_ #define _LINUX_SEMAPHORE_H_ #include #include #include /* * XXX BSD semaphores are disused and slow. They also do not provide a * sema_wait_sig method. This must be resolved eventually. */ struct semaphore { struct sema sema; }; #define down(_sem) sema_wait(&(_sem)->sema) #define down_interruptible(_sem) sema_wait(&(_sem)->sema), 0 #define down_trylock(_sem) !sema_trywait(&(_sem)->sema) #define up(_sem) sema_post(&(_sem)->sema) static inline void linux_sema_init(struct semaphore *sem, int val) { memset(&sem->sema, 0, sizeof(sem->sema)); sema_init(&sem->sema, val, "lnxsema"); } static inline void init_MUTEX(struct semaphore *sem) { memset(&sem->sema, 0, sizeof(sem->sema)); sema_init(&sem->sema, 1, "lnxsema"); } #define sema_init linux_sema_init #endif /* _LINUX_SEMAPHORE_H_ */ Index: stable/9/sys/ofed/include/linux/slab.h =================================================================== --- stable/9/sys/ofed/include/linux/slab.h (revision 273863) +++ stable/9/sys/ofed/include/linux/slab.h (revision 273864) @@ -1,102 +1,108 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_SLAB_H_ #define _LINUX_SLAB_H_ #include #include #include #include #include #include MALLOC_DECLARE(M_KMALLOC); -#define kmalloc(size, flags) malloc((size), M_KMALLOC, (flags)) -#define kzalloc(size, flags) kmalloc((size), (flags) | M_ZERO) -#define kfree(ptr) free(__DECONST(void *, (ptr)), M_KMALLOC) -#define krealloc(ptr, size, flags) realloc((ptr), (size), M_KMALLOC, (flags)) -#define kcalloc(n, size, flags) kmalloc((n) * (size), flags | M_ZERO) +#define kmalloc(size, flags) malloc((size), M_KMALLOC, (flags)) +#define kzalloc(size, flags) kmalloc((size), (flags) | M_ZERO) +#define kzalloc_node(size, flags, node) kzalloc(size, flags) +#define kfree(ptr) free(__DECONST(void *, (ptr)), M_KMALLOC) +#define krealloc(ptr, size, flags) realloc((ptr), (size), M_KMALLOC, (flags)) +#define kcalloc(n, size, flags) kmalloc((n) * (size), flags | M_ZERO) +#define vzalloc(size) kzalloc(size, GFP_KERNEL | __GFP_NOWARN) +#define vfree(arg) kfree(arg) +#define vmalloc(size) kmalloc(size, GFP_KERNEL) +#define vmalloc_node(size, node) kmalloc(size, GFP_KERNEL) struct kmem_cache { uma_zone_t cache_zone; void (*cache_ctor)(void *); }; #define SLAB_HWCACHE_ALIGN 0x0001 static inline int kmem_ctor(void *mem, int size, void *arg, int flags) { void (*ctor)(void *); ctor = arg; ctor(mem); return (0); } static inline struct kmem_cache * kmem_cache_create(char *name, size_t size, size_t align, u_long flags, void (*ctor)(void *)) { struct kmem_cache *c; c = malloc(sizeof(*c), M_KMALLOC, M_WAITOK); if (align) align--; if (flags & SLAB_HWCACHE_ALIGN) align = UMA_ALIGN_CACHE; c->cache_zone = uma_zcreate(name, size, ctor ? kmem_ctor : NULL, NULL, NULL, NULL, align, 0); c->cache_ctor = ctor; return c; } static inline void * kmem_cache_alloc(struct kmem_cache *c, int flags) { return uma_zalloc_arg(c->cache_zone, c->cache_ctor, flags); } static inline void kmem_cache_free(struct kmem_cache *c, void *m) { uma_zfree(c->cache_zone, m); } static inline void kmem_cache_destroy(struct kmem_cache *c) { uma_zdestroy(c->cache_zone); free(c, M_KMALLOC); } #endif /* _LINUX_SLAB_H_ */ Index: stable/9/sys/ofed/include/linux/socket.h =================================================================== --- stable/9/sys/ofed/include/linux/socket.h (revision 273863) +++ stable/9/sys/ofed/include/linux/socket.h (revision 273864) @@ -1,66 +1,67 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_SOCKET_H_ #define _LINUX_SOCKET_H_ #include #ifdef notyet static inline int memcpy_toiovec(struct iovec *v, unsigned char *kdata, int len) { struct uio uio; int error; uio.uio_iov = v; uio.uio_iovcnt = -1; uio.uio_offset = 0; uio.uio_resid = len; uio.uio_segflag = UIO_USERSPACE; uio.uio_rw = UIO_READ; error = -uiomove(kdata, len, &uio); return (error); } static inline int memcpy_fromiovec(unsigned char *kdata, struct iovec *iov, int len) { struct uio uio; int error; uio.uio_iov = v; uio.uio_iovcnt = -1; uio.uio_offset = 0; uio.uio_resid = len; uio.uio_segflag = UIO_USERSPACE; uio.uio_rw = UIO_WRITE; error = -uiomove(kdata, len, &uio); } #endif #endif /* _LINUX_SOCKET_H_ */ Index: stable/9/sys/ofed/include/linux/spinlock.h =================================================================== --- stable/9/sys/ofed/include/linux/spinlock.h (revision 273863) +++ stable/9/sys/ofed/include/linux/spinlock.h (revision 273864) @@ -1,68 +1,68 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_SPINLOCK_H_ #define _LINUX_SPINLOCK_H_ #include #include #include #include #include #include -#include #include typedef struct { struct mtx m; } spinlock_t; #define spin_lock(_l) mtx_lock(&(_l)->m) #define spin_unlock(_l) mtx_unlock(&(_l)->m) #define spin_trylock(_l) mtx_trylock(&(_l)->m) #define spin_lock_nested(_l, _n) mtx_lock_flags(&(_l)->m, MTX_DUPOK) #define spin_lock_irq(lock) spin_lock(lock) #define spin_unlock_irq(lock) spin_unlock(lock) #define spin_lock_irqsave(lock, flags) \ do {(flags) = 0; spin_lock(lock); } while (0) #define spin_unlock_irqrestore(lock, flags) \ do { spin_unlock(lock); } while (0) static inline void spin_lock_init(spinlock_t *lock) { memset(&lock->m, 0, sizeof(lock->m)); mtx_init(&lock->m, "lnxspin", NULL, MTX_DEF | MTX_NOWITNESS); } #define DEFINE_SPINLOCK(lock) \ spinlock_t lock; \ MTX_SYSINIT(lock, &(lock).m, "lnxspin", MTX_DEF) #endif /* _LINUX_SPINLOCK_H_ */ Index: stable/9/sys/ofed/include/linux/string.h =================================================================== --- stable/9/sys/ofed/include/linux/string.h (revision 273863) +++ stable/9/sys/ofed/include/linux/string.h (revision 273864) @@ -1,49 +1,53 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_STRING_H_ #define _LINUX_STRING_H_ #include #include #include #include + +#define strnicmp strncasecmp + static inline void * kmemdup(const void *src, size_t len, gfp_t gfp) { void *dst; dst = kmalloc(len, gfp); if (dst) memcpy(dst, src, len); return (dst); } #endif /* _LINUX_STRING_H_ */ Index: stable/9/sys/ofed/include/linux/sysfs.h =================================================================== --- stable/9/sys/ofed/include/linux/sysfs.h (revision 273863) +++ stable/9/sys/ofed/include/linux/sysfs.h (revision 273864) @@ -1,185 +1,191 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_SYSFS_H_ #define _LINUX_SYSFS_H_ #include struct attribute { const char *name; struct module *owner; mode_t mode; }; struct sysfs_ops { ssize_t (*show)(struct kobject *, struct attribute *, char *); ssize_t (*store)(struct kobject *, struct attribute *, const char *, size_t); }; struct attribute_group { const char *name; mode_t (*is_visible)(struct kobject *, struct attribute *, int); struct attribute **attrs; }; #define __ATTR(_name, _mode, _show, _store) { \ .attr = { .name = __stringify(_name), .mode = _mode }, \ .show = _show, .store = _store, \ } #define __ATTR_RO(_name) { \ .attr = { .name = __stringify(_name), .mode = 0444 }, \ .show = _name##_show, \ } #define __ATTR_NULL { .attr = { .name = NULL } } /* * Handle our generic '\0' terminated 'C' string. * Two cases: * a variable string: point arg1 at it, arg2 is max length. * a constant string: point arg1 at it, arg2 is zero. */ static inline int sysctl_handle_attr(SYSCTL_HANDLER_ARGS) { struct kobject *kobj; struct attribute *attr; const struct sysfs_ops *ops; char *buf; int error; ssize_t len; kobj = arg1; attr = (struct attribute *)arg2; if (kobj->ktype == NULL || kobj->ktype->sysfs_ops == NULL) return (ENODEV); buf = (char *)get_zeroed_page(GFP_KERNEL); if (buf == NULL) return (ENOMEM); ops = kobj->ktype->sysfs_ops; if (ops->show) { len = ops->show(kobj, attr, buf); /* * It's valid to not have a 'show' so just return an * empty string. */ if (len < 0) { error = -len; if (error != EIO) goto out; + buf[0] = '\0'; + } else if (len) { + len--; + if (len >= PAGE_SIZE) + len = PAGE_SIZE - 1; + /* Trim trailing newline. */ + buf[len] = '\0'; } - - /* Trim trailing newline. */ - len--; - buf[len] = '\0'; } /* Leave one trailing byte to append a newline. */ error = sysctl_handle_string(oidp, buf, PAGE_SIZE - 1, req); if (error != 0 || req->newptr == NULL || ops->store == NULL) goto out; len = strlcat(buf, "\n", PAGE_SIZE); KASSERT(len < PAGE_SIZE, ("new attribute truncated")); len = ops->store(kobj, attr, buf, len); if (len < 0) error = -len; out: free_page((unsigned long)buf); return (error); } static inline int sysfs_create_file(struct kobject *kobj, const struct attribute *attr) { sysctl_add_oid(NULL, SYSCTL_CHILDREN(kobj->oidp), OID_AUTO, attr->name, CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_MPSAFE, kobj, (uintptr_t)attr, sysctl_handle_attr, "A", ""); return (0); } static inline void sysfs_remove_file(struct kobject *kobj, const struct attribute *attr) { if (kobj->oidp) sysctl_remove_name(kobj->oidp, attr->name, 1, 1); } static inline void sysfs_remove_group(struct kobject *kobj, const struct attribute_group *grp) { if (kobj->oidp) sysctl_remove_name(kobj->oidp, grp->name, 1, 1); } static inline int sysfs_create_group(struct kobject *kobj, const struct attribute_group *grp) { struct attribute **attr; struct sysctl_oid *oidp; oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(kobj->oidp), OID_AUTO, grp->name, CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, grp->name); for (attr = grp->attrs; *attr != NULL; attr++) { sysctl_add_oid(NULL, SYSCTL_CHILDREN(oidp), OID_AUTO, (*attr)->name, CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_MPSAFE, kobj, (uintptr_t)*attr, sysctl_handle_attr, "A", ""); } return (0); } static inline int sysfs_create_dir(struct kobject *kobj) { kobj->oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(kobj->parent->oidp), OID_AUTO, kobj->name, CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, kobj->name); return (0); } static inline void sysfs_remove_dir(struct kobject *kobj) { if (kobj->oidp == NULL) return; sysctl_remove_oid(kobj->oidp, 1, 1); } + +#define sysfs_attr_init(attr) do {} while(0) #endif /* _LINUX_SYSFS_H_ */ Index: stable/9/sys/ofed/include/linux/timer.h =================================================================== --- stable/9/sys/ofed/include/linux/timer.h (revision 273863) +++ stable/9/sys/ofed/include/linux/timer.h (revision 273864) @@ -1,87 +1,92 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_TIMER_H_ #define _LINUX_TIMER_H_ #include #include #include #include struct timer_list { struct callout timer_callout; void (*function)(unsigned long); - unsigned long data; + unsigned long data; + unsigned long expires; }; -#define expires timer_callout.c_time - static inline void _timer_fn(void *context) { struct timer_list *timer; timer = context; timer->function(timer->data); } #define setup_timer(timer, func, dat) \ do { \ (timer)->function = (func); \ (timer)->data = (dat); \ callout_init(&(timer)->timer_callout, CALLOUT_MPSAFE); \ } while (0) #define init_timer(timer) \ do { \ (timer)->function = NULL; \ (timer)->data = 0; \ callout_init(&(timer)->timer_callout, CALLOUT_MPSAFE); \ } while (0) -#define mod_timer(timer, expire) \ - callout_reset(&(timer)->timer_callout, (expire) - jiffies, \ - _timer_fn, (timer)) +#define mod_timer(timer, exp) \ +do { \ + (timer)->expires = (exp); \ + callout_reset(&(timer)->timer_callout, (exp) - jiffies, \ + _timer_fn, (timer)); \ +} while (0) #define add_timer(timer) \ callout_reset(&(timer)->timer_callout, \ - (timer)->timer_callout.c_time - jiffies, _timer_fn, (timer)) + (timer)->expires - jiffies, _timer_fn, (timer)) #define del_timer(timer) callout_stop(&(timer)->timer_callout) #define del_timer_sync(timer) callout_drain(&(timer)->timer_callout) #define timer_pending(timer) callout_pending(&(timer)->timer_callout) static inline unsigned long round_jiffies(unsigned long j) { return roundup(j, hz); } + +#define round_jiffies_relative(j) round_jiffies(j) #endif /* _LINUX_TIMER_H_ */ Index: stable/9/sys/ofed/include/linux/types.h =================================================================== --- stable/9/sys/ofed/include/linux/types.h (revision 273863) +++ stable/9/sys/ofed/include/linux/types.h (revision 273864) @@ -1,57 +1,66 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_TYPES_H_ #define _LINUX_TYPES_H_ #include #include +#include +#include #include #include -typedef __u16 __le16; -typedef __u16 __be16; -typedef __u32 __le32; -typedef __u32 __be32; -typedef __u64 __le64; -typedef __u64 __be64; -#ifndef __bool_true_false_are_defined -typedef _Bool bool; -#define true TRUE -#define false FALSE +#define __read_mostly __attribute__((__section__(".data.read_mostly"))) + +#ifndef __bitwise__ +#ifdef __CHECKER__ +#define __bitwise__ __attribute__((bitwise)) +#else +#define __bitwise__ #endif +#endif -typedef unsigned long kernel_ulong_t; +typedef uint16_t __le16; +typedef uint16_t __be16; +typedef uint32_t __le32; +typedef uint32_t __be32; +typedef uint64_t __le64; +typedef uint64_t __be64; + typedef unsigned int uint; typedef unsigned gfp_t; typedef uint64_t loff_t; typedef vm_paddr_t resource_size_t; + +typedef u64 phys_addr_t; #define DECLARE_BITMAP(n, bits) \ unsigned long n[howmany(bits, sizeof(long) * 8)] #endif /* _LINUX_TYPES_H_ */ Index: stable/9/sys/ofed/include/linux/uaccess.h =================================================================== --- stable/9/sys/ofed/include/linux/uaccess.h (revision 273863) +++ stable/9/sys/ofed/include/linux/uaccess.h (revision 273864) @@ -1,34 +1,35 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_UACCESS_H_ #define _LINUX_UACCESS_H_ #define get_user(_x, _p) -copyin((_p), &(_x), sizeof(*(_p))) #define put_user(_x, _p) -copyout(&(_x), (_p), sizeof(*(_p))) #endif /* _LINUX_UACCESS_H_ */ Index: stable/9/sys/ofed/include/linux/vmalloc.h =================================================================== --- stable/9/sys/ofed/include/linux/vmalloc.h (revision 273863) +++ stable/9/sys/ofed/include/linux/vmalloc.h (revision 273864) @@ -1,41 +1,42 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_VMALLOC_H_ #define _LINUX_VMALLOC_H_ #include #define VM_MAP 0x0000 #define PAGE_KERNEL 0x0000 void *vmap(struct page **pages, unsigned int count, unsigned long flags, int prot); void vunmap(void *addr); #endif /* _LINUX_VMALLOC_H_ */ Index: stable/9/sys/ofed/include/linux/wait.h =================================================================== --- stable/9/sys/ofed/include/linux/wait.h (revision 273863) +++ stable/9/sys/ofed/include/linux/wait.h (revision 273864) @@ -1,112 +1,113 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_WAIT_H_ #define _LINUX_WAIT_H_ #include #include #include #include #include #include #include #include struct __wait_queue_head { unsigned int wchan; }; typedef struct __wait_queue_head wait_queue_head_t; #define init_waitqueue_head(x) static inline void __wake_up(struct __wait_queue_head *q, int all) { int wakeup_swapper; void *c; c = &q->wchan; sleepq_lock(c); if (all) wakeup_swapper = sleepq_broadcast(c, SLEEPQ_SLEEP, 0, 0); else wakeup_swapper = sleepq_signal(c, SLEEPQ_SLEEP, 0, 0); sleepq_release(c); if (wakeup_swapper) kick_proc0(); } #define wake_up(q) __wake_up(q, 0) #define wake_up_nr(q, nr) __wake_up(q, 1) #define wake_up_all(q) __wake_up(q, 1) #define wake_up_interruptible(q) __wake_up(q, 0) #define wake_up_interruptible_nr(q, nr) __wake_up(q, 1) #define wake_up_interruptible_all(q, nr) __wake_up(q, 1) #define wait_event(q, cond) \ do { \ void *c = &(q).wchan; \ if (!(cond)) { \ for (;;) { \ sleepq_lock(c); \ if (cond) { \ sleepq_release(c); \ break; \ } \ sleepq_add(c, NULL, "completion", SLEEPQ_SLEEP, 0); \ sleepq_wait(c, 0); \ } \ } \ } while (0) #define wait_event_interruptible(q, cond) \ ({ \ void *c = &(q).wchan; \ int _error; \ \ _error = 0; \ if (!(cond)) { \ for (; _error == 0;) { \ sleepq_lock(c); \ if (cond) { \ sleepq_release(c); \ break; \ } \ sleepq_add(c, NULL, "completion", \ SLEEPQ_SLEEP | SLEEPQ_INTERRUPTIBLE, 0); \ if (sleepq_wait_sig(c, 0)) \ _error = -ERESTARTSYS; \ } \ } \ -_error; \ }) #define DEFINE_WAIT(x) #endif /* _LINUX_WAIT_H_ */ Index: stable/9/sys/ofed/include/linux/workqueue.h =================================================================== --- stable/9/sys/ofed/include/linux/workqueue.h (revision 273863) +++ stable/9/sys/ofed/include/linux/workqueue.h (revision 273864) @@ -1,193 +1,223 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_WORKQUEUE_H_ #define _LINUX_WORKQUEUE_H_ #include #include #include #include #include struct workqueue_struct { struct taskqueue *taskqueue; }; struct work_struct { struct task work_task; struct taskqueue *taskqueue; void (*fn)(struct work_struct *); }; struct delayed_work { struct work_struct work; struct callout timer; }; static inline struct delayed_work * to_delayed_work(struct work_struct *work) { return container_of(work, struct delayed_work, work); } static inline void _work_fn(void *context, int pending) { struct work_struct *work; work = context; work->fn(work); } #define INIT_WORK(work, func) \ do { \ (work)->fn = (func); \ (work)->taskqueue = NULL; \ TASK_INIT(&(work)->work_task, 0, _work_fn, (work)); \ } while (0) #define INIT_DELAYED_WORK(_work, func) \ do { \ INIT_WORK(&(_work)->work, func); \ callout_init(&(_work)->timer, CALLOUT_MPSAFE); \ } while (0) -#define INIT_DELAYED_WORK_DEFERRABLE INIT_DELAYED_WORK +#define INIT_DEFERRABLE_WORK INIT_DELAYED_WORK #define schedule_work(work) \ do { \ (work)->taskqueue = taskqueue_thread; \ taskqueue_enqueue(taskqueue_thread, &(work)->work_task); \ } while (0) #define flush_scheduled_work() flush_taskqueue(taskqueue_thread) -#define queue_work(q, work) \ -do { \ - (work)->taskqueue = (q)->taskqueue; \ - taskqueue_enqueue((q)->taskqueue, &(work)->work_task); \ -} while (0) +static inline int queue_work (struct workqueue_struct *q, struct work_struct *work) +{ + (work)->taskqueue = (q)->taskqueue; + /* Return opposite val to align with Linux logic */ + return !taskqueue_enqueue((q)->taskqueue, &(work)->work_task); +} static inline void _delayed_work_fn(void *arg) { struct delayed_work *work; work = arg; taskqueue_enqueue(work->work.taskqueue, &work->work.work_task); } static inline int queue_delayed_work(struct workqueue_struct *wq, struct delayed_work *work, unsigned long delay) { int pending; pending = work->work.work_task.ta_pending; work->work.taskqueue = wq->taskqueue; if (delay != 0) callout_reset(&work->timer, delay, _delayed_work_fn, work); else _delayed_work_fn((void *)work); return (!pending); } +static inline bool schedule_delayed_work(struct delayed_work *dwork, + unsigned long delay) +{ + struct workqueue_struct wq; + wq.taskqueue = taskqueue_thread; + return queue_delayed_work(&wq, dwork, delay); +} + static inline struct workqueue_struct * _create_workqueue_common(char *name, int cpus) { struct workqueue_struct *wq; wq = kmalloc(sizeof(*wq), M_WAITOK); wq->taskqueue = taskqueue_create((name), M_WAITOK, taskqueue_thread_enqueue, &wq->taskqueue); taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name); return (wq); } #define create_singlethread_workqueue(name) \ _create_workqueue_common(name, 1) #define create_workqueue(name) \ _create_workqueue_common(name, MAXCPU) static inline void destroy_workqueue(struct workqueue_struct *wq) { taskqueue_free(wq->taskqueue); kfree(wq); } #define flush_workqueue(wq) flush_taskqueue((wq)->taskqueue) static inline void _flush_fn(void *context, int pending) { } static inline void flush_taskqueue(struct taskqueue *tq) { struct task flushtask; PHOLD(curproc); TASK_INIT(&flushtask, 0, _flush_fn, NULL); taskqueue_enqueue(tq, &flushtask); taskqueue_drain(tq, &flushtask); PRELE(curproc); } static inline int cancel_work_sync(struct work_struct *work) { if (work->taskqueue && taskqueue_cancel(work->taskqueue, &work->work_task, NULL)) taskqueue_drain(work->taskqueue, &work->work_task); return 0; } /* * This may leave work running on another CPU as it does on Linux. */ static inline int cancel_delayed_work(struct delayed_work *work) { callout_stop(&work->timer); if (work->work.taskqueue) return (taskqueue_cancel(work->work.taskqueue, &work->work.work_task, NULL) == 0); return 0; +} + +static inline int +cancel_delayed_work_sync(struct delayed_work *work) +{ + + callout_drain(&work->timer); + if (work->work.taskqueue && + taskqueue_cancel(work->work.taskqueue, &work->work.work_task, NULL)) + taskqueue_drain(work->work.taskqueue, &work->work.work_task); + return 0; +} + +static inline bool +mod_delayed_work(struct workqueue_struct *wq, struct delayed_work *dwork, + unsigned long delay) +{ + cancel_delayed_work(dwork); + queue_delayed_work(wq, dwork, delay); + return false; } #endif /* _LINUX_WORKQUEUE_H_ */ Index: stable/9/sys/ofed/include/net/arp.h =================================================================== --- stable/9/sys/ofed/include/net/arp.h (revision 273863) +++ stable/9/sys/ofed/include/net/arp.h (nonexistent) @@ -1,27 +0,0 @@ -/*- - * Copyright (c) 2010 Isilon Systems, Inc. - * Copyright (c) 2010 iX Systems, Inc. - * Copyright (c) 2010 Panasas, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ Property changes on: stable/9/sys/ofed/include/net/arp.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/9/sys/ofed/include/net/addrconf.h =================================================================== --- stable/9/sys/ofed/include/net/addrconf.h (revision 273863) +++ stable/9/sys/ofed/include/net/addrconf.h (nonexistent) @@ -1,27 +0,0 @@ -/*- - * Copyright (c) 2010 Isilon Systems, Inc. - * Copyright (c) 2010 iX Systems, Inc. - * Copyright (c) 2010 Panasas, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ Property changes on: stable/9/sys/ofed/include/net/addrconf.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/9/sys/ofed/include/net/neighbour.h =================================================================== --- stable/9/sys/ofed/include/net/neighbour.h (revision 273863) +++ stable/9/sys/ofed/include/net/neighbour.h (nonexistent) @@ -1,27 +0,0 @@ -/*- - * Copyright (c) 2010 Isilon Systems, Inc. - * Copyright (c) 2010 iX Systems, Inc. - * Copyright (c) 2010 Panasas, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ Property changes on: stable/9/sys/ofed/include/net/neighbour.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/9/sys/ofed/include/net/ip6_route.h =================================================================== --- stable/9/sys/ofed/include/net/ip6_route.h (revision 273863) +++ stable/9/sys/ofed/include/net/ip6_route.h (nonexistent) @@ -1,27 +0,0 @@ -/*- - * Copyright (c) 2010 Isilon Systems, Inc. - * Copyright (c) 2010 iX Systems, Inc. - * Copyright (c) 2010 Panasas, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ Property changes on: stable/9/sys/ofed/include/net/ip6_route.h ___________________________________________________________________ Deleted: fbsd:nokeywords ## -1 +0,0 ## -true \ No newline at end of property Index: stable/9/sys/ofed/include/net/if_inet6.h =================================================================== --- stable/9/sys/ofed/include/net/if_inet6.h (nonexistent) +++ stable/9/sys/ofed/include/net/if_inet6.h (revision 273864) @@ -0,0 +1,47 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _NET_IF_INET6_H_ +#define _NET_IF_INET6_H_ + +static inline void ipv6_eth_mc_map(const struct in6_addr *addr, char *buf) +{ +/* + * +-------+-------+-------+-------+-------+-------+ + * | 33 | 33 | DST13 | DST14 | DST15 | DST16 | + * +-------+-------+-------+-------+-------+-------+ + */ + + buf[0]= 0x33; + buf[1]= 0x33; + + memcpy(buf + 2, &addr->s6_addr32[3], sizeof(__u32)); +} + +#endif /* _NET_IF_INET6_H_ */ Property changes on: stable/9/sys/ofed/include/net/if_inet6.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/include/net/ip.h =================================================================== --- stable/9/sys/ofed/include/net/ip.h (revision 273863) +++ stable/9/sys/ofed/include/net/ip.h (revision 273864) @@ -1,81 +1,82 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_NET_IP_H_ #define _LINUX_NET_IP_H_ #include "opt_inet.h" #include #include #include #include #include #include #include #ifdef INET static inline void inet_get_local_port_range(int *low, int *high) { *low = V_ipport_firstauto; *high = V_ipport_lastauto; } static inline void ip_ib_mc_map(uint32_t addr, const unsigned char *bcast, char *buf) { unsigned char scope; addr = ntohl(addr); scope = bcast[5] & 0xF; buf[0] = 0; buf[1] = 0xff; buf[2] = 0xff; buf[3] = 0xff; buf[4] = 0xff; buf[5] = 0x10 | scope; buf[6] = 0x40; buf[7] = 0x1b; buf[8] = bcast[8]; buf[9] = bcast[9]; buf[10] = 0; buf[11] = 0; buf[12] = 0; buf[13] = 0; buf[14] = 0; buf[15] = 0; buf[16] = (addr >> 24) & 0x0f; buf[17] = (addr >> 16) & 0xff; buf[18] = (addr >> 8) & 0xff; buf[19] = addr & 0xff; } #endif #endif /* _LINUX_NET_IP_H_ */ Index: stable/9/sys/ofed/include/net/ipv6.h =================================================================== --- stable/9/sys/ofed/include/net/ipv6.h (revision 273863) +++ stable/9/sys/ofed/include/net/ipv6.h (revision 273864) @@ -1,60 +1,110 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_NET_IPV6_H_ #define _LINUX_NET_IPV6_H_ #include "opt_inet6.h" #define ipv6_addr_loopback IN6_IS_ADDR_LOOPBACK #define ipv6_addr_copy(dst, src) \ memcpy((dst), (src), sizeof(struct in6_addr)) #ifdef INET6 static inline void ipv6_ib_mc_map(const struct in6_addr *addr, const unsigned char *broadcast, char *buf) { unsigned char scope; scope = broadcast[5] & 0xF; buf[0] = 0; buf[1] = 0xff; buf[2] = 0xff; buf[3] = 0xff; buf[4] = 0xff; buf[5] = 0x10 | scope; buf[6] = 0x60; buf[7] = 0x1b; buf[8] = broadcast[8]; buf[9] = broadcast[9]; memcpy(&buf[10], &addr->s6_addr[6], 10); } #endif + +static inline void __ipv6_addr_set_half(__be32 *addr, + __be32 wh, __be32 wl) +{ +#if BITS_PER_LONG == 64 +#if defined(__BIG_ENDIAN) + if (__builtin_constant_p(wh) && __builtin_constant_p(wl)) { + *(__force u64 *)addr = ((__force u64)(wh) << 32 | (__force u64)(wl)); + return; + } +#elif defined(__LITTLE_ENDIAN) + if (__builtin_constant_p(wl) && __builtin_constant_p(wh)) { + *(__force u64 *)addr = ((__force u64)(wl) << 32 | (__force u64)(wh)); + return; + } +#endif +#endif + addr[0] = wh; + addr[1] = wl; +} + +static inline void ipv6_addr_set(struct in6_addr *addr, + __be32 w1, __be32 w2, + __be32 w3, __be32 w4) +{ + __ipv6_addr_set_half(&addr->s6_addr32[0], w1, w2); + __ipv6_addr_set_half(&addr->s6_addr32[2], w3, w4); +} + +static inline void ipv6_addr_set_v4mapped(const __be32 addr, + struct in6_addr *v4mapped) +{ + ipv6_addr_set(v4mapped, + 0, 0, + htonl(0x0000FFFF), + addr); +} + +static inline int ipv6_addr_v4mapped(const struct in6_addr *a) +{ + return ((a->s6_addr32[0] | a->s6_addr32[1] | + (a->s6_addr32[2] ^ htonl(0x0000ffff))) == 0); +} + +static inline int ipv6_addr_cmp(const struct in6_addr *a1, const struct in6_addr *a2) +{ + return memcmp(a1, a2, sizeof(struct in6_addr)); +} + #endif /* _LINUX_NET_IPV6_H_ */ Index: stable/9/sys/ofed/include/net/netevent.h =================================================================== --- stable/9/sys/ofed/include/net/netevent.h (revision 273863) +++ stable/9/sys/ofed/include/net/netevent.h (revision 273864) @@ -1,70 +1,71 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_NET_NETEVENT_H_ #define _LINUX_NET_NETEVENT_H_ #include enum netevent_notif_type { NETEVENT_NEIGH_UPDATE = 0, #if 0 /* Unsupported events. */ NETEVENT_PMTU_UPDATE, NETEVENT_REDIRECT, #endif }; struct llentry; static inline void _handle_arp_update_event(void *arg, struct llentry *lle, int evt __unused) { struct notifier_block *nb; nb = arg; nb->notifier_call(nb, NETEVENT_NEIGH_UPDATE, lle); } static inline int register_netevent_notifier(struct notifier_block *nb) { nb->tags[NETEVENT_NEIGH_UPDATE] = EVENTHANDLER_REGISTER( lle_event, _handle_arp_update_event, nb, 0); return (0); } static inline int unregister_netevent_notifier(struct notifier_block *nb) { EVENTHANDLER_DEREGISTER(lle_event, nb->tags[NETEVENT_NEIGH_UPDATE]); return (0); } #endif /* _LINUX_NET_NETEVENT_H_ */ Index: stable/9/sys/ofed/include/net/tcp.h =================================================================== --- stable/9/sys/ofed/include/net/tcp.h (revision 273863) +++ stable/9/sys/ofed/include/net/tcp.h (revision 273864) @@ -1,38 +1,39 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUX_NET_TCP_H_ #define _LINUX_NET_TCP_H_ #include #include #include #include #endif /* _LINUX_NET_TCP_H_ */ Index: stable/9/sys/ofed/include/rdma/ib_cm.h =================================================================== --- stable/9/sys/ofed/include/rdma/ib_cm.h (revision 273863) +++ stable/9/sys/ofed/include/rdma/ib_cm.h (revision 273864) @@ -1,589 +1,604 @@ /* * Copyright (c) 2004, 2005 Intel Corporation. All rights reserved. * Copyright (c) 2004 Topspin Corporation. All rights reserved. * Copyright (c) 2004 Voltaire Corporation. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #if !defined(IB_CM_H) #define IB_CM_H #include #include +/* ib_cm and ib_user_cm modules share /sys/class/infiniband_cm */ +extern struct class cm_class; + enum ib_cm_state { IB_CM_IDLE, IB_CM_LISTEN, IB_CM_REQ_SENT, IB_CM_REQ_RCVD, IB_CM_MRA_REQ_SENT, IB_CM_MRA_REQ_RCVD, IB_CM_REP_SENT, IB_CM_REP_RCVD, IB_CM_MRA_REP_SENT, IB_CM_MRA_REP_RCVD, IB_CM_ESTABLISHED, IB_CM_DREQ_SENT, IB_CM_DREQ_RCVD, IB_CM_TIMEWAIT, IB_CM_SIDR_REQ_SENT, IB_CM_SIDR_REQ_RCVD }; enum ib_cm_lap_state { IB_CM_LAP_UNINIT, IB_CM_LAP_IDLE, IB_CM_LAP_SENT, IB_CM_LAP_RCVD, IB_CM_MRA_LAP_SENT, IB_CM_MRA_LAP_RCVD, }; enum ib_cm_event_type { IB_CM_REQ_ERROR, IB_CM_REQ_RECEIVED, IB_CM_REP_ERROR, IB_CM_REP_RECEIVED, IB_CM_RTU_RECEIVED, IB_CM_USER_ESTABLISHED, IB_CM_DREQ_ERROR, IB_CM_DREQ_RECEIVED, IB_CM_DREP_RECEIVED, IB_CM_TIMEWAIT_EXIT, IB_CM_MRA_RECEIVED, IB_CM_REJ_RECEIVED, IB_CM_LAP_ERROR, IB_CM_LAP_RECEIVED, IB_CM_APR_RECEIVED, IB_CM_SIDR_REQ_ERROR, IB_CM_SIDR_REQ_RECEIVED, IB_CM_SIDR_REP_RECEIVED }; enum ib_cm_data_size { IB_CM_REQ_PRIVATE_DATA_SIZE = 92, IB_CM_MRA_PRIVATE_DATA_SIZE = 222, IB_CM_REJ_PRIVATE_DATA_SIZE = 148, IB_CM_REP_PRIVATE_DATA_SIZE = 196, IB_CM_RTU_PRIVATE_DATA_SIZE = 224, IB_CM_DREQ_PRIVATE_DATA_SIZE = 220, IB_CM_DREP_PRIVATE_DATA_SIZE = 224, IB_CM_REJ_ARI_LENGTH = 72, IB_CM_LAP_PRIVATE_DATA_SIZE = 168, IB_CM_APR_PRIVATE_DATA_SIZE = 148, IB_CM_APR_INFO_LENGTH = 72, IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE = 216, IB_CM_SIDR_REP_PRIVATE_DATA_SIZE = 136, IB_CM_SIDR_REP_INFO_LENGTH = 72, IB_CM_COMPARE_SIZE = 64 }; struct ib_cm_id; struct ib_cm_req_event_param { struct ib_cm_id *listen_id; u8 port; struct ib_sa_path_rec *primary_path; struct ib_sa_path_rec *alternate_path; __be64 remote_ca_guid; u32 remote_qkey; u32 remote_qpn; enum ib_qp_type qp_type; u32 starting_psn; u8 responder_resources; u8 initiator_depth; unsigned int local_cm_response_timeout:5; unsigned int flow_control:1; unsigned int remote_cm_response_timeout:5; unsigned int retry_count:3; unsigned int rnr_retry_count:3; unsigned int srq:1; }; struct ib_cm_rep_event_param { __be64 remote_ca_guid; u32 remote_qkey; u32 remote_qpn; u32 starting_psn; u8 responder_resources; u8 initiator_depth; unsigned int target_ack_delay:5; unsigned int failover_accepted:2; unsigned int flow_control:1; unsigned int rnr_retry_count:3; unsigned int srq:1; }; enum ib_cm_rej_reason { IB_CM_REJ_NO_QP = 1, IB_CM_REJ_NO_EEC = 2, IB_CM_REJ_NO_RESOURCES = 3, IB_CM_REJ_TIMEOUT = 4, IB_CM_REJ_UNSUPPORTED = 5, IB_CM_REJ_INVALID_COMM_ID = 6, IB_CM_REJ_INVALID_COMM_INSTANCE = 7, IB_CM_REJ_INVALID_SERVICE_ID = 8, IB_CM_REJ_INVALID_TRANSPORT_TYPE = 9, IB_CM_REJ_STALE_CONN = 10, IB_CM_REJ_RDC_NOT_EXIST = 11, IB_CM_REJ_INVALID_GID = 12, IB_CM_REJ_INVALID_LID = 13, IB_CM_REJ_INVALID_SL = 14, IB_CM_REJ_INVALID_TRAFFIC_CLASS = 15, IB_CM_REJ_INVALID_HOP_LIMIT = 16, IB_CM_REJ_INVALID_PACKET_RATE = 17, IB_CM_REJ_INVALID_ALT_GID = 18, IB_CM_REJ_INVALID_ALT_LID = 19, IB_CM_REJ_INVALID_ALT_SL = 20, IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS = 21, IB_CM_REJ_INVALID_ALT_HOP_LIMIT = 22, IB_CM_REJ_INVALID_ALT_PACKET_RATE = 23, IB_CM_REJ_PORT_CM_REDIRECT = 24, IB_CM_REJ_PORT_REDIRECT = 25, IB_CM_REJ_INVALID_MTU = 26, IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES = 27, IB_CM_REJ_CONSUMER_DEFINED = 28, IB_CM_REJ_INVALID_RNR_RETRY = 29, IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID = 30, IB_CM_REJ_INVALID_CLASS_VERSION = 31, IB_CM_REJ_INVALID_FLOW_LABEL = 32, IB_CM_REJ_INVALID_ALT_FLOW_LABEL = 33 }; struct ib_cm_rej_event_param { enum ib_cm_rej_reason reason; void *ari; u8 ari_length; }; struct ib_cm_mra_event_param { u8 service_timeout; }; struct ib_cm_lap_event_param { struct ib_sa_path_rec *alternate_path; }; enum ib_cm_apr_status { IB_CM_APR_SUCCESS, IB_CM_APR_INVALID_COMM_ID, IB_CM_APR_UNSUPPORTED, IB_CM_APR_REJECT, IB_CM_APR_REDIRECT, IB_CM_APR_IS_CURRENT, IB_CM_APR_INVALID_QPN_EECN, IB_CM_APR_INVALID_LID, IB_CM_APR_INVALID_GID, IB_CM_APR_INVALID_FLOW_LABEL, IB_CM_APR_INVALID_TCLASS, IB_CM_APR_INVALID_HOP_LIMIT, IB_CM_APR_INVALID_PACKET_RATE, IB_CM_APR_INVALID_SL }; struct ib_cm_apr_event_param { enum ib_cm_apr_status ap_status; void *apr_info; u8 info_len; }; struct ib_cm_sidr_req_event_param { struct ib_cm_id *listen_id; u8 port; u16 pkey; }; enum ib_cm_sidr_status { IB_SIDR_SUCCESS, IB_SIDR_UNSUPPORTED, IB_SIDR_REJECT, IB_SIDR_NO_QP, IB_SIDR_REDIRECT, IB_SIDR_UNSUPPORTED_VERSION }; struct ib_cm_sidr_rep_event_param { enum ib_cm_sidr_status status; u32 qkey; u32 qpn; void *info; u8 info_len; }; struct ib_cm_event { enum ib_cm_event_type event; union { struct ib_cm_req_event_param req_rcvd; struct ib_cm_rep_event_param rep_rcvd; /* No data for RTU received events. */ struct ib_cm_rej_event_param rej_rcvd; struct ib_cm_mra_event_param mra_rcvd; struct ib_cm_lap_event_param lap_rcvd; struct ib_cm_apr_event_param apr_rcvd; /* No data for DREQ/DREP received events. */ struct ib_cm_sidr_req_event_param sidr_req_rcvd; struct ib_cm_sidr_rep_event_param sidr_rep_rcvd; enum ib_wc_status send_status; } param; void *private_data; }; + +#define CM_REQ_ATTR_ID cpu_to_be16(0x0010) +#define CM_MRA_ATTR_ID cpu_to_be16(0x0011) +#define CM_REJ_ATTR_ID cpu_to_be16(0x0012) +#define CM_REP_ATTR_ID cpu_to_be16(0x0013) +#define CM_RTU_ATTR_ID cpu_to_be16(0x0014) +#define CM_DREQ_ATTR_ID cpu_to_be16(0x0015) +#define CM_DREP_ATTR_ID cpu_to_be16(0x0016) +#define CM_SIDR_REQ_ATTR_ID cpu_to_be16(0x0017) +#define CM_SIDR_REP_ATTR_ID cpu_to_be16(0x0018) +#define CM_LAP_ATTR_ID cpu_to_be16(0x0019) +#define CM_APR_ATTR_ID cpu_to_be16(0x001A) /** * ib_cm_handler - User-defined callback to process communication events. * @cm_id: Communication identifier associated with the reported event. * @event: Information about the communication event. * * IB_CM_REQ_RECEIVED and IB_CM_SIDR_REQ_RECEIVED communication events * generated as a result of listen requests result in the allocation of a * new @cm_id. The new @cm_id is returned to the user through this callback. * Clients are responsible for destroying the new @cm_id. For peer-to-peer * IB_CM_REQ_RECEIVED and all other events, the returned @cm_id corresponds * to a user's existing communication identifier. * * Users may not call ib_destroy_cm_id while in the context of this callback; * however, returning a non-zero value instructs the communication manager to * destroy the @cm_id after the callback completes. */ typedef int (*ib_cm_handler)(struct ib_cm_id *cm_id, struct ib_cm_event *event); struct ib_cm_id { ib_cm_handler cm_handler; void *context; struct ib_device *device; __be64 service_id; __be64 service_mask; enum ib_cm_state state; /* internal CM/debug use */ enum ib_cm_lap_state lap_state; /* internal CM/debug use */ __be32 local_id; __be32 remote_id; u32 remote_cm_qpn; /* 1 unless redirected */ }; /** * ib_create_cm_id - Allocate a communication identifier. * @device: Device associated with the cm_id. All related communication will * be associated with the specified device. * @cm_handler: Callback invoked to notify the user of CM events. * @context: User specified context associated with the communication * identifier. * * Communication identifiers are used to track connection states, service * ID resolution requests, and listen requests. */ struct ib_cm_id *ib_create_cm_id(struct ib_device *device, ib_cm_handler cm_handler, void *context); /** * ib_destroy_cm_id - Destroy a connection identifier. * @cm_id: Connection identifier to destroy. * * This call blocks until the connection identifier is destroyed. */ void ib_destroy_cm_id(struct ib_cm_id *cm_id); #define IB_SERVICE_ID_AGN_MASK cpu_to_be64(0xFF00000000000000ULL) #define IB_CM_ASSIGN_SERVICE_ID cpu_to_be64(0x0200000000000000ULL) #define IB_CMA_SERVICE_ID cpu_to_be64(0x0000000001000000ULL) #define IB_CMA_SERVICE_ID_MASK cpu_to_be64(0xFFFFFFFFFF000000ULL) #define IB_SDP_SERVICE_ID cpu_to_be64(0x0000000000010000ULL) #define IB_SDP_SERVICE_ID_MASK cpu_to_be64(0xFFFFFFFFFFFF0000ULL) struct ib_cm_compare_data { u8 data[IB_CM_COMPARE_SIZE]; u8 mask[IB_CM_COMPARE_SIZE]; }; /** * ib_cm_listen - Initiates listening on the specified service ID for * connection and service ID resolution requests. * @cm_id: Connection identifier associated with the listen request. * @service_id: Service identifier matched against incoming connection * and service ID resolution requests. The service ID should be specified * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will * assign a service ID to the caller. * @service_mask: Mask applied to service ID used to listen across a * range of service IDs. If set to 0, the service ID is matched * exactly. This parameter is ignored if %service_id is set to * IB_CM_ASSIGN_SERVICE_ID. * @compare_data: This parameter is optional. It specifies data that must * appear in the private data of a connection request for the specified * listen request. */ int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask, struct ib_cm_compare_data *compare_data); struct ib_cm_req_param { struct ib_sa_path_rec *primary_path; struct ib_sa_path_rec *alternate_path; __be64 service_id; u32 qp_num; enum ib_qp_type qp_type; u32 starting_psn; const void *private_data; u8 private_data_len; u8 peer_to_peer; u8 responder_resources; u8 initiator_depth; u8 remote_cm_response_timeout; u8 flow_control; u8 local_cm_response_timeout; u8 retry_count; u8 rnr_retry_count; u8 max_cm_retries; u8 srq; }; /** * ib_send_cm_req - Sends a connection request to the remote node. * @cm_id: Connection identifier that will be associated with the * connection request. * @param: Connection request information needed to establish the * connection. */ int ib_send_cm_req(struct ib_cm_id *cm_id, struct ib_cm_req_param *param); struct ib_cm_rep_param { u32 qp_num; u32 starting_psn; const void *private_data; u8 private_data_len; u8 responder_resources; u8 initiator_depth; u8 failover_accepted; u8 flow_control; u8 rnr_retry_count; u8 srq; }; /** * ib_send_cm_rep - Sends a connection reply in response to a connection * request. * @cm_id: Connection identifier that will be associated with the * connection request. * @param: Connection reply information needed to establish the * connection. */ int ib_send_cm_rep(struct ib_cm_id *cm_id, struct ib_cm_rep_param *param); /** * ib_send_cm_rtu - Sends a connection ready to use message in response * to a connection reply message. * @cm_id: Connection identifier associated with the connection request. * @private_data: Optional user-defined private data sent with the * ready to use message. * @private_data_len: Size of the private data buffer, in bytes. */ int ib_send_cm_rtu(struct ib_cm_id *cm_id, const void *private_data, u8 private_data_len); /** * ib_send_cm_dreq - Sends a disconnection request for an existing * connection. * @cm_id: Connection identifier associated with the connection being * released. * @private_data: Optional user-defined private data sent with the * disconnection request message. * @private_data_len: Size of the private data buffer, in bytes. */ int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data, u8 private_data_len); /** * ib_send_cm_drep - Sends a disconnection reply to a disconnection request. * @cm_id: Connection identifier associated with the connection being * released. * @private_data: Optional user-defined private data sent with the * disconnection reply message. * @private_data_len: Size of the private data buffer, in bytes. * * If the cm_id is in the correct state, the CM will transition the connection * to the timewait state, even if an error occurs sending the DREP message. */ int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data, u8 private_data_len); /** * ib_cm_notify - Notifies the CM of an event reported to the consumer. * @cm_id: Connection identifier to transition to established. * @event: Type of event. * * This routine should be invoked by users to notify the CM of relevant * communication events. Events that should be reported to the CM and * when to report them are: * * IB_EVENT_COMM_EST - Used when a message is received on a connected * QP before an RTU has been received. * IB_EVENT_PATH_MIG - Notifies the CM that the connection has failed over * to the alternate path. */ int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event); /** * ib_send_cm_rej - Sends a connection rejection message to the * remote node. * @cm_id: Connection identifier associated with the connection being * rejected. * @reason: Reason for the connection request rejection. * @ari: Optional additional rejection information. * @ari_length: Size of the additional rejection information, in bytes. * @private_data: Optional user-defined private data sent with the * rejection message. * @private_data_len: Size of the private data buffer, in bytes. */ int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason, void *ari, u8 ari_length, const void *private_data, u8 private_data_len); #define IB_CM_MRA_FLAG_DELAY 0x80 /* Send MRA only after a duplicate msg */ /** * ib_send_cm_mra - Sends a message receipt acknowledgement to a connection * message. * @cm_id: Connection identifier associated with the connection message. * @service_timeout: The lower 5-bits specify the maximum time required for * the sender to reply to to the connection message. The upper 3-bits * specify additional control flags. * @private_data: Optional user-defined private data sent with the * message receipt acknowledgement. * @private_data_len: Size of the private data buffer, in bytes. */ int ib_send_cm_mra(struct ib_cm_id *cm_id, u8 service_timeout, const void *private_data, u8 private_data_len); /** * ib_send_cm_lap - Sends a load alternate path request. * @cm_id: Connection identifier associated with the load alternate path * message. * @alternate_path: A path record that identifies the alternate path to * load. * @private_data: Optional user-defined private data sent with the * load alternate path message. * @private_data_len: Size of the private data buffer, in bytes. */ int ib_send_cm_lap(struct ib_cm_id *cm_id, struct ib_sa_path_rec *alternate_path, const void *private_data, u8 private_data_len); /** * ib_cm_init_qp_attr - Initializes the QP attributes for use in transitioning * to a specified QP state. * @cm_id: Communication identifier associated with the QP attributes to * initialize. * @qp_attr: On input, specifies the desired QP state. On output, the * mandatory and desired optional attributes will be set in order to * modify the QP to the specified state. * @qp_attr_mask: The QP attribute mask that may be used to transition the * QP to the specified state. * * Users must set the @qp_attr->qp_state to the desired QP state. This call * will set all required attributes for the given transition, along with * known optional attributes. Users may override the attributes returned from * this call before calling ib_modify_qp. */ int ib_cm_init_qp_attr(struct ib_cm_id *cm_id, struct ib_qp_attr *qp_attr, int *qp_attr_mask); /** * ib_send_cm_apr - Sends an alternate path response message in response to * a load alternate path request. * @cm_id: Connection identifier associated with the alternate path response. * @status: Reply status sent with the alternate path response. * @info: Optional additional information sent with the alternate path * response. * @info_length: Size of the additional information, in bytes. * @private_data: Optional user-defined private data sent with the * alternate path response message. * @private_data_len: Size of the private data buffer, in bytes. */ int ib_send_cm_apr(struct ib_cm_id *cm_id, enum ib_cm_apr_status status, void *info, u8 info_length, const void *private_data, u8 private_data_len); struct ib_cm_sidr_req_param { struct ib_sa_path_rec *path; __be64 service_id; int timeout_ms; const void *private_data; u8 private_data_len; u8 max_cm_retries; }; /** * ib_send_cm_sidr_req - Sends a service ID resolution request to the * remote node. * @cm_id: Communication identifier that will be associated with the * service ID resolution request. * @param: Service ID resolution request information. */ int ib_send_cm_sidr_req(struct ib_cm_id *cm_id, struct ib_cm_sidr_req_param *param); struct ib_cm_sidr_rep_param { u32 qp_num; u32 qkey; enum ib_cm_sidr_status status; const void *info; u8 info_length; const void *private_data; u8 private_data_len; }; /** * ib_send_cm_sidr_rep - Sends a service ID resolution reply to the * remote node. * @cm_id: Communication identifier associated with the received service ID * resolution request. * @param: Service ID resolution reply information. */ int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id, struct ib_cm_sidr_rep_param *param); #endif /* IB_CM_H */ Index: stable/9/sys/ofed/include/rdma/ib_mad.h =================================================================== --- stable/9/sys/ofed/include/rdma/ib_mad.h (revision 273863) +++ stable/9/sys/ofed/include/rdma/ib_mad.h (revision 273864) @@ -1,655 +1,655 @@ /* * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2004 Infinicon Corporation. All rights reserved. * Copyright (c) 2004 Intel Corporation. All rights reserved. * Copyright (c) 2004 Topspin Corporation. All rights reserved. * Copyright (c) 2004-2006 Voltaire Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #if !defined(IB_MAD_H) #define IB_MAD_H #include #include /* Management base version */ #define IB_MGMT_BASE_VERSION 1 /* Management classes */ #define IB_MGMT_CLASS_SUBN_LID_ROUTED 0x01 #define IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE 0x81 #define IB_MGMT_CLASS_SUBN_ADM 0x03 #define IB_MGMT_CLASS_PERF_MGMT 0x04 #define IB_MGMT_CLASS_BM 0x05 #define IB_MGMT_CLASS_DEVICE_MGMT 0x06 #define IB_MGMT_CLASS_CM 0x07 #define IB_MGMT_CLASS_SNMP 0x08 #define IB_MGMT_CLASS_DEVICE_ADM 0x10 #define IB_MGMT_CLASS_BOOT_MGMT 0x11 #define IB_MGMT_CLASS_BIS 0x12 #define IB_MGMT_CLASS_CONG_MGMT 0x21 #define IB_MGMT_CLASS_VENDOR_RANGE2_START 0x30 #define IB_MGMT_CLASS_VENDOR_RANGE2_END 0x4F #define IB_OPENIB_OUI (0x001405) /* Management methods */ #define IB_MGMT_METHOD_GET 0x01 #define IB_MGMT_METHOD_SET 0x02 #define IB_MGMT_METHOD_GET_RESP 0x81 #define IB_MGMT_METHOD_SEND 0x03 #define IB_MGMT_METHOD_TRAP 0x05 #define IB_MGMT_METHOD_REPORT 0x06 #define IB_MGMT_METHOD_REPORT_RESP 0x86 #define IB_MGMT_METHOD_TRAP_REPRESS 0x07 #define IB_MGMT_METHOD_RESP 0x80 #define IB_BM_ATTR_MOD_RESP cpu_to_be32(1) #define IB_MGMT_MAX_METHODS 128 /* RMPP information */ #define IB_MGMT_RMPP_VERSION 1 #define IB_MGMT_RMPP_TYPE_DATA 1 #define IB_MGMT_RMPP_TYPE_ACK 2 #define IB_MGMT_RMPP_TYPE_STOP 3 #define IB_MGMT_RMPP_TYPE_ABORT 4 #define IB_MGMT_RMPP_FLAG_ACTIVE 1 #define IB_MGMT_RMPP_FLAG_FIRST (1<<1) #define IB_MGMT_RMPP_FLAG_LAST (1<<2) #define IB_MGMT_RMPP_NO_RESPTIME 0x1F #define IB_MGMT_RMPP_STATUS_SUCCESS 0 #define IB_MGMT_RMPP_STATUS_RESX 1 #define IB_MGMT_RMPP_STATUS_ABORT_MIN 118 #define IB_MGMT_RMPP_STATUS_T2L 118 #define IB_MGMT_RMPP_STATUS_BAD_LEN 119 #define IB_MGMT_RMPP_STATUS_BAD_SEG 120 #define IB_MGMT_RMPP_STATUS_BADT 121 #define IB_MGMT_RMPP_STATUS_W2S 122 #define IB_MGMT_RMPP_STATUS_S2B 123 #define IB_MGMT_RMPP_STATUS_BAD_STATUS 124 #define IB_MGMT_RMPP_STATUS_UNV 125 #define IB_MGMT_RMPP_STATUS_TMR 126 #define IB_MGMT_RMPP_STATUS_UNSPEC 127 #define IB_MGMT_RMPP_STATUS_ABORT_MAX 127 #define IB_QP0 0 #define IB_QP1 cpu_to_be32(1) #define IB_QP1_QKEY 0x80010000 #define IB_QP_SET_QKEY 0x80000000 #define IB_DEFAULT_PKEY_PARTIAL 0x7FFF #define IB_DEFAULT_PKEY_FULL 0xFFFF enum { IB_MGMT_MAD_HDR = 24, IB_MGMT_MAD_DATA = 232, IB_MGMT_RMPP_HDR = 36, IB_MGMT_RMPP_DATA = 220, IB_MGMT_VENDOR_HDR = 40, IB_MGMT_VENDOR_DATA = 216, IB_MGMT_SA_HDR = 56, IB_MGMT_SA_DATA = 200, IB_MGMT_DEVICE_HDR = 64, IB_MGMT_DEVICE_DATA = 192, }; struct ib_mad_hdr { u8 base_version; u8 mgmt_class; u8 class_version; u8 method; __be16 status; __be16 class_specific; __be64 tid; __be16 attr_id; __be16 resv; __be32 attr_mod; }; struct ib_rmpp_hdr { u8 rmpp_version; u8 rmpp_type; u8 rmpp_rtime_flags; u8 rmpp_status; __be32 seg_num; __be32 paylen_newwin; }; typedef u64 __bitwise ib_sa_comp_mask; -#define IB_SA_COMP_MASK(n) ((__force ib_sa_comp_mask) cpu_to_be64(1ull << n)) +#define IB_SA_COMP_MASK(n) ((__force ib_sa_comp_mask) cpu_to_be64(1ull << (n))) /* * ib_sa_hdr and ib_sa_mad structures must be packed because they have * 64-bit fields that are only 32-bit aligned. 64-bit architectures will * lay them out wrong otherwise. (And unfortunately they are sent on * the wire so we can't change the layout) */ struct ib_sa_hdr { __be64 sm_key; __be16 attr_offset; __be16 reserved; ib_sa_comp_mask comp_mask; } __attribute__ ((packed)); struct ib_mad { struct ib_mad_hdr mad_hdr; u8 data[IB_MGMT_MAD_DATA]; }; struct ib_rmpp_mad { struct ib_mad_hdr mad_hdr; struct ib_rmpp_hdr rmpp_hdr; u8 data[IB_MGMT_RMPP_DATA]; }; struct ib_sa_mad { struct ib_mad_hdr mad_hdr; struct ib_rmpp_hdr rmpp_hdr; struct ib_sa_hdr sa_hdr; u8 data[IB_MGMT_SA_DATA]; } __attribute__ ((packed)); struct ib_vendor_mad { struct ib_mad_hdr mad_hdr; struct ib_rmpp_hdr rmpp_hdr; u8 reserved; u8 oui[3]; u8 data[IB_MGMT_VENDOR_DATA]; }; struct ib_class_port_info { u8 base_version; u8 class_version; __be16 capability_mask; u8 reserved[3]; u8 resp_time_value; u8 redirect_gid[16]; __be32 redirect_tcslfl; __be16 redirect_lid; __be16 redirect_pkey; __be32 redirect_qp; __be32 redirect_qkey; u8 trap_gid[16]; __be32 trap_tcslfl; __be16 trap_lid; __be16 trap_pkey; __be32 trap_hlqp; __be32 trap_qkey; }; /** * ib_mad_send_buf - MAD data buffer and work request for sends. * @next: A pointer used to chain together MADs for posting. * @mad: References an allocated MAD data buffer for MADs that do not have * RMPP active. For MADs using RMPP, references the common and management * class specific headers. * @mad_agent: MAD agent that allocated the buffer. * @ah: The address handle to use when sending the MAD. * @context: User-controlled context fields. * @hdr_len: Indicates the size of the data header of the MAD. This length * includes the common MAD, RMPP, and class specific headers. * @data_len: Indicates the total size of user-transferred data. * @seg_count: The number of RMPP segments allocated for this send. * @seg_size: Size of each RMPP segment. * @timeout_ms: Time to wait for a response. * @retries: Number of times to retry a request for a response. For MADs * using RMPP, this applies per window. On completion, returns the number * of retries needed to complete the transfer. * * Users are responsible for initializing the MAD buffer itself, with the * exception of any RMPP header. Additional segment buffer space allocated * beyond data_len is padding. */ struct ib_mad_send_buf { struct ib_mad_send_buf *next; void *mad; struct ib_mad_agent *mad_agent; struct ib_ah *ah; void *context[2]; int hdr_len; int data_len; int seg_count; int seg_size; int timeout_ms; int retries; }; /** * ib_response_mad - Returns if the specified MAD has been generated in * response to a sent request or trap. */ int ib_response_mad(struct ib_mad *mad); /** * ib_get_rmpp_resptime - Returns the RMPP response time. * @rmpp_hdr: An RMPP header. */ static inline u8 ib_get_rmpp_resptime(struct ib_rmpp_hdr *rmpp_hdr) { return rmpp_hdr->rmpp_rtime_flags >> 3; } /** * ib_get_rmpp_flags - Returns the RMPP flags. * @rmpp_hdr: An RMPP header. */ static inline u8 ib_get_rmpp_flags(struct ib_rmpp_hdr *rmpp_hdr) { return rmpp_hdr->rmpp_rtime_flags & 0x7; } /** * ib_set_rmpp_resptime - Sets the response time in an RMPP header. * @rmpp_hdr: An RMPP header. * @rtime: The response time to set. */ static inline void ib_set_rmpp_resptime(struct ib_rmpp_hdr *rmpp_hdr, u8 rtime) { rmpp_hdr->rmpp_rtime_flags = ib_get_rmpp_flags(rmpp_hdr) | (rtime << 3); } /** * ib_set_rmpp_flags - Sets the flags in an RMPP header. * @rmpp_hdr: An RMPP header. * @flags: The flags to set. */ static inline void ib_set_rmpp_flags(struct ib_rmpp_hdr *rmpp_hdr, u8 flags) { rmpp_hdr->rmpp_rtime_flags = (rmpp_hdr->rmpp_rtime_flags & 0xF8) | (flags & 0x7); } struct ib_mad_agent; struct ib_mad_send_wc; struct ib_mad_recv_wc; /** * ib_mad_send_handler - callback handler for a sent MAD. * @mad_agent: MAD agent that sent the MAD. * @mad_send_wc: Send work completion information on the sent MAD. */ typedef void (*ib_mad_send_handler)(struct ib_mad_agent *mad_agent, struct ib_mad_send_wc *mad_send_wc); /** * ib_mad_snoop_handler - Callback handler for snooping sent MADs. * @mad_agent: MAD agent that snooped the MAD. * @send_wr: Work request information on the sent MAD. * @mad_send_wc: Work completion information on the sent MAD. Valid * only for snooping that occurs on a send completion. * * Clients snooping MADs should not modify data referenced by the @send_wr * or @mad_send_wc. */ typedef void (*ib_mad_snoop_handler)(struct ib_mad_agent *mad_agent, struct ib_mad_send_buf *send_buf, struct ib_mad_send_wc *mad_send_wc); /** * ib_mad_recv_handler - callback handler for a received MAD. * @mad_agent: MAD agent requesting the received MAD. * @mad_recv_wc: Received work completion information on the received MAD. * * MADs received in response to a send request operation will be handed to * the user before the send operation completes. All data buffers given * to registered agents through this routine are owned by the receiving * client, except for snooping agents. Clients snooping MADs should not * modify the data referenced by @mad_recv_wc. */ typedef void (*ib_mad_recv_handler)(struct ib_mad_agent *mad_agent, struct ib_mad_recv_wc *mad_recv_wc); /** * ib_mad_agent - Used to track MAD registration with the access layer. * @device: Reference to device registration is on. * @qp: Reference to QP used for sending and receiving MADs. * @mr: Memory region for system memory usable for DMA. * @recv_handler: Callback handler for a received MAD. * @send_handler: Callback handler for a sent MAD. * @snoop_handler: Callback handler for snooped sent MADs. * @context: User-specified context associated with this registration. * @hi_tid: Access layer assigned transaction ID for this client. * Unsolicited MADs sent by this client will have the upper 32-bits * of their TID set to this value. * @port_num: Port number on which QP is registered * @rmpp_version: If set, indicates the RMPP version used by this agent. */ struct ib_mad_agent { struct ib_device *device; struct ib_qp *qp; struct ib_mr *mr; ib_mad_recv_handler recv_handler; ib_mad_send_handler send_handler; ib_mad_snoop_handler snoop_handler; void *context; u32 hi_tid; u8 port_num; u8 rmpp_version; }; /** * ib_mad_send_wc - MAD send completion information. * @send_buf: Send MAD data buffer associated with the send MAD request. * @status: Completion status. * @vendor_err: Optional vendor error information returned with a failed * request. */ struct ib_mad_send_wc { struct ib_mad_send_buf *send_buf; enum ib_wc_status status; u32 vendor_err; }; /** * ib_mad_recv_buf - received MAD buffer information. * @list: Reference to next data buffer for a received RMPP MAD. * @grh: References a data buffer containing the global route header. * The data refereced by this buffer is only valid if the GRH is * valid. * @mad: References the start of the received MAD. */ struct ib_mad_recv_buf { struct list_head list; struct ib_grh *grh; struct ib_mad *mad; }; /** * ib_mad_recv_wc - received MAD information. * @wc: Completion information for the received data. * @recv_buf: Specifies the location of the received data buffer(s). * @rmpp_list: Specifies a list of RMPP reassembled received MAD buffers. * @mad_len: The length of the received MAD, without duplicated headers. * * For received response, the wr_id contains a pointer to the ib_mad_send_buf * for the corresponding send request. */ struct ib_mad_recv_wc { struct ib_wc *wc; struct ib_mad_recv_buf recv_buf; struct list_head rmpp_list; int mad_len; }; /** * ib_mad_reg_req - MAD registration request * @mgmt_class: Indicates which management class of MADs should be receive * by the caller. This field is only required if the user wishes to * receive unsolicited MADs, otherwise it should be 0. * @mgmt_class_version: Indicates which version of MADs for the given * management class to receive. * @oui: Indicates IEEE OUI when mgmt_class is a vendor class * in the range from 0x30 to 0x4f. Otherwise not used. * @method_mask: The caller will receive unsolicited MADs for any method * where @method_mask = 1. */ struct ib_mad_reg_req { u8 mgmt_class; u8 mgmt_class_version; u8 oui[3]; DECLARE_BITMAP(method_mask, IB_MGMT_MAX_METHODS); }; /** * ib_register_mad_agent - Register to send/receive MADs. * @device: The device to register with. * @port_num: The port on the specified device to use. * @qp_type: Specifies which QP to access. Must be either * IB_QPT_SMI or IB_QPT_GSI. * @mad_reg_req: Specifies which unsolicited MADs should be received * by the caller. This parameter may be NULL if the caller only * wishes to receive solicited responses. * @rmpp_version: If set, indicates that the client will send * and receive MADs that contain the RMPP header for the given version. * If set to 0, indicates that RMPP is not used by this client. * @send_handler: The completion callback routine invoked after a send * request has completed. * @recv_handler: The completion callback routine invoked for a received * MAD. * @context: User specified context associated with the registration. */ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device, u8 port_num, enum ib_qp_type qp_type, struct ib_mad_reg_req *mad_reg_req, u8 rmpp_version, ib_mad_send_handler send_handler, ib_mad_recv_handler recv_handler, void *context); enum ib_mad_snoop_flags { /*IB_MAD_SNOOP_POSTED_SENDS = 1,*/ /*IB_MAD_SNOOP_RMPP_SENDS = (1<<1),*/ IB_MAD_SNOOP_SEND_COMPLETIONS = (1<<2), /*IB_MAD_SNOOP_RMPP_SEND_COMPLETIONS = (1<<3),*/ IB_MAD_SNOOP_RECVS = (1<<4) /*IB_MAD_SNOOP_RMPP_RECVS = (1<<5),*/ /*IB_MAD_SNOOP_REDIRECTED_QPS = (1<<6)*/ }; /** * ib_register_mad_snoop - Register to snoop sent and received MADs. * @device: The device to register with. * @port_num: The port on the specified device to use. * @qp_type: Specifies which QP traffic to snoop. Must be either * IB_QPT_SMI or IB_QPT_GSI. * @mad_snoop_flags: Specifies information where snooping occurs. * @send_handler: The callback routine invoked for a snooped send. * @recv_handler: The callback routine invoked for a snooped receive. * @context: User specified context associated with the registration. */ struct ib_mad_agent *ib_register_mad_snoop(struct ib_device *device, u8 port_num, enum ib_qp_type qp_type, int mad_snoop_flags, ib_mad_snoop_handler snoop_handler, ib_mad_recv_handler recv_handler, void *context); /** * ib_unregister_mad_agent - Unregisters a client from using MAD services. * @mad_agent: Corresponding MAD registration request to deregister. * * After invoking this routine, MAD services are no longer usable by the * client on the associated QP. */ int ib_unregister_mad_agent(struct ib_mad_agent *mad_agent); /** * ib_post_send_mad - Posts MAD(s) to the send queue of the QP associated * with the registered client. * @send_buf: Specifies the information needed to send the MAD(s). * @bad_send_buf: Specifies the MAD on which an error was encountered. This * parameter is optional if only a single MAD is posted. * * Sent MADs are not guaranteed to complete in the order that they were posted. * * If the MAD requires RMPP, the data buffer should contain a single copy * of the common MAD, RMPP, and class specific headers, followed by the class * defined data. If the class defined data would not divide evenly into * RMPP segments, then space must be allocated at the end of the referenced * buffer for any required padding. To indicate the amount of class defined * data being transferred, the paylen_newwin field in the RMPP header should * be set to the size of the class specific header plus the amount of class * defined data being transferred. The paylen_newwin field should be * specified in network-byte order. */ int ib_post_send_mad(struct ib_mad_send_buf *send_buf, struct ib_mad_send_buf **bad_send_buf); /** * ib_free_recv_mad - Returns data buffers used to receive a MAD. * @mad_recv_wc: Work completion information for a received MAD. * * Clients receiving MADs through their ib_mad_recv_handler must call this * routine to return the work completion buffers to the access layer. */ void ib_free_recv_mad(struct ib_mad_recv_wc *mad_recv_wc); /** * ib_cancel_mad - Cancels an outstanding send MAD operation. * @mad_agent: Specifies the registration associated with sent MAD. * @send_buf: Indicates the MAD to cancel. * * MADs will be returned to the user through the corresponding * ib_mad_send_handler. */ void ib_cancel_mad(struct ib_mad_agent *mad_agent, struct ib_mad_send_buf *send_buf); /** * ib_modify_mad - Modifies an outstanding send MAD operation. * @mad_agent: Specifies the registration associated with sent MAD. * @send_buf: Indicates the MAD to modify. * @timeout_ms: New timeout value for sent MAD. * * This call will reset the timeout value for a sent MAD to the specified * value. */ int ib_modify_mad(struct ib_mad_agent *mad_agent, struct ib_mad_send_buf *send_buf, u32 timeout_ms); /** * ib_redirect_mad_qp - Registers a QP for MAD services. * @qp: Reference to a QP that requires MAD services. * @rmpp_version: If set, indicates that the client will send * and receive MADs that contain the RMPP header for the given version. * If set to 0, indicates that RMPP is not used by this client. * @send_handler: The completion callback routine invoked after a send * request has completed. * @recv_handler: The completion callback routine invoked for a received * MAD. * @context: User specified context associated with the registration. * * Use of this call allows clients to use MAD services, such as RMPP, * on user-owned QPs. After calling this routine, users may send * MADs on the specified QP by calling ib_mad_post_send. */ struct ib_mad_agent *ib_redirect_mad_qp(struct ib_qp *qp, u8 rmpp_version, ib_mad_send_handler send_handler, ib_mad_recv_handler recv_handler, void *context); /** * ib_process_mad_wc - Processes a work completion associated with a * MAD sent or received on a redirected QP. * @mad_agent: Specifies the registered MAD service using the redirected QP. * @wc: References a work completion associated with a sent or received * MAD segment. * * This routine is used to complete or continue processing on a MAD request. * If the work completion is associated with a send operation, calling * this routine is required to continue an RMPP transfer or to wait for a * corresponding response, if it is a request. If the work completion is * associated with a receive operation, calling this routine is required to * process an inbound or outbound RMPP transfer, or to match a response MAD * with its corresponding request. */ int ib_process_mad_wc(struct ib_mad_agent *mad_agent, struct ib_wc *wc); /** * ib_create_send_mad - Allocate and initialize a data buffer and work request * for sending a MAD. * @mad_agent: Specifies the registered MAD service to associate with the MAD. * @remote_qpn: Specifies the QPN of the receiving node. * @pkey_index: Specifies which PKey the MAD will be sent using. This field * is valid only if the remote_qpn is QP 1. * @rmpp_active: Indicates if the send will enable RMPP. * @hdr_len: Indicates the size of the data header of the MAD. This length * should include the common MAD header, RMPP header, plus any class * specific header. * @data_len: Indicates the size of any user-transferred data. The call will * automatically adjust the allocated buffer size to account for any * additional padding that may be necessary. * @gfp_mask: GFP mask used for the memory allocation. * * This routine allocates a MAD for sending. The returned MAD send buffer * will reference a data buffer usable for sending a MAD, along * with an initialized work request structure. Users may modify the returned * MAD data buffer before posting the send. * * The returned MAD header, class specific headers, and any padding will be * cleared. Users are responsible for initializing the common MAD header, * any class specific header, and MAD data area. * If @rmpp_active is set, the RMPP header will be initialized for sending. */ struct ib_mad_send_buf *ib_create_send_mad(struct ib_mad_agent *mad_agent, u32 remote_qpn, u16 pkey_index, int rmpp_active, int hdr_len, int data_len, gfp_t gfp_mask); /** * ib_is_mad_class_rmpp - returns whether given management class * supports RMPP. * @mgmt_class: management class * * This routine returns whether the management class supports RMPP. */ int ib_is_mad_class_rmpp(u8 mgmt_class); /** * ib_get_mad_data_offset - returns the data offset for a given * management class. * @mgmt_class: management class * * This routine returns the data offset in the MAD for the management * class requested. */ int ib_get_mad_data_offset(u8 mgmt_class); /** * ib_get_rmpp_segment - returns the data buffer for a given RMPP segment. * @send_buf: Previously allocated send data buffer. * @seg_num: number of segment to return * * This routine returns a pointer to the data buffer of an RMPP MAD. * Users must provide synchronization to @send_buf around this call. */ void *ib_get_rmpp_segment(struct ib_mad_send_buf *send_buf, int seg_num); /** * ib_free_send_mad - Returns data buffers used to send a MAD. * @send_buf: Previously allocated send data buffer. */ void ib_free_send_mad(struct ib_mad_send_buf *send_buf); #endif /* IB_MAD_H */ Index: stable/9/sys/ofed/include/rdma/ib_pma.h =================================================================== --- stable/9/sys/ofed/include/rdma/ib_pma.h (nonexistent) +++ stable/9/sys/ofed/include/rdma/ib_pma.h (revision 273864) @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation. + * All rights reserved. + * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if !defined(IB_PMA_H) +#define IB_PMA_H + +#include + +/* + * PMA class portinfo capability mask bits + */ +#define IB_PMA_CLASS_CAP_ALLPORTSELECT cpu_to_be16(1 << 8) +#define IB_PMA_CLASS_CAP_EXT_WIDTH cpu_to_be16(1 << 9) +#define IB_PMA_CLASS_CAP_XMIT_WAIT cpu_to_be16(1 << 12) + +#define IB_PMA_CLASS_PORT_INFO cpu_to_be16(0x0001) +#define IB_PMA_PORT_SAMPLES_CONTROL cpu_to_be16(0x0010) +#define IB_PMA_PORT_SAMPLES_RESULT cpu_to_be16(0x0011) +#define IB_PMA_PORT_COUNTERS cpu_to_be16(0x0012) +#define IB_PMA_PORT_COUNTERS_EXT cpu_to_be16(0x001D) +#define IB_PMA_PORT_SAMPLES_RESULT_EXT cpu_to_be16(0x001E) + +struct ib_pma_mad { + struct ib_mad_hdr mad_hdr; + u8 reserved[40]; + u8 data[192]; +} __packed; + +struct ib_pma_portsamplescontrol { + u8 opcode; + u8 port_select; + u8 tick; + u8 counter_width; /* resv: 7:3, counter width: 2:0 */ + __be32 counter_mask0_9; /* 2, 10 3-bit fields */ + __be16 counter_mask10_14; /* 1, 5 3-bit fields */ + u8 sample_mechanisms; + u8 sample_status; /* only lower 2 bits */ + __be64 option_mask; + __be64 vendor_mask; + __be32 sample_start; + __be32 sample_interval; + __be16 tag; + __be16 counter_select[15]; + __be32 reserved1; + __be64 samples_only_option_mask; + __be32 reserved2[28]; +}; + +struct ib_pma_portsamplesresult { + __be16 tag; + __be16 sample_status; /* only lower 2 bits */ + __be32 counter[15]; +}; + +struct ib_pma_portsamplesresult_ext { + __be16 tag; + __be16 sample_status; /* only lower 2 bits */ + __be32 extended_width; /* only upper 2 bits */ + __be64 counter[15]; +}; + +struct ib_pma_portcounters { + u8 reserved; + u8 port_select; + __be16 counter_select; + __be16 symbol_error_counter; + u8 link_error_recovery_counter; + u8 link_downed_counter; + __be16 port_rcv_errors; + __be16 port_rcv_remphys_errors; + __be16 port_rcv_switch_relay_errors; + __be16 port_xmit_discards; + u8 port_xmit_constraint_errors; + u8 port_rcv_constraint_errors; + u8 reserved1; + u8 link_overrun_errors; /* LocalLink: 7:4, BufferOverrun: 3:0 */ + __be16 reserved2; + __be16 vl15_dropped; + __be32 port_xmit_data; + __be32 port_rcv_data; + __be32 port_xmit_packets; + __be32 port_rcv_packets; + __be32 port_xmit_wait; +} __packed; + + +#define IB_PMA_SEL_SYMBOL_ERROR cpu_to_be16(0x0001) +#define IB_PMA_SEL_LINK_ERROR_RECOVERY cpu_to_be16(0x0002) +#define IB_PMA_SEL_LINK_DOWNED cpu_to_be16(0x0004) +#define IB_PMA_SEL_PORT_RCV_ERRORS cpu_to_be16(0x0008) +#define IB_PMA_SEL_PORT_RCV_REMPHYS_ERRORS cpu_to_be16(0x0010) +#define IB_PMA_SEL_PORT_XMIT_DISCARDS cpu_to_be16(0x0040) +#define IB_PMA_SEL_LOCAL_LINK_INTEGRITY_ERRORS cpu_to_be16(0x0200) +#define IB_PMA_SEL_EXCESSIVE_BUFFER_OVERRUNS cpu_to_be16(0x0400) +#define IB_PMA_SEL_PORT_VL15_DROPPED cpu_to_be16(0x0800) +#define IB_PMA_SEL_PORT_XMIT_DATA cpu_to_be16(0x1000) +#define IB_PMA_SEL_PORT_RCV_DATA cpu_to_be16(0x2000) +#define IB_PMA_SEL_PORT_XMIT_PACKETS cpu_to_be16(0x4000) +#define IB_PMA_SEL_PORT_RCV_PACKETS cpu_to_be16(0x8000) + +struct ib_pma_portcounters_ext { + u8 reserved; + u8 port_select; + __be16 counter_select; + __be32 reserved1; + __be64 port_xmit_data; + __be64 port_rcv_data; + __be64 port_xmit_packets; + __be64 port_rcv_packets; + __be64 port_unicast_xmit_packets; + __be64 port_unicast_rcv_packets; + __be64 port_multicast_xmit_packets; + __be64 port_multicast_rcv_packets; +} __packed; + +#define IB_PMA_SELX_PORT_XMIT_DATA cpu_to_be16(0x0001) +#define IB_PMA_SELX_PORT_RCV_DATA cpu_to_be16(0x0002) +#define IB_PMA_SELX_PORT_XMIT_PACKETS cpu_to_be16(0x0004) +#define IB_PMA_SELX_PORT_RCV_PACKETS cpu_to_be16(0x0008) +#define IB_PMA_SELX_PORT_UNI_XMIT_PACKETS cpu_to_be16(0x0010) +#define IB_PMA_SELX_PORT_UNI_RCV_PACKETS cpu_to_be16(0x0020) +#define IB_PMA_SELX_PORT_MULTI_XMIT_PACKETS cpu_to_be16(0x0040) +#define IB_PMA_SELX_PORT_MULTI_RCV_PACKETS cpu_to_be16(0x0080) + +#endif /* IB_PMA_H */ Property changes on: stable/9/sys/ofed/include/rdma/ib_pma.h ___________________________________________________________________ Added: fbsd:nokeywords ## -0,0 +1 ## +true \ No newline at end of property Index: stable/9/sys/ofed/include/rdma/ib_sa.h =================================================================== --- stable/9/sys/ofed/include/rdma/ib_sa.h (revision 273863) +++ stable/9/sys/ofed/include/rdma/ib_sa.h (revision 273864) @@ -1,559 +1,593 @@ /* * Copyright (c) 2004 Topspin Communications. All rights reserved. * Copyright (c) 2005 Voltaire, Inc. All rights reserved. * Copyright (c) 2006 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef IB_SA_H #define IB_SA_H #include #include #include #include #include enum { IB_SA_CLASS_VERSION = 2, /* IB spec version 1.1/1.2 */ IB_SA_METHOD_GET_TABLE = 0x12, IB_SA_METHOD_GET_TABLE_RESP = 0x92, IB_SA_METHOD_DELETE = 0x15, IB_SA_METHOD_DELETE_RESP = 0x95, IB_SA_METHOD_GET_MULTI = 0x14, IB_SA_METHOD_GET_MULTI_RESP = 0x94, IB_SA_METHOD_GET_TRACE_TBL = 0x13 }; enum { IB_SA_ATTR_CLASS_PORTINFO = 0x01, IB_SA_ATTR_NOTICE = 0x02, IB_SA_ATTR_INFORM_INFO = 0x03, IB_SA_ATTR_NODE_REC = 0x11, IB_SA_ATTR_PORT_INFO_REC = 0x12, IB_SA_ATTR_SL2VL_REC = 0x13, IB_SA_ATTR_SWITCH_REC = 0x14, IB_SA_ATTR_LINEAR_FDB_REC = 0x15, IB_SA_ATTR_RANDOM_FDB_REC = 0x16, IB_SA_ATTR_MCAST_FDB_REC = 0x17, IB_SA_ATTR_SM_INFO_REC = 0x18, IB_SA_ATTR_LINK_REC = 0x20, IB_SA_ATTR_GUID_INFO_REC = 0x30, IB_SA_ATTR_SERVICE_REC = 0x31, IB_SA_ATTR_PARTITION_REC = 0x33, IB_SA_ATTR_PATH_REC = 0x35, IB_SA_ATTR_VL_ARB_REC = 0x36, IB_SA_ATTR_MC_MEMBER_REC = 0x38, IB_SA_ATTR_TRACE_REC = 0x39, IB_SA_ATTR_MULTI_PATH_REC = 0x3a, IB_SA_ATTR_SERVICE_ASSOC_REC = 0x3b, IB_SA_ATTR_INFORM_INFO_REC = 0xf3 }; enum ib_sa_selector { IB_SA_GT = 0, IB_SA_LT = 1, IB_SA_EQ = 2, /* * The meaning of "best" depends on the attribute: for * example, for MTU best will return the largest available * MTU, while for packet life time, best will return the * smallest available life time. */ IB_SA_BEST = 3 }; /* * Structures for SA records are named "struct ib_sa_xxx_rec." No * attempt is made to pack structures to match the physical layout of * SA records in SA MADs; all packing and unpacking is handled by the * SA query code. * * For a record with structure ib_sa_xxx_rec, the naming convention * for the component mask value for field yyy is IB_SA_XXX_REC_YYY (we * never use different abbreviations or otherwise change the spelling * of xxx/yyy between ib_sa_xxx_rec.yyy and IB_SA_XXX_REC_YYY). * * Reserved rows are indicated with comments to help maintainability. */ #define IB_SA_PATH_REC_SERVICE_ID (IB_SA_COMP_MASK( 0) |\ IB_SA_COMP_MASK( 1)) #define IB_SA_PATH_REC_DGID IB_SA_COMP_MASK( 2) #define IB_SA_PATH_REC_SGID IB_SA_COMP_MASK( 3) #define IB_SA_PATH_REC_DLID IB_SA_COMP_MASK( 4) #define IB_SA_PATH_REC_SLID IB_SA_COMP_MASK( 5) #define IB_SA_PATH_REC_RAW_TRAFFIC IB_SA_COMP_MASK( 6) /* reserved: 7 */ #define IB_SA_PATH_REC_FLOW_LABEL IB_SA_COMP_MASK( 8) #define IB_SA_PATH_REC_HOP_LIMIT IB_SA_COMP_MASK( 9) #define IB_SA_PATH_REC_TRAFFIC_CLASS IB_SA_COMP_MASK(10) #define IB_SA_PATH_REC_REVERSIBLE IB_SA_COMP_MASK(11) #define IB_SA_PATH_REC_NUMB_PATH IB_SA_COMP_MASK(12) #define IB_SA_PATH_REC_PKEY IB_SA_COMP_MASK(13) #define IB_SA_PATH_REC_QOS_CLASS IB_SA_COMP_MASK(14) #define IB_SA_PATH_REC_SL IB_SA_COMP_MASK(15) #define IB_SA_PATH_REC_MTU_SELECTOR IB_SA_COMP_MASK(16) #define IB_SA_PATH_REC_MTU IB_SA_COMP_MASK(17) #define IB_SA_PATH_REC_RATE_SELECTOR IB_SA_COMP_MASK(18) #define IB_SA_PATH_REC_RATE IB_SA_COMP_MASK(19) #define IB_SA_PATH_REC_PACKET_LIFE_TIME_SELECTOR IB_SA_COMP_MASK(20) #define IB_SA_PATH_REC_PACKET_LIFE_TIME IB_SA_COMP_MASK(21) #define IB_SA_PATH_REC_PREFERENCE IB_SA_COMP_MASK(22) struct ib_sa_path_rec { __be64 service_id; union ib_gid dgid; union ib_gid sgid; __be16 dlid; __be16 slid; int raw_traffic; /* reserved */ __be32 flow_label; u8 hop_limit; u8 traffic_class; int reversible; u8 numb_path; __be16 pkey; __be16 qos_class; u8 sl; u8 mtu_selector; u8 mtu; u8 rate_selector; u8 rate; u8 packet_life_time_selector; u8 packet_life_time; u8 preference; }; #define IB_SA_MCMEMBER_REC_MGID IB_SA_COMP_MASK( 0) #define IB_SA_MCMEMBER_REC_PORT_GID IB_SA_COMP_MASK( 1) #define IB_SA_MCMEMBER_REC_QKEY IB_SA_COMP_MASK( 2) #define IB_SA_MCMEMBER_REC_MLID IB_SA_COMP_MASK( 3) #define IB_SA_MCMEMBER_REC_MTU_SELECTOR IB_SA_COMP_MASK( 4) #define IB_SA_MCMEMBER_REC_MTU IB_SA_COMP_MASK( 5) #define IB_SA_MCMEMBER_REC_TRAFFIC_CLASS IB_SA_COMP_MASK( 6) #define IB_SA_MCMEMBER_REC_PKEY IB_SA_COMP_MASK( 7) #define IB_SA_MCMEMBER_REC_RATE_SELECTOR IB_SA_COMP_MASK( 8) #define IB_SA_MCMEMBER_REC_RATE IB_SA_COMP_MASK( 9) #define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR IB_SA_COMP_MASK(10) #define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME IB_SA_COMP_MASK(11) #define IB_SA_MCMEMBER_REC_SL IB_SA_COMP_MASK(12) #define IB_SA_MCMEMBER_REC_FLOW_LABEL IB_SA_COMP_MASK(13) #define IB_SA_MCMEMBER_REC_HOP_LIMIT IB_SA_COMP_MASK(14) #define IB_SA_MCMEMBER_REC_SCOPE IB_SA_COMP_MASK(15) #define IB_SA_MCMEMBER_REC_JOIN_STATE IB_SA_COMP_MASK(16) #define IB_SA_MCMEMBER_REC_PROXY_JOIN IB_SA_COMP_MASK(17) struct ib_sa_mcmember_rec { union ib_gid mgid; union ib_gid port_gid; __be32 qkey; __be16 mlid; u8 mtu_selector; u8 mtu; u8 traffic_class; __be16 pkey; u8 rate_selector; u8 rate; u8 packet_life_time_selector; u8 packet_life_time; u8 sl; __be32 flow_label; u8 hop_limit; u8 scope; u8 join_state; int proxy_join; }; /* Service Record Component Mask Sec 15.2.5.14 Ver 1.1 */ #define IB_SA_SERVICE_REC_SERVICE_ID IB_SA_COMP_MASK( 0) #define IB_SA_SERVICE_REC_SERVICE_GID IB_SA_COMP_MASK( 1) #define IB_SA_SERVICE_REC_SERVICE_PKEY IB_SA_COMP_MASK( 2) /* reserved: 3 */ #define IB_SA_SERVICE_REC_SERVICE_LEASE IB_SA_COMP_MASK( 4) #define IB_SA_SERVICE_REC_SERVICE_KEY IB_SA_COMP_MASK( 5) #define IB_SA_SERVICE_REC_SERVICE_NAME IB_SA_COMP_MASK( 6) #define IB_SA_SERVICE_REC_SERVICE_DATA8_0 IB_SA_COMP_MASK( 7) #define IB_SA_SERVICE_REC_SERVICE_DATA8_1 IB_SA_COMP_MASK( 8) #define IB_SA_SERVICE_REC_SERVICE_DATA8_2 IB_SA_COMP_MASK( 9) #define IB_SA_SERVICE_REC_SERVICE_DATA8_3 IB_SA_COMP_MASK(10) #define IB_SA_SERVICE_REC_SERVICE_DATA8_4 IB_SA_COMP_MASK(11) #define IB_SA_SERVICE_REC_SERVICE_DATA8_5 IB_SA_COMP_MASK(12) #define IB_SA_SERVICE_REC_SERVICE_DATA8_6 IB_SA_COMP_MASK(13) #define IB_SA_SERVICE_REC_SERVICE_DATA8_7 IB_SA_COMP_MASK(14) #define IB_SA_SERVICE_REC_SERVICE_DATA8_8 IB_SA_COMP_MASK(15) #define IB_SA_SERVICE_REC_SERVICE_DATA8_9 IB_SA_COMP_MASK(16) #define IB_SA_SERVICE_REC_SERVICE_DATA8_10 IB_SA_COMP_MASK(17) #define IB_SA_SERVICE_REC_SERVICE_DATA8_11 IB_SA_COMP_MASK(18) #define IB_SA_SERVICE_REC_SERVICE_DATA8_12 IB_SA_COMP_MASK(19) #define IB_SA_SERVICE_REC_SERVICE_DATA8_13 IB_SA_COMP_MASK(20) #define IB_SA_SERVICE_REC_SERVICE_DATA8_14 IB_SA_COMP_MASK(21) #define IB_SA_SERVICE_REC_SERVICE_DATA8_15 IB_SA_COMP_MASK(22) #define IB_SA_SERVICE_REC_SERVICE_DATA16_0 IB_SA_COMP_MASK(23) #define IB_SA_SERVICE_REC_SERVICE_DATA16_1 IB_SA_COMP_MASK(24) #define IB_SA_SERVICE_REC_SERVICE_DATA16_2 IB_SA_COMP_MASK(25) #define IB_SA_SERVICE_REC_SERVICE_DATA16_3 IB_SA_COMP_MASK(26) #define IB_SA_SERVICE_REC_SERVICE_DATA16_4 IB_SA_COMP_MASK(27) #define IB_SA_SERVICE_REC_SERVICE_DATA16_5 IB_SA_COMP_MASK(28) #define IB_SA_SERVICE_REC_SERVICE_DATA16_6 IB_SA_COMP_MASK(29) #define IB_SA_SERVICE_REC_SERVICE_DATA16_7 IB_SA_COMP_MASK(30) #define IB_SA_SERVICE_REC_SERVICE_DATA32_0 IB_SA_COMP_MASK(31) #define IB_SA_SERVICE_REC_SERVICE_DATA32_1 IB_SA_COMP_MASK(32) #define IB_SA_SERVICE_REC_SERVICE_DATA32_2 IB_SA_COMP_MASK(33) #define IB_SA_SERVICE_REC_SERVICE_DATA32_3 IB_SA_COMP_MASK(34) #define IB_SA_SERVICE_REC_SERVICE_DATA64_0 IB_SA_COMP_MASK(35) #define IB_SA_SERVICE_REC_SERVICE_DATA64_1 IB_SA_COMP_MASK(36) #define IB_DEFAULT_SERVICE_LEASE 0xFFFFFFFF struct ib_sa_service_rec { u64 id; union ib_gid gid; __be16 pkey; /* reserved */ u32 lease; u8 key[16]; u8 name[64]; u8 data8[16]; u16 data16[8]; u32 data32[4]; u64 data64[2]; }; enum { IB_SA_EVENT_TYPE_FATAL = 0x0, IB_SA_EVENT_TYPE_URGENT = 0x1, IB_SA_EVENT_TYPE_SECURITY = 0x2, IB_SA_EVENT_TYPE_SM = 0x3, IB_SA_EVENT_TYPE_INFO = 0x4, IB_SA_EVENT_TYPE_EMPTY = 0x7F, IB_SA_EVENT_TYPE_ALL = 0xFFFF }; enum { IB_SA_EVENT_PRODUCER_TYPE_CA = 0x1, IB_SA_EVENT_PRODUCER_TYPE_SWITCH = 0x2, IB_SA_EVENT_PRODUCER_TYPE_ROUTER = 0x3, IB_SA_EVENT_PRODUCER_TYPE_CLASS_MANAGER = 0x4, IB_SA_EVENT_PRODUCER_TYPE_ALL = 0xFFFFFF }; enum { IB_SA_SM_TRAP_GID_IN_SERVICE = 64, IB_SA_SM_TRAP_GID_OUT_OF_SERVICE = 65, IB_SA_SM_TRAP_CREATE_MC_GROUP = 66, IB_SA_SM_TRAP_DELETE_MC_GROUP = 67, IB_SA_SM_TRAP_PORT_CHANGE_STATE = 128, IB_SA_SM_TRAP_LINK_INTEGRITY = 129, IB_SA_SM_TRAP_EXCESSIVE_BUFFER_OVERRUN = 130, IB_SA_SM_TRAP_FLOW_CONTROL_UPDATE_EXPIRED = 131, IB_SA_SM_TRAP_BAD_M_KEY = 256, IB_SA_SM_TRAP_BAD_P_KEY = 257, IB_SA_SM_TRAP_BAD_Q_KEY = 258, IB_SA_SM_TRAP_SWITCH_BAD_P_KEY = 259, IB_SA_SM_TRAP_ALL = 0xFFFF }; struct ib_sa_inform { union ib_gid gid; __be16 lid_range_begin; __be16 lid_range_end; u8 is_generic; u8 subscribe; __be16 type; union { struct { __be16 trap_num; __be32 qpn; u8 resp_time; __be32 producer_type; } generic; struct { __be16 device_id; __be32 qpn; u8 resp_time; __be32 vendor_id; } vendor; } trap; }; struct ib_sa_notice { u8 is_generic; u8 type; union { struct { __be32 producer_type; __be16 trap_num; } generic; struct { __be32 vendor_id; __be16 device_id; } vendor; } trap; __be16 issuer_lid; __be16 notice_count; u8 notice_toggle; /* * Align data 16 bits off 64 bit field to match InformInfo definition. * Data contained within this field will then align properly. * See IB spec 1.2, sections 13.4.8.2 and 14.2.5.1. */ u8 reserved[5]; u8 data_details[54]; union ib_gid issuer_gid; }; /* * SM notice data details for: * * IB_SA_SM_TRAP_GID_IN_SERVICE = 64 * IB_SA_SM_TRAP_GID_OUT_OF_SERVICE = 65 * IB_SA_SM_TRAP_CREATE_MC_GROUP = 66 * IB_SA_SM_TRAP_DELETE_MC_GROUP = 67 */ struct ib_sa_notice_data_gid { u8 reserved[6]; u8 gid[16]; u8 padding[32]; }; /* * SM notice data details for: * * IB_SA_SM_TRAP_PORT_CHANGE_STATE = 128 */ struct ib_sa_notice_data_port_change { __be16 lid; u8 padding[52]; }; /* * SM notice data details for: * * IB_SA_SM_TRAP_LINK_INTEGRITY = 129 * IB_SA_SM_TRAP_EXCESSIVE_BUFFER_OVERRUN = 130 * IB_SA_SM_TRAP_FLOW_CONTROL_UPDATE_EXPIRED = 131 */ struct ib_sa_notice_data_port_error { u8 reserved[2]; __be16 lid; u8 port_num; u8 padding[49]; }; +#define IB_SA_GUIDINFO_REC_LID IB_SA_COMP_MASK(0) +#define IB_SA_GUIDINFO_REC_BLOCK_NUM IB_SA_COMP_MASK(1) +#define IB_SA_GUIDINFO_REC_RES1 IB_SA_COMP_MASK(2) +#define IB_SA_GUIDINFO_REC_RES2 IB_SA_COMP_MASK(3) +#define IB_SA_GUIDINFO_REC_GID0 IB_SA_COMP_MASK(4) +#define IB_SA_GUIDINFO_REC_GID1 IB_SA_COMP_MASK(5) +#define IB_SA_GUIDINFO_REC_GID2 IB_SA_COMP_MASK(6) +#define IB_SA_GUIDINFO_REC_GID3 IB_SA_COMP_MASK(7) +#define IB_SA_GUIDINFO_REC_GID4 IB_SA_COMP_MASK(8) +#define IB_SA_GUIDINFO_REC_GID5 IB_SA_COMP_MASK(9) +#define IB_SA_GUIDINFO_REC_GID6 IB_SA_COMP_MASK(10) +#define IB_SA_GUIDINFO_REC_GID7 IB_SA_COMP_MASK(11) + +struct ib_sa_guidinfo_rec { + __be16 lid; + u8 block_num; + /* reserved */ + u8 res1; + __be32 res2; + u8 guid_info_list[64]; +}; + struct ib_sa_client { atomic_t users; struct completion comp; }; /** * ib_sa_register_client - Register an SA client. */ void ib_sa_register_client(struct ib_sa_client *client); /** * ib_sa_unregister_client - Deregister an SA client. * @client: Client object to deregister. */ void ib_sa_unregister_client(struct ib_sa_client *client); struct ib_sa_query; void ib_sa_cancel_query(int id, struct ib_sa_query *query); int ib_sa_path_rec_get(struct ib_sa_client *client, struct ib_device *device, u8 port_num, struct ib_sa_path_rec *rec, ib_sa_comp_mask comp_mask, int timeout_ms, gfp_t gfp_mask, void (*callback)(int status, struct ib_sa_path_rec *resp, void *context), void *context, struct ib_sa_query **query); int ib_sa_service_rec_query(struct ib_sa_client *client, struct ib_device *device, u8 port_num, u8 method, struct ib_sa_service_rec *rec, ib_sa_comp_mask comp_mask, int timeout_ms, gfp_t gfp_mask, void (*callback)(int status, struct ib_sa_service_rec *resp, void *context), void *context, struct ib_sa_query **sa_query); struct ib_sa_multicast { struct ib_sa_mcmember_rec rec; ib_sa_comp_mask comp_mask; int (*callback)(int status, struct ib_sa_multicast *multicast); void *context; }; /** * ib_sa_join_multicast - Initiates a join request to the specified multicast * group. * @client: SA client * @device: Device associated with the multicast group. * @port_num: Port on the specified device to associate with the multicast * group. * @rec: SA multicast member record specifying group attributes. * @comp_mask: Component mask indicating which group attributes of %rec are * valid. * @gfp_mask: GFP mask for memory allocations. * @callback: User callback invoked once the join operation completes. * @context: User specified context stored with the ib_sa_multicast structure. * * This call initiates a multicast join request with the SA for the specified * multicast group. If the join operation is started successfully, it returns * an ib_sa_multicast structure that is used to track the multicast operation. * Users must free this structure by calling ib_free_multicast, even if the * join operation later fails. (The callback status is non-zero.) * * If the join operation fails; status will be non-zero, with the following * failures possible: * -ETIMEDOUT: The request timed out. * -EIO: An error occurred sending the query. * -EINVAL: The MCMemberRecord values differed from the existing group's. * -ENETRESET: Indicates that an fatal error has occurred on the multicast * group, and the user must rejoin the group to continue using it. */ struct ib_sa_multicast *ib_sa_join_multicast(struct ib_sa_client *client, struct ib_device *device, u8 port_num, struct ib_sa_mcmember_rec *rec, ib_sa_comp_mask comp_mask, gfp_t gfp_mask, int (*callback)(int status, struct ib_sa_multicast *multicast), void *context); /** * ib_free_multicast - Frees the multicast tracking structure, and releases * any reference on the multicast group. * @multicast: Multicast tracking structure allocated by ib_join_multicast. * * This call blocks until the multicast identifier is destroyed. It may * not be called from within the multicast callback; however, returning a non- * zero value from the callback will result in destroying the multicast * tracking structure. */ void ib_sa_free_multicast(struct ib_sa_multicast *multicast); /** * ib_get_mcmember_rec - Looks up a multicast member record by its MGID and * returns it if found. * @device: Device associated with the multicast group. * @port_num: Port on the specified device to associate with the multicast * group. * @mgid: MGID of multicast group. * @rec: Location to copy SA multicast member record. */ int ib_sa_get_mcmember_rec(struct ib_device *device, u8 port_num, union ib_gid *mgid, struct ib_sa_mcmember_rec *rec); /** * ib_init_ah_from_mcmember - Initialize address handle attributes based on * an SA multicast member record. */ int ib_init_ah_from_mcmember(struct ib_device *device, u8 port_num, struct ib_sa_mcmember_rec *rec, struct ib_ah_attr *ah_attr); /** * ib_init_ah_from_path - Initialize address handle attributes based on an SA * path record. */ int ib_init_ah_from_path(struct ib_device *device, u8 port_num, struct ib_sa_path_rec *rec, struct ib_ah_attr *ah_attr); /** * ib_sa_unpack_path - Convert a path record from MAD format to struct * ib_sa_path_rec. */ void ib_sa_unpack_path(void *attribute, struct ib_sa_path_rec *rec); struct ib_inform_info { void *context; int (*callback)(int status, struct ib_inform_info *info, struct ib_sa_notice *notice); u16 trap_number; }; /** * ib_sa_register_inform_info - Registers to receive notice events. * @device: Device associated with the registration. * @port_num: Port on the specified device to associate with the registration. * @trap_number: InformInfo trap number to register for. * @gfp_mask: GFP mask for memory allocations. * @callback: User callback invoked once the registration completes and to * report noticed events. * @context: User specified context stored with the ib_inform_reg structure. * * This call initiates a registration request with the SA for the specified * trap number. If the operation is started successfully, it returns * an ib_inform_info structure that is used to track the registration operation. * Users must free this structure by calling ib_unregister_inform_info, * even if the operation later fails. (The callback status is non-zero.) * * If the registration fails; status will be non-zero. If the registration * succeeds, the callback status will be zero, but the notice parameter will * be NULL. If the notice parameter is not NULL, a trap or notice is being * reported to the user. * * A status of -ENETRESET indicates that an error occurred which requires * reregisteration. */ struct ib_inform_info * ib_sa_register_inform_info(struct ib_sa_client *client, struct ib_device *device, u8 port_num, u16 trap_number, gfp_t gfp_mask, int (*callback)(int status, struct ib_inform_info *info, struct ib_sa_notice *notice), void *context); /** * ib_sa_unregister_inform_info - Releases an InformInfo registration. * @info: InformInfo registration tracking structure. * * This call blocks until the registration request is destroyed. It may * not be called from within the registration callback. */ void ib_sa_unregister_inform_info(struct ib_inform_info *info); + +int ib_sa_guid_info_rec_query(struct ib_sa_client *client, + struct ib_device *device, u8 port_num, + struct ib_sa_guidinfo_rec *rec, + ib_sa_comp_mask comp_mask, u8 method, + int timeout_ms, gfp_t gfp_mask, + void (*callback)(int status, + struct ib_sa_guidinfo_rec *resp, + void *context), + void *context, + struct ib_sa_query **sa_query); + #endif /* IB_SA_H */ Index: stable/9/sys/ofed/include/rdma/ib_umem.h =================================================================== --- stable/9/sys/ofed/include/rdma/ib_umem.h (revision 273863) +++ stable/9/sys/ofed/include/rdma/ib_umem.h (revision 273864) @@ -1,73 +1,102 @@ /* * Copyright (c) 2007 Cisco Systems. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef IB_UMEM_H #define IB_UMEM_H #include #include #include #include struct ib_ucontext; +struct vm_area_struct; struct ib_umem { struct ib_ucontext *context; size_t length; int offset; int page_size; int writable; int hugetlb; struct list_head chunk_list; #ifdef __linux__ struct work_struct work; struct mm_struct *mm; #else unsigned long start; #endif unsigned long diff; }; +struct ib_cmem { + + struct ib_ucontext *context; + size_t length; + /* Link list of contiguous blocks being part of that cmem */ + struct list_head ib_cmem_block; + + /* Order of cmem block, 2^ block_order will equal number + of physical pages per block + */ + unsigned long block_order; + /* Refernce counter for that memory area + - When value became 0 pages will be returned to the kernel. + */ + struct kref refcount; +}; + + struct ib_umem_chunk { struct list_head list; int nents; int nmap; struct dma_attrs attrs; struct scatterlist page_list[0]; }; struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, size_t size, int access, int dmasync); void ib_umem_release(struct ib_umem *umem); int ib_umem_page_count(struct ib_umem *umem); + +int ib_cmem_map_contiguous_pages_to_vma(struct ib_cmem *ib_cmem, + struct vm_area_struct *vma); +struct ib_cmem *ib_cmem_alloc_contiguous_pages(struct ib_ucontext *context, + unsigned long total_size, + unsigned long page_size_order); +void ib_cmem_release_contiguous_pages(struct ib_cmem *cmem); +int ib_umem_map_to_vma(struct ib_umem *umem, + struct vm_area_struct *vma); + #endif /* IB_UMEM_H */ Index: stable/9/sys/ofed/include/rdma/ib_user_verbs.h =================================================================== --- stable/9/sys/ofed/include/rdma/ib_user_verbs.h (revision 273863) +++ stable/9/sys/ofed/include/rdma/ib_user_verbs.h (revision 273864) @@ -1,801 +1,882 @@ /* * Copyright (c) 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. * Copyright (c) 2005 PathScale, Inc. All rights reserved. * Copyright (c) 2006 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef IB_USER_VERBS_H #define IB_USER_VERBS_H #include /* * Increment this value if any changes that break userspace ABI * compatibility are made. */ #define IB_USER_VERBS_ABI_VERSION 6 enum { IB_USER_VERBS_CMD_GET_CONTEXT, IB_USER_VERBS_CMD_QUERY_DEVICE, IB_USER_VERBS_CMD_QUERY_PORT, IB_USER_VERBS_CMD_ALLOC_PD, IB_USER_VERBS_CMD_DEALLOC_PD, IB_USER_VERBS_CMD_CREATE_AH, IB_USER_VERBS_CMD_MODIFY_AH, IB_USER_VERBS_CMD_QUERY_AH, IB_USER_VERBS_CMD_DESTROY_AH, IB_USER_VERBS_CMD_REG_MR, IB_USER_VERBS_CMD_REG_SMR, IB_USER_VERBS_CMD_REREG_MR, IB_USER_VERBS_CMD_QUERY_MR, IB_USER_VERBS_CMD_DEREG_MR, IB_USER_VERBS_CMD_ALLOC_MW, IB_USER_VERBS_CMD_BIND_MW, IB_USER_VERBS_CMD_DEALLOC_MW, IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL, IB_USER_VERBS_CMD_CREATE_CQ, IB_USER_VERBS_CMD_RESIZE_CQ, IB_USER_VERBS_CMD_DESTROY_CQ, IB_USER_VERBS_CMD_POLL_CQ, IB_USER_VERBS_CMD_PEEK_CQ, IB_USER_VERBS_CMD_REQ_NOTIFY_CQ, IB_USER_VERBS_CMD_CREATE_QP, IB_USER_VERBS_CMD_QUERY_QP, IB_USER_VERBS_CMD_MODIFY_QP, IB_USER_VERBS_CMD_DESTROY_QP, IB_USER_VERBS_CMD_POST_SEND, IB_USER_VERBS_CMD_POST_RECV, IB_USER_VERBS_CMD_ATTACH_MCAST, IB_USER_VERBS_CMD_DETACH_MCAST, IB_USER_VERBS_CMD_CREATE_SRQ, IB_USER_VERBS_CMD_MODIFY_SRQ, IB_USER_VERBS_CMD_QUERY_SRQ, IB_USER_VERBS_CMD_DESTROY_SRQ, IB_USER_VERBS_CMD_POST_SRQ_RECV, + IB_USER_VERBS_CMD_OPEN_XRCD, + IB_USER_VERBS_CMD_CLOSE_XRCD, + IB_USER_VERBS_CMD_CREATE_XSRQ, + IB_USER_VERBS_CMD_OPEN_QP, + IB_USER_VERBS_CMD_ATTACH_FLOW, + IB_USER_VERBS_CMD_DETACH_FLOW, IB_USER_VERBS_CMD_CREATE_XRC_SRQ, - IB_USER_VERBS_CMD_OPEN_XRC_DOMAIN, - IB_USER_VERBS_CMD_CLOSE_XRC_DOMAIN, IB_USER_VERBS_CMD_CREATE_XRC_RCV_QP, IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP, IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP, IB_USER_VERBS_CMD_REG_XRC_RCV_QP, IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP, }; /* * Make sure that all structs defined in this file remain laid out so * that they pack the same way on 32-bit and 64-bit architectures (to * avoid incompatibility between 32-bit userspace and 64-bit kernels). * Specifically: * - Do not use pointer types -- pass pointers in __u64 instead. * - Make sure that any structure larger than 4 bytes is padded to a * multiple of 8 bytes. Otherwise the structure size will be * different between 32-bit and 64-bit architectures. */ struct ib_uverbs_async_event_desc { __u64 element; __u32 event_type; /* enum ib_event_type */ __u32 reserved; }; struct ib_uverbs_comp_event_desc { __u64 cq_handle; }; /* * All commands from userspace should start with a __u32 command field * followed by __u16 in_words and out_words fields (which give the * length of the command block and response buffer if any in 32-bit * words). The kernel driver will read these fields first and read * the rest of the command struct based on these value. */ struct ib_uverbs_cmd_hdr { __u32 command; __u16 in_words; __u16 out_words; }; struct ib_uverbs_get_context { __u64 response; __u64 driver_data[0]; }; struct ib_uverbs_get_context_resp { __u32 async_fd; __u32 num_comp_vectors; }; struct ib_uverbs_query_device { __u64 response; __u64 driver_data[0]; }; struct ib_uverbs_query_device_resp { __u64 fw_ver; __be64 node_guid; __be64 sys_image_guid; __u64 max_mr_size; __u64 page_size_cap; __u32 vendor_id; __u32 vendor_part_id; __u32 hw_ver; __u32 max_qp; __u32 max_qp_wr; __u32 device_cap_flags; __u32 max_sge; __u32 max_sge_rd; __u32 max_cq; __u32 max_cqe; __u32 max_mr; __u32 max_pd; __u32 max_qp_rd_atom; __u32 max_ee_rd_atom; __u32 max_res_rd_atom; __u32 max_qp_init_rd_atom; __u32 max_ee_init_rd_atom; __u32 atomic_cap; __u32 max_ee; __u32 max_rdd; __u32 max_mw; __u32 max_raw_ipv6_qp; __u32 max_raw_ethy_qp; __u32 max_mcast_grp; __u32 max_mcast_qp_attach; __u32 max_total_mcast_qp_attach; __u32 max_ah; __u32 max_fmr; __u32 max_map_per_fmr; __u32 max_srq; __u32 max_srq_wr; __u32 max_srq_sge; __u16 max_pkeys; __u8 local_ca_ack_delay; __u8 phys_port_cnt; __u8 reserved[4]; }; struct ib_uverbs_query_port { __u64 response; __u8 port_num; __u8 reserved[7]; __u64 driver_data[0]; }; struct ib_uverbs_query_port_resp { __u32 port_cap_flags; __u32 max_msg_sz; __u32 bad_pkey_cntr; __u32 qkey_viol_cntr; __u32 gid_tbl_len; __u16 pkey_tbl_len; __u16 lid; __u16 sm_lid; __u8 state; __u8 max_mtu; __u8 active_mtu; __u8 lmc; __u8 max_vl_num; __u8 sm_sl; __u8 subnet_timeout; __u8 init_type_reply; __u8 active_width; __u8 active_speed; __u8 phys_state; __u8 link_layer; __u8 reserved[2]; }; struct ib_uverbs_alloc_pd { __u64 response; __u64 driver_data[0]; }; struct ib_uverbs_alloc_pd_resp { __u32 pd_handle; }; struct ib_uverbs_dealloc_pd { __u32 pd_handle; }; +struct ib_uverbs_open_xrcd { + __u64 response; + __u32 fd; + __u32 oflags; + __u64 driver_data[0]; +}; + +struct ib_uverbs_open_xrcd_resp { + __u32 xrcd_handle; +}; + +struct ib_uverbs_close_xrcd { + __u32 xrcd_handle; +}; + struct ib_uverbs_reg_mr { __u64 response; __u64 start; __u64 length; __u64 hca_va; __u32 pd_handle; __u32 access_flags; __u64 driver_data[0]; }; struct ib_uverbs_reg_mr_resp { __u32 mr_handle; __u32 lkey; __u32 rkey; }; struct ib_uverbs_dereg_mr { __u32 mr_handle; }; struct ib_uverbs_create_comp_channel { __u64 response; }; struct ib_uverbs_create_comp_channel_resp { __u32 fd; }; struct ib_uverbs_create_cq { __u64 response; __u64 user_handle; __u32 cqe; __u32 comp_vector; __s32 comp_channel; __u32 reserved; __u64 driver_data[0]; }; struct ib_uverbs_create_cq_resp { __u32 cq_handle; __u32 cqe; }; struct ib_uverbs_resize_cq { __u64 response; __u32 cq_handle; __u32 cqe; __u64 driver_data[0]; }; struct ib_uverbs_resize_cq_resp { __u32 cqe; __u32 reserved; __u64 driver_data[0]; }; struct ib_uverbs_poll_cq { __u64 response; __u32 cq_handle; __u32 ne; }; struct ib_uverbs_wc { __u64 wr_id; __u32 status; __u32 opcode; __u32 vendor_err; __u32 byte_len; union { __u32 imm_data; __u32 invalidate_rkey; } ex; __u32 qp_num; __u32 src_qp; __u32 wc_flags; __u16 pkey_index; __u16 slid; __u8 sl; __u8 dlid_path_bits; __u8 port_num; __u8 reserved; }; struct ib_uverbs_poll_cq_resp { __u32 count; __u32 reserved; struct ib_uverbs_wc wc[0]; }; struct ib_uverbs_req_notify_cq { __u32 cq_handle; __u32 solicited_only; }; struct ib_uverbs_destroy_cq { __u64 response; __u32 cq_handle; __u32 reserved; }; struct ib_uverbs_destroy_cq_resp { __u32 comp_events_reported; __u32 async_events_reported; }; struct ib_uverbs_global_route { __u8 dgid[16]; __u32 flow_label; __u8 sgid_index; __u8 hop_limit; __u8 traffic_class; __u8 reserved; }; struct ib_uverbs_ah_attr { struct ib_uverbs_global_route grh; __u16 dlid; __u8 sl; __u8 src_path_bits; __u8 static_rate; __u8 is_global; __u8 port_num; __u8 reserved; }; struct ib_uverbs_qp_attr { __u32 qp_attr_mask; __u32 qp_state; __u32 cur_qp_state; __u32 path_mtu; __u32 path_mig_state; __u32 qkey; __u32 rq_psn; __u32 sq_psn; __u32 dest_qp_num; __u32 qp_access_flags; struct ib_uverbs_ah_attr ah_attr; struct ib_uverbs_ah_attr alt_ah_attr; /* ib_qp_cap */ __u32 max_send_wr; __u32 max_recv_wr; __u32 max_send_sge; __u32 max_recv_sge; __u32 max_inline_data; __u16 pkey_index; __u16 alt_pkey_index; __u8 en_sqd_async_notify; __u8 sq_draining; __u8 max_rd_atomic; __u8 max_dest_rd_atomic; __u8 min_rnr_timer; __u8 port_num; __u8 timeout; __u8 retry_cnt; __u8 rnr_retry; __u8 alt_port_num; __u8 alt_timeout; __u8 reserved[5]; }; struct ib_uverbs_create_qp { __u64 response; __u64 user_handle; __u32 pd_handle; __u32 send_cq_handle; __u32 recv_cq_handle; __u32 srq_handle; __u32 max_send_wr; __u32 max_recv_wr; __u32 max_send_sge; __u32 max_recv_sge; __u32 max_inline_data; __u8 sq_sig_all; __u8 qp_type; __u8 is_srq; __u8 reserved; __u64 driver_data[0]; }; +struct ib_uverbs_open_qp { + __u64 response; + __u64 user_handle; + __u32 pd_handle; + __u32 qpn; + __u8 qp_type; + __u8 reserved[7]; + __u64 driver_data[0]; +}; + +/* also used for open response */ struct ib_uverbs_create_qp_resp { __u32 qp_handle; __u32 qpn; __u32 max_send_wr; __u32 max_recv_wr; __u32 max_send_sge; __u32 max_recv_sge; __u32 max_inline_data; __u32 reserved; }; /* * This struct needs to remain a multiple of 8 bytes to keep the * alignment of the modify QP parameters. */ struct ib_uverbs_qp_dest { __u8 dgid[16]; __u32 flow_label; __u16 dlid; __u16 reserved; __u8 sgid_index; __u8 hop_limit; __u8 traffic_class; __u8 sl; __u8 src_path_bits; __u8 static_rate; __u8 is_global; __u8 port_num; }; struct ib_uverbs_query_qp { __u64 response; __u32 qp_handle; __u32 attr_mask; __u64 driver_data[0]; }; struct ib_uverbs_query_qp_resp { struct ib_uverbs_qp_dest dest; struct ib_uverbs_qp_dest alt_dest; __u32 max_send_wr; __u32 max_recv_wr; __u32 max_send_sge; __u32 max_recv_sge; __u32 max_inline_data; __u32 qkey; __u32 rq_psn; __u32 sq_psn; __u32 dest_qp_num; __u32 qp_access_flags; __u16 pkey_index; __u16 alt_pkey_index; __u8 qp_state; __u8 cur_qp_state; __u8 path_mtu; __u8 path_mig_state; __u8 sq_draining; __u8 max_rd_atomic; __u8 max_dest_rd_atomic; __u8 min_rnr_timer; __u8 port_num; __u8 timeout; __u8 retry_cnt; __u8 rnr_retry; __u8 alt_port_num; __u8 alt_timeout; __u8 sq_sig_all; __u8 reserved[5]; __u64 driver_data[0]; }; struct ib_uverbs_modify_qp { struct ib_uverbs_qp_dest dest; struct ib_uverbs_qp_dest alt_dest; __u32 qp_handle; __u32 attr_mask; __u32 qkey; __u32 rq_psn; __u32 sq_psn; __u32 dest_qp_num; __u32 qp_access_flags; __u16 pkey_index; __u16 alt_pkey_index; __u8 qp_state; __u8 cur_qp_state; __u8 path_mtu; __u8 path_mig_state; __u8 en_sqd_async_notify; __u8 max_rd_atomic; __u8 max_dest_rd_atomic; __u8 min_rnr_timer; __u8 port_num; __u8 timeout; __u8 retry_cnt; __u8 rnr_retry; __u8 alt_port_num; __u8 alt_timeout; __u8 reserved[2]; __u64 driver_data[0]; }; struct ib_uverbs_modify_qp_resp { }; struct ib_uverbs_destroy_qp { __u64 response; __u32 qp_handle; __u32 reserved; }; struct ib_uverbs_destroy_qp_resp { __u32 events_reported; }; /* * The ib_uverbs_sge structure isn't used anywhere, since we assume * the ib_sge structure is packed the same way on 32-bit and 64-bit * architectures in both kernel and user space. It's just here to * document the ABI. */ struct ib_uverbs_sge { __u64 addr; __u32 length; __u32 lkey; }; struct ib_uverbs_send_wr { __u64 wr_id; __u32 num_sge; __u32 opcode; __u32 send_flags; union { __u32 imm_data; __u32 invalidate_rkey; } ex; union { struct { __u64 remote_addr; __u32 rkey; __u32 reserved; } rdma; struct { __u64 remote_addr; __u64 compare_add; __u64 swap; __u32 rkey; __u32 reserved; } atomic; struct { __u32 ah; __u32 remote_qpn; __u32 remote_qkey; __u32 reserved; } ud; } wr; }; +struct ibv_uverbs_flow_spec { + __u32 type; + __be32 src_ip; + __be32 dst_ip; + __be16 src_port; + __be16 dst_port; + __u8 l4_protocol; + __u8 block_mc_loopback; +}; + struct ib_uverbs_post_send { __u64 response; __u32 qp_handle; __u32 wr_count; __u32 sge_count; __u32 wqe_size; struct ib_uverbs_send_wr send_wr[0]; }; struct ib_uverbs_post_send_resp { __u32 bad_wr; }; struct ib_uverbs_recv_wr { __u64 wr_id; __u32 num_sge; __u32 reserved; }; struct ib_uverbs_post_recv { __u64 response; __u32 qp_handle; __u32 wr_count; __u32 sge_count; __u32 wqe_size; struct ib_uverbs_recv_wr recv_wr[0]; }; struct ib_uverbs_post_recv_resp { __u32 bad_wr; }; struct ib_uverbs_post_srq_recv { __u64 response; __u32 srq_handle; __u32 wr_count; __u32 sge_count; __u32 wqe_size; struct ib_uverbs_recv_wr recv[0]; }; struct ib_uverbs_post_srq_recv_resp { __u32 bad_wr; }; struct ib_uverbs_create_ah { __u64 response; __u64 user_handle; __u32 pd_handle; __u32 reserved; struct ib_uverbs_ah_attr attr; }; struct ib_uverbs_create_ah_resp { __u32 ah_handle; }; struct ib_uverbs_destroy_ah { __u32 ah_handle; }; struct ib_uverbs_attach_mcast { __u8 gid[16]; __u32 qp_handle; __u16 mlid; __u16 reserved; __u64 driver_data[0]; }; struct ib_uverbs_detach_mcast { __u8 gid[16]; __u32 qp_handle; __u16 mlid; __u16 reserved; __u64 driver_data[0]; }; +struct ibv_kern_flow_spec { + __u32 type; + __u32 reserved1; + union { + struct { + __be16 ethertype; + __be16 vlan; + __u8 vlan_present; + __u8 mac[6]; + __u8 port; + } eth; + struct { + __be32 qpn; + } ib_uc; + struct { + __u8 mgid[16]; + } ib_mc; + } l2_id; + __be32 src_ip; + __be32 dst_ip; + __be16 src_port; + __be16 dst_port; + __u8 l4_protocol; + __u8 block_mc_loopback; + __u8 reserved[2]; +}; + +struct ib_uverbs_attach_flow { + __u32 qp_handle; + __u32 priority; + struct ibv_kern_flow_spec spec; +}; + +struct ib_uverbs_detach_flow { + __u32 qp_handle; + __u32 priority; + struct ibv_kern_flow_spec spec; +}; + struct ib_uverbs_create_srq { __u64 response; __u64 user_handle; __u32 pd_handle; __u32 max_wr; __u32 max_sge; __u32 srq_limit; __u64 driver_data[0]; }; -struct ib_uverbs_create_xrc_srq { +struct ib_uverbs_create_xsrq { __u64 response; __u64 user_handle; + __u32 srq_type; __u32 pd_handle; __u32 max_wr; __u32 max_sge; __u32 srq_limit; + __u32 reserved; __u32 xrcd_handle; - __u32 xrc_cq; + __u32 cq_handle; __u64 driver_data[0]; }; struct ib_uverbs_create_srq_resp { __u32 srq_handle; __u32 max_wr; __u32 max_sge; - __u32 reserved; + __u32 srqn; }; struct ib_uverbs_modify_srq { __u32 srq_handle; __u32 attr_mask; __u32 max_wr; __u32 srq_limit; __u64 driver_data[0]; }; struct ib_uverbs_query_srq { __u64 response; __u32 srq_handle; __u32 reserved; __u64 driver_data[0]; }; struct ib_uverbs_query_srq_resp { __u32 max_wr; __u32 max_sge; __u32 srq_limit; __u32 reserved; }; struct ib_uverbs_destroy_srq { __u64 response; __u32 srq_handle; __u32 reserved; }; struct ib_uverbs_destroy_srq_resp { __u32 events_reported; }; struct ib_uverbs_open_xrc_domain { __u64 response; __u32 fd; __u32 oflags; __u64 driver_data[0]; }; struct ib_uverbs_open_xrc_domain_resp { __u32 xrcd_handle; }; struct ib_uverbs_close_xrc_domain { __u64 response; __u32 xrcd_handle; __u32 reserved; __u64 driver_data[0]; }; struct ib_uverbs_create_xrc_rcv_qp { __u64 response; __u64 user_handle; __u32 xrc_domain_handle; __u32 max_send_wr; __u32 max_recv_wr; __u32 max_send_sge; __u32 max_recv_sge; __u32 max_inline_data; __u8 sq_sig_all; __u8 qp_type; __u8 reserved[6]; __u64 driver_data[0]; }; struct ib_uverbs_create_xrc_rcv_qp_resp { __u32 qpn; __u32 reserved; }; struct ib_uverbs_modify_xrc_rcv_qp { __u32 xrc_domain_handle; __u32 qp_num; struct ib_uverbs_qp_dest dest; struct ib_uverbs_qp_dest alt_dest; __u32 attr_mask; __u32 qkey; __u32 rq_psn; __u32 sq_psn; __u32 dest_qp_num; __u32 qp_access_flags; __u16 pkey_index; __u16 alt_pkey_index; __u8 qp_state; __u8 cur_qp_state; __u8 path_mtu; __u8 path_mig_state; __u8 en_sqd_async_notify; __u8 max_rd_atomic; __u8 max_dest_rd_atomic; __u8 min_rnr_timer; __u8 port_num; __u8 timeout; __u8 retry_cnt; __u8 rnr_retry; __u8 alt_port_num; __u8 alt_timeout; __u8 reserved[6]; __u64 driver_data[0]; }; struct ib_uverbs_query_xrc_rcv_qp { __u64 response; __u32 xrc_domain_handle; __u32 qp_num; __u32 attr_mask; __u32 reserved; __u64 driver_data[0]; }; struct ib_uverbs_reg_xrc_rcv_qp { __u32 xrc_domain_handle; __u32 qp_num; __u64 driver_data[0]; }; struct ib_uverbs_unreg_xrc_rcv_qp { __u32 xrc_domain_handle; __u32 qp_num; __u64 driver_data[0]; }; #endif /* IB_USER_VERBS_H */ Index: stable/9/sys/ofed/include/rdma/ib_verbs.h =================================================================== --- stable/9/sys/ofed/include/rdma/ib_verbs.h (revision 273863) +++ stable/9/sys/ofed/include/rdma/ib_verbs.h (revision 273864) @@ -1,2170 +1,2340 @@ /* * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2004 Infinicon Corporation. All rights reserved. * Copyright (c) 2004 Intel Corporation. All rights reserved. * Copyright (c) 2004 Topspin Corporation. All rights reserved. * Copyright (c) 2004 Voltaire Corporation. All rights reserved. * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #if !defined(IB_VERBS_H) #define IB_VERBS_H #include #include #include #include #include #include #include #include +#include -#include #include #include #include +extern struct workqueue_struct *ib_wq; + union ib_gid { u8 raw[16]; struct { __be64 subnet_prefix; __be64 interface_id; } global; }; enum rdma_node_type { /* IB values map to NodeInfo:NodeType. */ RDMA_NODE_IB_CA = 1, RDMA_NODE_IB_SWITCH, RDMA_NODE_IB_ROUTER, RDMA_NODE_RNIC }; enum rdma_transport_type { RDMA_TRANSPORT_IB, RDMA_TRANSPORT_IWARP }; enum rdma_transport_type rdma_node_get_transport(enum rdma_node_type node_type) __attribute_const__; enum rdma_link_layer { IB_LINK_LAYER_UNSPECIFIED, IB_LINK_LAYER_INFINIBAND, IB_LINK_LAYER_ETHERNET, }; enum ib_device_cap_flags { IB_DEVICE_RESIZE_MAX_WR = 1, IB_DEVICE_BAD_PKEY_CNTR = (1<<1), IB_DEVICE_BAD_QKEY_CNTR = (1<<2), IB_DEVICE_RAW_MULTI = (1<<3), IB_DEVICE_AUTO_PATH_MIG = (1<<4), IB_DEVICE_CHANGE_PHY_PORT = (1<<5), IB_DEVICE_UD_AV_PORT_ENFORCE = (1<<6), IB_DEVICE_CURR_QP_STATE_MOD = (1<<7), IB_DEVICE_SHUTDOWN_PORT = (1<<8), IB_DEVICE_INIT_TYPE = (1<<9), IB_DEVICE_PORT_ACTIVE_EVENT = (1<<10), IB_DEVICE_SYS_IMAGE_GUID = (1<<11), IB_DEVICE_RC_RNR_NAK_GEN = (1<<12), IB_DEVICE_SRQ_RESIZE = (1<<13), IB_DEVICE_N_NOTIFY_CQ = (1<<14), IB_DEVICE_LOCAL_DMA_LKEY = (1<<15), IB_DEVICE_RESERVED = (1<<16), /* old SEND_W_INV */ IB_DEVICE_MEM_WINDOW = (1<<17), /* * Devices should set IB_DEVICE_UD_IP_SUM if they support * insertion of UDP and TCP checksum on outgoing UD IPoIB * messages and can verify the validity of checksum for * incoming messages. Setting this flag implies that the * IPoIB driver may set NETIF_F_IP_CSUM for datagram mode. */ IB_DEVICE_UD_IP_CSUM = (1<<18), IB_DEVICE_UD_TSO = (1<<19), IB_DEVICE_XRC = (1<<20), IB_DEVICE_MEM_MGT_EXTENSIONS = (1<<21), IB_DEVICE_BLOCK_MULTICAST_LOOPBACK = (1<<22), + IB_DEVICE_MR_ALLOCATE = (1<<23), + IB_DEVICE_SHARED_MR = (1<<24), + IB_DEVICE_QPG = (1<<25), + IB_DEVICE_UD_RSS = (1<<26), + IB_DEVICE_UD_TSS = (1<<27) }; enum ib_atomic_cap { IB_ATOMIC_NONE, IB_ATOMIC_HCA, IB_ATOMIC_GLOB }; struct ib_device_attr { u64 fw_ver; __be64 sys_image_guid; u64 max_mr_size; u64 page_size_cap; u32 vendor_id; u32 vendor_part_id; u32 hw_ver; int max_qp; int max_qp_wr; int device_cap_flags; int max_sge; int max_sge_rd; int max_cq; int max_cqe; int max_mr; int max_pd; int max_qp_rd_atom; int max_ee_rd_atom; int max_res_rd_atom; int max_qp_init_rd_atom; int max_ee_init_rd_atom; enum ib_atomic_cap atomic_cap; enum ib_atomic_cap masked_atomic_cap; int max_ee; int max_rdd; int max_mw; int max_raw_ipv6_qp; int max_raw_ethy_qp; int max_mcast_grp; int max_mcast_qp_attach; int max_total_mcast_qp_attach; int max_ah; int max_fmr; int max_map_per_fmr; int max_srq; int max_srq_wr; int max_srq_sge; unsigned int max_fast_reg_page_list_len; + int max_rss_tbl_sz; u16 max_pkeys; u8 local_ca_ack_delay; }; enum ib_mtu { IB_MTU_256 = 1, IB_MTU_512 = 2, IB_MTU_1024 = 3, IB_MTU_2048 = 4, IB_MTU_4096 = 5 }; static inline int ib_mtu_enum_to_int(enum ib_mtu mtu) { switch (mtu) { case IB_MTU_256: return 256; case IB_MTU_512: return 512; case IB_MTU_1024: return 1024; case IB_MTU_2048: return 2048; case IB_MTU_4096: return 4096; default: return -1; } } enum ib_port_state { IB_PORT_NOP = 0, IB_PORT_DOWN = 1, IB_PORT_INIT = 2, IB_PORT_ARMED = 3, IB_PORT_ACTIVE = 4, IB_PORT_ACTIVE_DEFER = 5 }; enum ib_port_cap_flags { IB_PORT_SM = 1 << 1, IB_PORT_NOTICE_SUP = 1 << 2, IB_PORT_TRAP_SUP = 1 << 3, IB_PORT_OPT_IPD_SUP = 1 << 4, IB_PORT_AUTO_MIGR_SUP = 1 << 5, IB_PORT_SL_MAP_SUP = 1 << 6, IB_PORT_MKEY_NVRAM = 1 << 7, IB_PORT_PKEY_NVRAM = 1 << 8, IB_PORT_LED_INFO_SUP = 1 << 9, IB_PORT_SM_DISABLED = 1 << 10, IB_PORT_SYS_IMAGE_GUID_SUP = 1 << 11, IB_PORT_PKEY_SW_EXT_PORT_TRAP_SUP = 1 << 12, + IB_PORT_EXTENDED_SPEEDS_SUP = 1 << 14, IB_PORT_CM_SUP = 1 << 16, IB_PORT_SNMP_TUNNEL_SUP = 1 << 17, IB_PORT_REINIT_SUP = 1 << 18, IB_PORT_DEVICE_MGMT_SUP = 1 << 19, IB_PORT_VENDOR_CLASS_SUP = 1 << 20, IB_PORT_DR_NOTICE_SUP = 1 << 21, IB_PORT_CAP_MASK_NOTICE_SUP = 1 << 22, IB_PORT_BOOT_MGMT_SUP = 1 << 23, IB_PORT_LINK_LATENCY_SUP = 1 << 24, IB_PORT_CLIENT_REG_SUP = 1 << 25 }; enum ib_port_width { IB_WIDTH_1X = 1, IB_WIDTH_4X = 2, IB_WIDTH_8X = 4, IB_WIDTH_12X = 8 }; static inline int ib_width_enum_to_int(enum ib_port_width width) { switch (width) { case IB_WIDTH_1X: return 1; case IB_WIDTH_4X: return 4; case IB_WIDTH_8X: return 8; case IB_WIDTH_12X: return 12; default: return -1; } } +enum ib_port_speed { + IB_SPEED_SDR = 1, + IB_SPEED_DDR = 2, + IB_SPEED_QDR = 4, + IB_SPEED_FDR10 = 8, + IB_SPEED_FDR = 16, + IB_SPEED_EDR = 32 +}; + struct ib_protocol_stats { /* TBD... */ }; struct iw_protocol_stats { u64 ipInReceives; u64 ipInHdrErrors; u64 ipInTooBigErrors; u64 ipInNoRoutes; u64 ipInAddrErrors; u64 ipInUnknownProtos; u64 ipInTruncatedPkts; u64 ipInDiscards; u64 ipInDelivers; u64 ipOutForwDatagrams; u64 ipOutRequests; u64 ipOutDiscards; u64 ipOutNoRoutes; u64 ipReasmTimeout; u64 ipReasmReqds; u64 ipReasmOKs; u64 ipReasmFails; u64 ipFragOKs; u64 ipFragFails; u64 ipFragCreates; u64 ipInMcastPkts; u64 ipOutMcastPkts; u64 ipInBcastPkts; u64 ipOutBcastPkts; u64 tcpRtoAlgorithm; u64 tcpRtoMin; u64 tcpRtoMax; u64 tcpMaxConn; u64 tcpActiveOpens; u64 tcpPassiveOpens; u64 tcpAttemptFails; u64 tcpEstabResets; u64 tcpCurrEstab; u64 tcpInSegs; u64 tcpOutSegs; u64 tcpRetransSegs; u64 tcpInErrs; u64 tcpOutRsts; }; union rdma_protocol_stats { struct ib_protocol_stats ib; struct iw_protocol_stats iw; }; struct ib_port_attr { enum ib_port_state state; enum ib_mtu max_mtu; enum ib_mtu active_mtu; int gid_tbl_len; u32 port_cap_flags; u32 max_msg_sz; u32 bad_pkey_cntr; u32 qkey_viol_cntr; u16 pkey_tbl_len; u16 lid; u16 sm_lid; u8 lmc; u8 max_vl_num; u8 sm_sl; u8 subnet_timeout; u8 init_type_reply; u8 active_width; u8 active_speed; u8 phys_state; enum rdma_link_layer link_layer; }; enum ib_device_modify_flags { IB_DEVICE_MODIFY_SYS_IMAGE_GUID = 1 << 0, IB_DEVICE_MODIFY_NODE_DESC = 1 << 1 }; struct ib_device_modify { u64 sys_image_guid; char node_desc[64]; }; enum ib_port_modify_flags { IB_PORT_SHUTDOWN = 1, IB_PORT_INIT_TYPE = (1<<2), IB_PORT_RESET_QKEY_CNTR = (1<<3) }; struct ib_port_modify { u32 set_port_cap_mask; u32 clr_port_cap_mask; u8 init_type; }; enum ib_event_type { IB_EVENT_CQ_ERR, IB_EVENT_QP_FATAL, IB_EVENT_QP_REQ_ERR, IB_EVENT_QP_ACCESS_ERR, IB_EVENT_COMM_EST, IB_EVENT_SQ_DRAINED, IB_EVENT_PATH_MIG, IB_EVENT_PATH_MIG_ERR, IB_EVENT_DEVICE_FATAL, IB_EVENT_PORT_ACTIVE, IB_EVENT_PORT_ERR, IB_EVENT_LID_CHANGE, IB_EVENT_PKEY_CHANGE, IB_EVENT_SM_CHANGE, IB_EVENT_SRQ_ERR, IB_EVENT_SRQ_LIMIT_REACHED, IB_EVENT_QP_LAST_WQE_REACHED, IB_EVENT_CLIENT_REREGISTER, IB_EVENT_GID_CHANGE, }; enum ib_event_flags { IB_XRC_QP_EVENT_FLAG = 0x80000000, }; struct ib_event { struct ib_device *device; union { struct ib_cq *cq; struct ib_qp *qp; struct ib_srq *srq; u8 port_num; u32 xrc_qp_num; } element; enum ib_event_type event; }; struct ib_event_handler { struct ib_device *device; void (*handler)(struct ib_event_handler *, struct ib_event *); struct list_head list; }; #define INIT_IB_EVENT_HANDLER(_ptr, _device, _handler) \ do { \ (_ptr)->device = _device; \ (_ptr)->handler = _handler; \ INIT_LIST_HEAD(&(_ptr)->list); \ } while (0) struct ib_global_route { union ib_gid dgid; u32 flow_label; u8 sgid_index; u8 hop_limit; u8 traffic_class; }; struct ib_grh { __be32 version_tclass_flow; __be16 paylen; u8 next_hdr; u8 hop_limit; union ib_gid sgid; union ib_gid dgid; }; enum { IB_MULTICAST_QPN = 0xffffff }; #define IB_LID_PERMISSIVE cpu_to_be16(0xFFFF) enum ib_ah_flags { IB_AH_GRH = 1 }; enum ib_rate { IB_RATE_PORT_CURRENT = 0, IB_RATE_2_5_GBPS = 2, IB_RATE_5_GBPS = 5, IB_RATE_10_GBPS = 3, IB_RATE_20_GBPS = 6, IB_RATE_30_GBPS = 4, IB_RATE_40_GBPS = 7, IB_RATE_60_GBPS = 8, IB_RATE_80_GBPS = 9, - IB_RATE_120_GBPS = 10 + IB_RATE_120_GBPS = 10, + IB_RATE_14_GBPS = 11, + IB_RATE_56_GBPS = 12, + IB_RATE_112_GBPS = 13, + IB_RATE_168_GBPS = 14, + IB_RATE_25_GBPS = 15, + IB_RATE_100_GBPS = 16, + IB_RATE_200_GBPS = 17, + IB_RATE_300_GBPS = 18 }; /** * ib_rate_to_mult - Convert the IB rate enum to a multiple of the * base rate of 2.5 Gbit/sec. For example, IB_RATE_5_GBPS will be * converted to 2, since 5 Gbit/sec is 2 * 2.5 Gbit/sec. * @rate: rate to convert. */ int ib_rate_to_mult(enum ib_rate rate) __attribute_const__; /** + * ib_rate_to_mbps - Convert the IB rate enum to Mbps. + * For example, IB_RATE_2_5_GBPS will be converted to 2500. + * @rate: rate to convert. + */ +int ib_rate_to_mbps(enum ib_rate rate) __attribute_const__; + +/** * mult_to_ib_rate - Convert a multiple of 2.5 Gbit/sec to an IB rate * enum. * @mult: multiple to convert. */ enum ib_rate mult_to_ib_rate(int mult) __attribute_const__; struct ib_ah_attr { struct ib_global_route grh; u16 dlid; u8 sl; u8 src_path_bits; u8 static_rate; u8 ah_flags; u8 port_num; }; enum ib_wc_status { IB_WC_SUCCESS, IB_WC_LOC_LEN_ERR, IB_WC_LOC_QP_OP_ERR, IB_WC_LOC_EEC_OP_ERR, IB_WC_LOC_PROT_ERR, IB_WC_WR_FLUSH_ERR, IB_WC_MW_BIND_ERR, IB_WC_BAD_RESP_ERR, IB_WC_LOC_ACCESS_ERR, IB_WC_REM_INV_REQ_ERR, IB_WC_REM_ACCESS_ERR, IB_WC_REM_OP_ERR, IB_WC_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR, IB_WC_LOC_RDD_VIOL_ERR, IB_WC_REM_INV_RD_REQ_ERR, IB_WC_REM_ABORT_ERR, IB_WC_INV_EECN_ERR, IB_WC_INV_EEC_STATE_ERR, IB_WC_FATAL_ERR, IB_WC_RESP_TIMEOUT_ERR, IB_WC_GENERAL_ERR }; enum ib_wc_opcode { IB_WC_SEND, IB_WC_RDMA_WRITE, IB_WC_RDMA_READ, IB_WC_COMP_SWAP, IB_WC_FETCH_ADD, IB_WC_BIND_MW, IB_WC_LSO, IB_WC_LOCAL_INV, IB_WC_FAST_REG_MR, IB_WC_MASKED_COMP_SWAP, IB_WC_MASKED_FETCH_ADD, /* * Set value of IB_WC_RECV so consumers can test if a completion is a * receive by testing (opcode & IB_WC_RECV). */ IB_WC_RECV = 1 << 7, IB_WC_RECV_RDMA_WITH_IMM }; enum ib_wc_flags { IB_WC_GRH = 1, IB_WC_WITH_IMM = (1<<1), IB_WC_WITH_INVALIDATE = (1<<2), + IB_WC_IP_CSUM_OK = (1<<3), }; struct ib_wc { u64 wr_id; enum ib_wc_status status; enum ib_wc_opcode opcode; u32 vendor_err; u32 byte_len; struct ib_qp *qp; union { __be32 imm_data; u32 invalidate_rkey; } ex; u32 src_qp; int wc_flags; u16 pkey_index; u16 slid; u8 sl; u8 dlid_path_bits; u8 port_num; /* valid only for DR SMPs on switches */ int csum_ok; }; enum ib_cq_notify_flags { IB_CQ_SOLICITED = 1 << 0, IB_CQ_NEXT_COMP = 1 << 1, IB_CQ_SOLICITED_MASK = IB_CQ_SOLICITED | IB_CQ_NEXT_COMP, IB_CQ_REPORT_MISSED_EVENTS = 1 << 2, }; +enum ib_srq_type { + IB_SRQT_BASIC, + IB_SRQT_XRC +}; + enum ib_srq_attr_mask { IB_SRQ_MAX_WR = 1 << 0, IB_SRQ_LIMIT = 1 << 1, }; struct ib_srq_attr { u32 max_wr; u32 max_sge; u32 srq_limit; }; struct ib_srq_init_attr { void (*event_handler)(struct ib_event *, void *); void *srq_context; struct ib_srq_attr attr; + enum ib_srq_type srq_type; + + union { + struct { + struct ib_xrcd *xrcd; + struct ib_cq *cq; + } xrc; + } ext; }; struct ib_qp_cap { u32 max_send_wr; u32 max_recv_wr; u32 max_send_sge; u32 max_recv_sge; u32 max_inline_data; + u32 qpg_tss_mask_sz; }; enum ib_sig_type { IB_SIGNAL_ALL_WR, IB_SIGNAL_REQ_WR }; enum ib_qp_type { /* * IB_QPT_SMI and IB_QPT_GSI have to be the first two entries * here (and in that order) since the MAD layer uses them as * indices into a 2-entry table. */ IB_QPT_SMI, IB_QPT_GSI, IB_QPT_RC, IB_QPT_UC, IB_QPT_UD, IB_QPT_XRC, IB_QPT_RAW_IPV6, - IB_QPT_RAW_ETY, - IB_QPT_RAW_ETH + IB_QPT_RAW_ETHERTYPE, + IB_QPT_RAW_PACKET = 8, + IB_QPT_XRC_INI = 9, + IB_QPT_XRC_TGT, + IB_QPT_MAX, }; enum ib_qp_create_flags { IB_QP_CREATE_IPOIB_UD_LSO = 1 << 0, IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK = 1 << 1, + IB_QP_CREATE_NETIF_QP = 1 << 2, + /* reserve bits 26-31 for low level drivers' internal use */ + IB_QP_CREATE_RESERVED_START = 1 << 26, + IB_QP_CREATE_RESERVED_END = 1 << 31, }; +enum ib_qpg_type { + IB_QPG_NONE = 0, + IB_QPG_PARENT = (1<<0), + IB_QPG_CHILD_RX = (1<<1), + IB_QPG_CHILD_TX = (1<<2) +}; + +struct ib_qpg_init_attrib { + u32 tss_child_count; + u32 rss_child_count; +}; + struct ib_qp_init_attr { void (*event_handler)(struct ib_event *, void *); void *qp_context; struct ib_cq *send_cq; struct ib_cq *recv_cq; struct ib_srq *srq; + struct ib_xrcd *xrcd; /* XRC TGT QPs only */ struct ib_qp_cap cap; + union { + struct ib_qp *qpg_parent; /* see qpg_type */ + struct ib_qpg_init_attrib parent_attrib; + } pp; enum ib_sig_type sq_sig_type; enum ib_qp_type qp_type; enum ib_qp_create_flags create_flags; - struct ib_xrcd *xrc_domain; /* XRC qp's only */ + enum ib_qpg_type qpg_type; u8 port_num; /* special QP types only */ }; +struct ib_qp_open_attr { + void (*event_handler)(struct ib_event *, void *); + void *qp_context; + u32 qp_num; + enum ib_qp_type qp_type; +}; + enum ib_rnr_timeout { IB_RNR_TIMER_655_36 = 0, IB_RNR_TIMER_000_01 = 1, IB_RNR_TIMER_000_02 = 2, IB_RNR_TIMER_000_03 = 3, IB_RNR_TIMER_000_04 = 4, IB_RNR_TIMER_000_06 = 5, IB_RNR_TIMER_000_08 = 6, IB_RNR_TIMER_000_12 = 7, IB_RNR_TIMER_000_16 = 8, IB_RNR_TIMER_000_24 = 9, IB_RNR_TIMER_000_32 = 10, IB_RNR_TIMER_000_48 = 11, IB_RNR_TIMER_000_64 = 12, IB_RNR_TIMER_000_96 = 13, IB_RNR_TIMER_001_28 = 14, IB_RNR_TIMER_001_92 = 15, IB_RNR_TIMER_002_56 = 16, IB_RNR_TIMER_003_84 = 17, IB_RNR_TIMER_005_12 = 18, IB_RNR_TIMER_007_68 = 19, IB_RNR_TIMER_010_24 = 20, IB_RNR_TIMER_015_36 = 21, IB_RNR_TIMER_020_48 = 22, IB_RNR_TIMER_030_72 = 23, IB_RNR_TIMER_040_96 = 24, IB_RNR_TIMER_061_44 = 25, IB_RNR_TIMER_081_92 = 26, IB_RNR_TIMER_122_88 = 27, IB_RNR_TIMER_163_84 = 28, IB_RNR_TIMER_245_76 = 29, IB_RNR_TIMER_327_68 = 30, IB_RNR_TIMER_491_52 = 31 }; enum ib_qp_attr_mask { IB_QP_STATE = 1, IB_QP_CUR_STATE = (1<<1), IB_QP_EN_SQD_ASYNC_NOTIFY = (1<<2), IB_QP_ACCESS_FLAGS = (1<<3), IB_QP_PKEY_INDEX = (1<<4), IB_QP_PORT = (1<<5), IB_QP_QKEY = (1<<6), IB_QP_AV = (1<<7), IB_QP_PATH_MTU = (1<<8), IB_QP_TIMEOUT = (1<<9), IB_QP_RETRY_CNT = (1<<10), IB_QP_RNR_RETRY = (1<<11), IB_QP_RQ_PSN = (1<<12), IB_QP_MAX_QP_RD_ATOMIC = (1<<13), IB_QP_ALT_PATH = (1<<14), IB_QP_MIN_RNR_TIMER = (1<<15), IB_QP_SQ_PSN = (1<<16), IB_QP_MAX_DEST_RD_ATOMIC = (1<<17), IB_QP_PATH_MIG_STATE = (1<<18), IB_QP_CAP = (1<<19), - IB_QP_DEST_QPN = (1<<20) + IB_QP_DEST_QPN = (1<<20), + IB_QP_GROUP_RSS = (1<<21) }; enum ib_qp_state { IB_QPS_RESET, IB_QPS_INIT, IB_QPS_RTR, IB_QPS_RTS, IB_QPS_SQD, IB_QPS_SQE, IB_QPS_ERR }; enum ib_mig_state { IB_MIG_MIGRATED, IB_MIG_REARM, IB_MIG_ARMED }; struct ib_qp_attr { enum ib_qp_state qp_state; enum ib_qp_state cur_qp_state; enum ib_mtu path_mtu; enum ib_mig_state path_mig_state; u32 qkey; u32 rq_psn; u32 sq_psn; u32 dest_qp_num; int qp_access_flags; struct ib_qp_cap cap; struct ib_ah_attr ah_attr; struct ib_ah_attr alt_ah_attr; u16 pkey_index; u16 alt_pkey_index; u8 en_sqd_async_notify; u8 sq_draining; u8 max_rd_atomic; u8 max_dest_rd_atomic; u8 min_rnr_timer; u8 port_num; u8 timeout; u8 retry_cnt; u8 rnr_retry; u8 alt_port_num; u8 alt_timeout; }; enum ib_wr_opcode { IB_WR_RDMA_WRITE, IB_WR_RDMA_WRITE_WITH_IMM, IB_WR_SEND, IB_WR_SEND_WITH_IMM, IB_WR_RDMA_READ, IB_WR_ATOMIC_CMP_AND_SWP, IB_WR_ATOMIC_FETCH_AND_ADD, IB_WR_LSO, IB_WR_BIG_LSO, IB_WR_SEND_WITH_INV, IB_WR_RDMA_READ_WITH_INV, IB_WR_LOCAL_INV, IB_WR_FAST_REG_MR, IB_WR_MASKED_ATOMIC_CMP_AND_SWP, IB_WR_MASKED_ATOMIC_FETCH_AND_ADD, }; enum ib_send_flags { IB_SEND_FENCE = 1, IB_SEND_SIGNALED = (1<<1), IB_SEND_SOLICITED = (1<<2), IB_SEND_INLINE = (1<<3), IB_SEND_IP_CSUM = (1<<4) }; +enum ib_flow_types { + IB_FLOW_ETH = 0, + IB_FLOW_IB_UC = 1, + IB_FLOW_IB_MC_IPV4 = 2, + IB_FLOW_IB_MC_IPV6 = 3 +}; + +enum { + IB_FLOW_L4_NONE = 0, + IB_FLOW_L4_OTHER = 3, + IB_FLOW_L4_UDP = 5, + IB_FLOW_L4_TCP = 6 +}; + struct ib_sge { u64 addr; u32 length; u32 lkey; }; struct ib_fast_reg_page_list { struct ib_device *device; u64 *page_list; unsigned int max_page_list_len; }; struct ib_send_wr { struct ib_send_wr *next; u64 wr_id; struct ib_sge *sg_list; int num_sge; enum ib_wr_opcode opcode; int send_flags; union { __be32 imm_data; u32 invalidate_rkey; } ex; union { struct { u64 remote_addr; u32 rkey; } rdma; struct { u64 remote_addr; u64 compare_add; u64 swap; u64 compare_add_mask; u64 swap_mask; u32 rkey; } atomic; struct { struct ib_ah *ah; void *header; int hlen; int mss; u32 remote_qpn; u32 remote_qkey; u16 pkey_index; /* valid for GSI only */ u8 port_num; /* valid for DR SMPs on switch only */ } ud; struct { u64 iova_start; struct ib_fast_reg_page_list *page_list; unsigned int page_shift; unsigned int page_list_len; u32 length; int access_flags; u32 rkey; } fast_reg; struct { struct ib_unpacked_lrh *lrh; u32 eth_type; u8 static_rate; } raw_ety; } wr; - u32 xrc_remote_srq_num; /* valid for XRC sends only */ + u32 xrc_remote_srq_num; /* XRC TGT QPs only */ }; struct ib_recv_wr { struct ib_recv_wr *next; u64 wr_id; struct ib_sge *sg_list; int num_sge; }; enum ib_access_flags { IB_ACCESS_LOCAL_WRITE = 1, IB_ACCESS_REMOTE_WRITE = (1<<1), IB_ACCESS_REMOTE_READ = (1<<2), IB_ACCESS_REMOTE_ATOMIC = (1<<3), - IB_ACCESS_MW_BIND = (1<<4) + IB_ACCESS_MW_BIND = (1<<4), + IB_ACCESS_ALLOCATE_MR = (1<<5), + IB_ACCESS_SHARED_MR_USER_READ = (1<<6), + IB_ACCESS_SHARED_MR_USER_WRITE = (1<<7), + IB_ACCESS_SHARED_MR_GROUP_READ = (1<<8), + IB_ACCESS_SHARED_MR_GROUP_WRITE = (1<<9), + IB_ACCESS_SHARED_MR_OTHER_READ = (1<<10), + IB_ACCESS_SHARED_MR_OTHER_WRITE = (1<<11) + }; struct ib_phys_buf { u64 addr; u64 size; }; struct ib_mr_attr { struct ib_pd *pd; u64 device_virt_addr; u64 size; int mr_access_flags; u32 lkey; u32 rkey; }; enum ib_mr_rereg_flags { IB_MR_REREG_TRANS = 1, IB_MR_REREG_PD = (1<<1), IB_MR_REREG_ACCESS = (1<<2) }; struct ib_mw_bind { struct ib_mr *mr; u64 wr_id; u64 addr; u32 length; int send_flags; int mw_access_flags; }; struct ib_fmr_attr { int max_pages; int max_maps; u8 page_shift; }; struct ib_ucontext { struct ib_device *device; struct list_head pd_list; struct list_head mr_list; struct list_head mw_list; struct list_head cq_list; struct list_head qp_list; struct list_head srq_list; struct list_head ah_list; - struct list_head xrc_domain_list; + struct list_head xrcd_list; int closing; }; struct ib_uobject { u64 user_handle; /* handle given to us by userspace */ struct ib_ucontext *context; /* associated user context */ void *object; /* containing object */ struct list_head list; /* link to context's list */ int id; /* index into kernel idr */ struct kref ref; struct rw_semaphore mutex; /* protects .live */ int live; }; struct ib_udata { void __user *inbuf; void __user *outbuf; size_t inlen; size_t outlen; }; struct ib_uxrc_rcv_object { struct list_head list; /* link to context's list */ u32 qp_num; u32 domain_handle; }; struct ib_pd { struct ib_device *device; struct ib_uobject *uobject; atomic_t usecnt; /* count all resources */ }; struct ib_xrcd { struct ib_device *device; struct ib_uobject *uobject; + atomic_t usecnt; /* count all exposed resources */ struct inode *inode; struct rb_node node; - atomic_t usecnt; /* count all resources */ + + struct mutex tgt_qp_mutex; + struct list_head tgt_qp_list; }; - struct ib_ah { struct ib_device *device; struct ib_pd *pd; struct ib_uobject *uobject; }; typedef void (*ib_comp_handler)(struct ib_cq *cq, void *cq_context); struct ib_cq { struct ib_device *device; struct ib_uobject *uobject; ib_comp_handler comp_handler; void (*event_handler)(struct ib_event *, void *); void *cq_context; int cqe; atomic_t usecnt; /* count number of work queues */ }; struct ib_srq { struct ib_device *device; struct ib_pd *pd; - struct ib_cq *xrc_cq; - struct ib_xrcd *xrcd; struct ib_uobject *uobject; void (*event_handler)(struct ib_event *, void *); void *srq_context; + enum ib_srq_type srq_type; atomic_t usecnt; - u32 xrc_srq_num; + + union { + struct { + struct ib_xrcd *xrcd; + struct ib_cq *cq; + u32 srq_num; + } xrc; + } ext; }; struct ib_qp { struct ib_device *device; struct ib_pd *pd; struct ib_cq *send_cq; struct ib_cq *recv_cq; struct ib_srq *srq; + struct ib_xrcd *xrcd; /* XRC TGT QPs only */ + struct list_head xrcd_list; + atomic_t usecnt; /* count times opened, mcast attaches */ + struct list_head open_list; + struct ib_qp *real_qp; struct ib_uobject *uobject; void (*event_handler)(struct ib_event *, void *); void *qp_context; u32 qp_num; enum ib_qp_type qp_type; - struct ib_xrcd *xrcd; /* XRC QPs only */ + enum ib_qpg_type qpg_type; }; struct ib_mr { struct ib_device *device; struct ib_pd *pd; struct ib_uobject *uobject; u32 lkey; u32 rkey; atomic_t usecnt; /* count number of MWs */ }; struct ib_mw { struct ib_device *device; struct ib_pd *pd; struct ib_uobject *uobject; u32 rkey; }; struct ib_fmr { struct ib_device *device; struct ib_pd *pd; struct list_head list; u32 lkey; u32 rkey; }; +struct ib_flow_spec { + enum ib_flow_types type; + union { + struct { + __be16 ethertype; + __be16 vlan; + u8 vlan_present; + u8 mac[6]; + u8 port; + } eth; + struct { + __be32 qpn; + } ib_uc; + struct { + u8 mgid[16]; + } ib_mc; + } l2_id; + __be32 src_ip; + __be32 dst_ip; + __be16 src_port; + __be16 dst_port; + u8 l4_protocol; + u8 block_mc_loopback; + u8 rule_type; +}; + struct ib_mad; struct ib_grh; enum ib_process_mad_flags { IB_MAD_IGNORE_MKEY = 1, IB_MAD_IGNORE_BKEY = 2, IB_MAD_IGNORE_ALL = IB_MAD_IGNORE_MKEY | IB_MAD_IGNORE_BKEY }; enum ib_mad_result { IB_MAD_RESULT_FAILURE = 0, /* (!SUCCESS is the important flag) */ IB_MAD_RESULT_SUCCESS = 1 << 0, /* MAD was successfully processed */ IB_MAD_RESULT_REPLY = 1 << 1, /* Reply packet needs to be sent */ IB_MAD_RESULT_CONSUMED = 1 << 2 /* Packet consumed: stop processing */ }; #define IB_DEVICE_NAME_MAX 64 struct ib_cache { rwlock_t lock; struct ib_event_handler event_handler; struct ib_pkey_cache **pkey_cache; struct ib_gid_cache **gid_cache; u8 *lmc_cache; }; struct ib_dma_mapping_ops { int (*mapping_error)(struct ib_device *dev, u64 dma_addr); u64 (*map_single)(struct ib_device *dev, void *ptr, size_t size, enum dma_data_direction direction); void (*unmap_single)(struct ib_device *dev, u64 addr, size_t size, enum dma_data_direction direction); u64 (*map_page)(struct ib_device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction); void (*unmap_page)(struct ib_device *dev, u64 addr, size_t size, enum dma_data_direction direction); int (*map_sg)(struct ib_device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction); void (*unmap_sg)(struct ib_device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction); u64 (*dma_address)(struct ib_device *dev, struct scatterlist *sg); unsigned int (*dma_len)(struct ib_device *dev, struct scatterlist *sg); void (*sync_single_for_cpu)(struct ib_device *dev, u64 dma_handle, size_t size, enum dma_data_direction dir); void (*sync_single_for_device)(struct ib_device *dev, u64 dma_handle, size_t size, enum dma_data_direction dir); void *(*alloc_coherent)(struct ib_device *dev, size_t size, u64 *dma_handle, gfp_t flag); void (*free_coherent)(struct ib_device *dev, size_t size, void *cpu_addr, u64 dma_handle); }; struct iw_cm_verbs; struct ib_device { struct device *dma_device; char name[IB_DEVICE_NAME_MAX]; struct list_head event_handler_list; spinlock_t event_handler_lock; + spinlock_t client_data_lock; struct list_head core_list; struct list_head client_data_list; - spinlock_t client_data_lock; struct ib_cache cache; int *pkey_tbl_len; int *gid_tbl_len; int num_comp_vectors; struct iw_cm_verbs *iwcm; int (*get_protocol_stats)(struct ib_device *device, union rdma_protocol_stats *stats); int (*query_device)(struct ib_device *device, struct ib_device_attr *device_attr); int (*query_port)(struct ib_device *device, u8 port_num, struct ib_port_attr *port_attr); enum rdma_link_layer (*get_link_layer)(struct ib_device *device, u8 port_num); int (*query_gid)(struct ib_device *device, u8 port_num, int index, union ib_gid *gid); int (*query_pkey)(struct ib_device *device, u8 port_num, u16 index, u16 *pkey); int (*modify_device)(struct ib_device *device, int device_modify_mask, struct ib_device_modify *device_modify); int (*modify_port)(struct ib_device *device, u8 port_num, int port_modify_mask, struct ib_port_modify *port_modify); struct ib_ucontext * (*alloc_ucontext)(struct ib_device *device, struct ib_udata *udata); int (*dealloc_ucontext)(struct ib_ucontext *context); int (*mmap)(struct ib_ucontext *context, struct vm_area_struct *vma); struct ib_pd * (*alloc_pd)(struct ib_device *device, struct ib_ucontext *context, struct ib_udata *udata); int (*dealloc_pd)(struct ib_pd *pd); struct ib_ah * (*create_ah)(struct ib_pd *pd, struct ib_ah_attr *ah_attr); int (*modify_ah)(struct ib_ah *ah, struct ib_ah_attr *ah_attr); int (*query_ah)(struct ib_ah *ah, struct ib_ah_attr *ah_attr); int (*destroy_ah)(struct ib_ah *ah); struct ib_srq * (*create_srq)(struct ib_pd *pd, struct ib_srq_init_attr *srq_init_attr, struct ib_udata *udata); int (*modify_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr, enum ib_srq_attr_mask srq_attr_mask, struct ib_udata *udata); int (*query_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr); int (*destroy_srq)(struct ib_srq *srq); int (*post_srq_recv)(struct ib_srq *srq, struct ib_recv_wr *recv_wr, struct ib_recv_wr **bad_recv_wr); struct ib_qp * (*create_qp)(struct ib_pd *pd, struct ib_qp_init_attr *qp_init_attr, struct ib_udata *udata); int (*modify_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_udata *udata); int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); int (*destroy_qp)(struct ib_qp *qp); int (*post_send)(struct ib_qp *qp, struct ib_send_wr *send_wr, struct ib_send_wr **bad_send_wr); int (*post_recv)(struct ib_qp *qp, struct ib_recv_wr *recv_wr, struct ib_recv_wr **bad_recv_wr); struct ib_cq * (*create_cq)(struct ib_device *device, int cqe, int comp_vector, struct ib_ucontext *context, struct ib_udata *udata); int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period); int (*destroy_cq)(struct ib_cq *cq); int (*resize_cq)(struct ib_cq *cq, int cqe, struct ib_udata *udata); int (*poll_cq)(struct ib_cq *cq, int num_entries, struct ib_wc *wc); int (*peek_cq)(struct ib_cq *cq, int wc_cnt); int (*req_notify_cq)(struct ib_cq *cq, enum ib_cq_notify_flags flags); int (*req_ncomp_notif)(struct ib_cq *cq, int wc_cnt); struct ib_mr * (*get_dma_mr)(struct ib_pd *pd, int mr_access_flags); struct ib_mr * (*reg_phys_mr)(struct ib_pd *pd, struct ib_phys_buf *phys_buf_array, int num_phys_buf, int mr_access_flags, u64 *iova_start); struct ib_mr * (*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int mr_access_flags, - struct ib_udata *udata); + struct ib_udata *udata, + int mr_id); int (*query_mr)(struct ib_mr *mr, struct ib_mr_attr *mr_attr); int (*dereg_mr)(struct ib_mr *mr); struct ib_mr * (*alloc_fast_reg_mr)(struct ib_pd *pd, int max_page_list_len); struct ib_fast_reg_page_list * (*alloc_fast_reg_page_list)(struct ib_device *device, int page_list_len); void (*free_fast_reg_page_list)(struct ib_fast_reg_page_list *page_list); int (*rereg_phys_mr)(struct ib_mr *mr, int mr_rereg_mask, struct ib_pd *pd, struct ib_phys_buf *phys_buf_array, int num_phys_buf, int mr_access_flags, u64 *iova_start); struct ib_mw * (*alloc_mw)(struct ib_pd *pd); int (*bind_mw)(struct ib_qp *qp, struct ib_mw *mw, struct ib_mw_bind *mw_bind); int (*dealloc_mw)(struct ib_mw *mw); struct ib_fmr * (*alloc_fmr)(struct ib_pd *pd, int mr_access_flags, struct ib_fmr_attr *fmr_attr); int (*map_phys_fmr)(struct ib_fmr *fmr, u64 *page_list, int list_len, u64 iova); int (*unmap_fmr)(struct list_head *fmr_list); int (*dealloc_fmr)(struct ib_fmr *fmr); int (*attach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid); int (*detach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid); int (*process_mad)(struct ib_device *device, int process_mad_flags, u8 port_num, struct ib_wc *in_wc, struct ib_grh *in_grh, struct ib_mad *in_mad, struct ib_mad *out_mad); struct ib_srq * (*create_xrc_srq)(struct ib_pd *pd, struct ib_cq *xrc_cq, struct ib_xrcd *xrcd, struct ib_srq_init_attr *srq_init_attr, struct ib_udata *udata); struct ib_xrcd * (*alloc_xrcd)(struct ib_device *device, - struct ib_ucontext *context, + struct ib_ucontext *ucontext, struct ib_udata *udata); int (*dealloc_xrcd)(struct ib_xrcd *xrcd); int (*create_xrc_rcv_qp)(struct ib_qp_init_attr *init_attr, u32 *qp_num); int (*modify_xrc_rcv_qp)(struct ib_xrcd *xrcd, u32 qp_num, struct ib_qp_attr *attr, int attr_mask); int (*query_xrc_rcv_qp)(struct ib_xrcd *xrcd, u32 qp_num, struct ib_qp_attr *attr, int attr_mask, struct ib_qp_init_attr *init_attr); int (*reg_xrc_rcv_qp)(struct ib_xrcd *xrcd, void *context, u32 qp_num); int (*unreg_xrc_rcv_qp)(struct ib_xrcd *xrcd, void *context, u32 qp_num); + int (*attach_flow)(struct ib_qp *qp, + struct ib_flow_spec *spec, + int priority); + int (*detach_flow)(struct ib_qp *qp, + struct ib_flow_spec *spec, + int priority); + unsigned long (*get_unmapped_area)(struct file *file, + unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags); struct ib_dma_mapping_ops *dma_ops; struct module *owner; struct device dev; struct kobject *ports_parent; struct list_head port_list; enum { IB_DEV_UNINITIALIZED, IB_DEV_REGISTERED, IB_DEV_UNREGISTERED } reg_state; - u64 uverbs_cmd_mask; int uverbs_abi_ver; + u64 uverbs_cmd_mask; char node_desc[64]; __be64 node_guid; u32 local_dma_lkey; u8 node_type; u8 phys_port_cnt; struct rb_root ib_uverbs_xrcd_table; struct mutex xrcd_table_mutex; }; struct ib_client { char *name; void (*add) (struct ib_device *); void (*remove)(struct ib_device *); struct list_head list; }; struct ib_device *ib_alloc_device(size_t size); void ib_dealloc_device(struct ib_device *device); -int ib_register_device (struct ib_device *device); +int ib_register_device(struct ib_device *device, + int (*port_callback)(struct ib_device *, + u8, struct kobject *)); void ib_unregister_device(struct ib_device *device); int ib_register_client (struct ib_client *client); void ib_unregister_client(struct ib_client *client); void *ib_get_client_data(struct ib_device *device, struct ib_client *client); void ib_set_client_data(struct ib_device *device, struct ib_client *client, void *data); static inline int ib_copy_from_udata(void *dest, struct ib_udata *udata, size_t len) { return copy_from_user(dest, udata->inbuf, len) ? -EFAULT : 0; } static inline int ib_copy_to_udata(struct ib_udata *udata, void *src, size_t len) { return copy_to_user(udata->outbuf, src, len) ? -EFAULT : 0; } /** - * ib_sysfs_create_port_files - iterate over port sysfs directories - * @device: the IB device - * @create: a function to create sysfs files in each port directory - */ -int ib_sysfs_create_port_files(struct ib_device *device, - int (*create)(struct ib_device *dev, u8 port_num, - struct kobject *kobj)); - -/** * ib_modify_qp_is_ok - Check that the supplied attribute mask * contains all required attributes and no attributes not allowed for * the given QP state transition. * @cur_state: Current QP state * @next_state: Next QP state * @type: QP type * @mask: Mask of supplied QP attributes * * This function is a helper function that a low-level driver's * modify_qp method can use to validate the consumer's input. It * checks that cur_state and next_state are valid QP states, that a * transition from cur_state to next_state is allowed by the IB spec, * and that the attribute mask supplied is allowed for the transition. */ int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, enum ib_qp_type type, enum ib_qp_attr_mask mask); int ib_register_event_handler (struct ib_event_handler *event_handler); int ib_unregister_event_handler(struct ib_event_handler *event_handler); void ib_dispatch_event(struct ib_event *event); int ib_query_device(struct ib_device *device, struct ib_device_attr *device_attr); int ib_query_port(struct ib_device *device, u8 port_num, struct ib_port_attr *port_attr); enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num); int ib_query_gid(struct ib_device *device, u8 port_num, int index, union ib_gid *gid); int ib_query_pkey(struct ib_device *device, u8 port_num, u16 index, u16 *pkey); int ib_modify_device(struct ib_device *device, int device_modify_mask, struct ib_device_modify *device_modify); int ib_modify_port(struct ib_device *device, u8 port_num, int port_modify_mask, struct ib_port_modify *port_modify); int ib_find_gid(struct ib_device *device, union ib_gid *gid, u8 *port_num, u16 *index); int ib_find_pkey(struct ib_device *device, u8 port_num, u16 pkey, u16 *index); /** * ib_alloc_pd - Allocates an unused protection domain. * @device: The device on which to allocate the protection domain. * * A protection domain object provides an association between QPs, shared * receive queues, address handles, memory regions, and memory windows. */ struct ib_pd *ib_alloc_pd(struct ib_device *device); /** * ib_dealloc_pd - Deallocates a protection domain. * @pd: The protection domain to deallocate. */ int ib_dealloc_pd(struct ib_pd *pd); /** * ib_create_ah - Creates an address handle for the given address vector. * @pd: The protection domain associated with the address handle. * @ah_attr: The attributes of the address vector. * * The address handle is used to reference a local or global destination * in all UD QP post sends. */ struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr); /** * ib_init_ah_from_wc - Initializes address handle attributes from a * work completion. * @device: Device on which the received message arrived. * @port_num: Port on which the received message arrived. * @wc: Work completion associated with the received message. * @grh: References the received global route header. This parameter is * ignored unless the work completion indicates that the GRH is valid. * @ah_attr: Returned attributes that can be used when creating an address * handle for replying to the message. */ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc, struct ib_grh *grh, struct ib_ah_attr *ah_attr); /** * ib_create_ah_from_wc - Creates an address handle associated with the * sender of the specified work completion. * @pd: The protection domain associated with the address handle. * @wc: Work completion information associated with a received message. * @grh: References the received global route header. This parameter is * ignored unless the work completion indicates that the GRH is valid. * @port_num: The outbound port number to associate with the address. * * The address handle is used to reference a local or global destination * in all UD QP post sends. */ struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc, struct ib_grh *grh, u8 port_num); /** * ib_modify_ah - Modifies the address vector associated with an address * handle. * @ah: The address handle to modify. * @ah_attr: The new address vector attributes to associate with the * address handle. */ int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr); /** * ib_query_ah - Queries the address vector associated with an address * handle. * @ah: The address handle to query. * @ah_attr: The address vector attributes associated with the address * handle. */ int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr); /** * ib_destroy_ah - Destroys an address handle. * @ah: The address handle to destroy. */ int ib_destroy_ah(struct ib_ah *ah); /** * ib_create_xrc_srq - Creates an XRC SRQ associated with the specified * protection domain, cq, and xrc domain. * @pd: The protection domain associated with the SRQ. * @xrc_cq: The cq to be associated with the XRC SRQ. * @xrcd: The XRC domain to be associated with the XRC SRQ. * @srq_init_attr: A list of initial attributes required to create the * XRC SRQ. If XRC SRQ creation succeeds, then the attributes are updated * to the actual capabilities of the created XRC SRQ. * * srq_attr->max_wr and srq_attr->max_sge are read the determine the * requested size of the XRC SRQ, and set to the actual values allocated * on return. If ib_create_xrc_srq() succeeds, then max_wr and max_sge * will always be at least as large as the requested values. */ struct ib_srq *ib_create_xrc_srq(struct ib_pd *pd, struct ib_cq *xrc_cq, struct ib_xrcd *xrcd, struct ib_srq_init_attr *srq_init_attr); /** - * ib_create_srq - Creates an SRQ associated with the specified - * protection domain. + * ib_create_srq - Creates a SRQ associated with the specified protection + * domain. * @pd: The protection domain associated with the SRQ. * @srq_init_attr: A list of initial attributes required to create the * SRQ. If SRQ creation succeeds, then the attributes are updated to * the actual capabilities of the created SRQ. * * srq_attr->max_wr and srq_attr->max_sge are read the determine the * requested size of the SRQ, and set to the actual values allocated * on return. If ib_create_srq() succeeds, then max_wr and max_sge * will always be at least as large as the requested values. */ struct ib_srq *ib_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *srq_init_attr); /** * ib_modify_srq - Modifies the attributes for the specified SRQ. * @srq: The SRQ to modify. * @srq_attr: On input, specifies the SRQ attributes to modify. On output, * the current values of selected SRQ attributes are returned. * @srq_attr_mask: A bit-mask used to specify which attributes of the SRQ * are being modified. * * The mask may contain IB_SRQ_MAX_WR to resize the SRQ and/or * IB_SRQ_LIMIT to set the SRQ's limit and request notification when * the number of receives queued drops below the limit. */ int ib_modify_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr, enum ib_srq_attr_mask srq_attr_mask); /** * ib_query_srq - Returns the attribute list and current values for the * specified SRQ. * @srq: The SRQ to query. * @srq_attr: The attributes of the specified SRQ. */ int ib_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); /** * ib_destroy_srq - Destroys the specified SRQ. * @srq: The SRQ to destroy. */ int ib_destroy_srq(struct ib_srq *srq); /** * ib_post_srq_recv - Posts a list of work requests to the specified SRQ. * @srq: The SRQ to post the work request on. * @recv_wr: A list of work requests to post on the receive queue. * @bad_recv_wr: On an immediate failure, this parameter will reference * the work request that failed to be posted on the QP. */ static inline int ib_post_srq_recv(struct ib_srq *srq, struct ib_recv_wr *recv_wr, struct ib_recv_wr **bad_recv_wr) { return srq->device->post_srq_recv(srq, recv_wr, bad_recv_wr); } /** * ib_create_qp - Creates a QP associated with the specified protection * domain. * @pd: The protection domain associated with the QP. * @qp_init_attr: A list of initial attributes required to create the * QP. If QP creation succeeds, then the attributes are updated to * the actual capabilities of the created QP. */ struct ib_qp *ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *qp_init_attr); /** * ib_modify_qp - Modifies the attributes for the specified QP and then * transitions the QP to the given state. * @qp: The QP to modify. * @qp_attr: On input, specifies the QP attributes to modify. On output, * the current values of selected QP attributes are returned. * @qp_attr_mask: A bit-mask used to specify which attributes of the QP * are being modified. */ int ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask); /** * ib_query_qp - Returns the attribute list and current values for the * specified QP. * @qp: The QP to query. * @qp_attr: The attributes of the specified QP. * @qp_attr_mask: A bit-mask used to select specific attributes to query. * @qp_init_attr: Additional attributes of the selected QP. * * The qp_attr_mask may be used to limit the query to gathering only the * selected attributes. */ int ib_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); /** * ib_destroy_qp - Destroys the specified QP. * @qp: The QP to destroy. */ int ib_destroy_qp(struct ib_qp *qp); /** + * ib_open_qp - Obtain a reference to an existing sharable QP. + * @xrcd - XRC domain + * @qp_open_attr: Attributes identifying the QP to open. + * + * Returns a reference to a sharable QP. + */ +struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd, + struct ib_qp_open_attr *qp_open_attr); + +/** + * ib_close_qp - Release an external reference to a QP. + * @qp: The QP handle to release + * + * The opened QP handle is released by the caller. The underlying + * shared QP is not destroyed until all internal references are released. + */ +int ib_close_qp(struct ib_qp *qp); + +/** * ib_post_send - Posts a list of work requests to the send queue of * the specified QP. * @qp: The QP to post the work request on. * @send_wr: A list of work requests to post on the send queue. * @bad_send_wr: On an immediate failure, this parameter will reference * the work request that failed to be posted on the QP. + * + * While IBA Vol. 1 section 11.4.1.1 specifies that if an immediate + * error is returned, the QP state shall not be affected, + * ib_post_send() will return an immediate error after queueing any + * earlier work requests in the list. */ static inline int ib_post_send(struct ib_qp *qp, struct ib_send_wr *send_wr, struct ib_send_wr **bad_send_wr) { return qp->device->post_send(qp, send_wr, bad_send_wr); } /** * ib_post_recv - Posts a list of work requests to the receive queue of * the specified QP. * @qp: The QP to post the work request on. * @recv_wr: A list of work requests to post on the receive queue. * @bad_recv_wr: On an immediate failure, this parameter will reference * the work request that failed to be posted on the QP. */ static inline int ib_post_recv(struct ib_qp *qp, struct ib_recv_wr *recv_wr, struct ib_recv_wr **bad_recv_wr) { return qp->device->post_recv(qp, recv_wr, bad_recv_wr); } /* * IB_CQ_VECTOR_LEAST_ATTACHED: The constant specifies that * the CQ will be attached to the completion vector that has * the least number of CQs already attached to it. */ #define IB_CQ_VECTOR_LEAST_ATTACHED 0xffffffff /** * ib_create_cq - Creates a CQ on the specified device. * @device: The device on which to create the CQ. * @comp_handler: A user-specified callback that is invoked when a * completion event occurs on the CQ. * @event_handler: A user-specified callback that is invoked when an * asynchronous event not associated with a completion occurs on the CQ. * @cq_context: Context associated with the CQ returned to the user via * the associated completion and event handlers. * @cqe: The minimum size of the CQ. * @comp_vector - Completion vector used to signal completion events. - * Must be >= 0 and < context->num_comp_vectors - * or IB_CQ_VECTOR_LEAST_ATTACHED. + * Must be >= 0 and < context->num_comp_vectors. * * Users can examine the cq structure to determine the actual CQ size. */ struct ib_cq *ib_create_cq(struct ib_device *device, ib_comp_handler comp_handler, void (*event_handler)(struct ib_event *, void *), void *cq_context, int cqe, int comp_vector); /** * ib_resize_cq - Modifies the capacity of the CQ. * @cq: The CQ to resize. * @cqe: The minimum size of the CQ. * * Users can examine the cq structure to determine the actual CQ size. */ int ib_resize_cq(struct ib_cq *cq, int cqe); /** * ib_modify_cq - Modifies moderation params of the CQ * @cq: The CQ to modify. * @cq_count: number of CQEs that will trigger an event * @cq_period: max period of time in usec before triggering an event * */ int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period); /** * ib_destroy_cq - Destroys the specified CQ. * @cq: The CQ to destroy. */ int ib_destroy_cq(struct ib_cq *cq); /** * ib_poll_cq - poll a CQ for completion(s) * @cq:the CQ being polled * @num_entries:maximum number of completions to return * @wc:array of at least @num_entries &struct ib_wc where completions * will be returned * * Poll a CQ for (possibly multiple) completions. If the return value * is < 0, an error occurred. If the return value is >= 0, it is the * number of completions returned. If the return value is * non-negative and < num_entries, then the CQ was emptied. */ static inline int ib_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc) { return cq->device->poll_cq(cq, num_entries, wc); } /** * ib_peek_cq - Returns the number of unreaped completions currently * on the specified CQ. * @cq: The CQ to peek. * @wc_cnt: A minimum number of unreaped completions to check for. * * If the number of unreaped completions is greater than or equal to wc_cnt, * this function returns wc_cnt, otherwise, it returns the actual number of * unreaped completions. */ int ib_peek_cq(struct ib_cq *cq, int wc_cnt); /** * ib_req_notify_cq - Request completion notification on a CQ. * @cq: The CQ to generate an event for. * @flags: * Must contain exactly one of %IB_CQ_SOLICITED or %IB_CQ_NEXT_COMP * to request an event on the next solicited event or next work * completion at any type, respectively. %IB_CQ_REPORT_MISSED_EVENTS * may also be |ed in to request a hint about missed events, as * described below. * * Return Value: * < 0 means an error occurred while requesting notification * == 0 means notification was requested successfully, and if * IB_CQ_REPORT_MISSED_EVENTS was passed in, then no events * were missed and it is safe to wait for another event. In * this case is it guaranteed that any work completions added * to the CQ since the last CQ poll will trigger a completion * notification event. * > 0 is only returned if IB_CQ_REPORT_MISSED_EVENTS was passed * in. It means that the consumer must poll the CQ again to * make sure it is empty to avoid missing an event because of a * race between requesting notification and an entry being * added to the CQ. This return value means it is possible * (but not guaranteed) that a work completion has been added * to the CQ since the last poll without triggering a * completion notification event. */ static inline int ib_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags) { return cq->device->req_notify_cq(cq, flags); } /** * ib_req_ncomp_notif - Request completion notification when there are * at least the specified number of unreaped completions on the CQ. * @cq: The CQ to generate an event for. * @wc_cnt: The number of unreaped completions that should be on the * CQ before an event is generated. */ static inline int ib_req_ncomp_notif(struct ib_cq *cq, int wc_cnt) { return cq->device->req_ncomp_notif ? cq->device->req_ncomp_notif(cq, wc_cnt) : -ENOSYS; } /** * ib_get_dma_mr - Returns a memory region for system memory that is * usable for DMA. * @pd: The protection domain associated with the memory region. * @mr_access_flags: Specifies the memory access rights. * * Note that the ib_dma_*() functions defined below must be used * to create/destroy addresses used with the Lkey or Rkey returned * by ib_get_dma_mr(). */ struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags); /** * ib_dma_mapping_error - check a DMA addr for error * @dev: The device for which the dma_addr was created * @dma_addr: The DMA address to check */ static inline int ib_dma_mapping_error(struct ib_device *dev, u64 dma_addr) { if (dev->dma_ops) return dev->dma_ops->mapping_error(dev, dma_addr); return dma_mapping_error(dev->dma_device, dma_addr); } /** * ib_dma_map_single - Map a kernel virtual address to DMA address * @dev: The device for which the dma_addr is to be created * @cpu_addr: The kernel virtual address * @size: The size of the region in bytes * @direction: The direction of the DMA */ static inline u64 ib_dma_map_single(struct ib_device *dev, void *cpu_addr, size_t size, enum dma_data_direction direction) { if (dev->dma_ops) return dev->dma_ops->map_single(dev, cpu_addr, size, direction); return dma_map_single(dev->dma_device, cpu_addr, size, direction); } /** * ib_dma_unmap_single - Destroy a mapping created by ib_dma_map_single() * @dev: The device for which the DMA address was created * @addr: The DMA address * @size: The size of the region in bytes * @direction: The direction of the DMA */ static inline void ib_dma_unmap_single(struct ib_device *dev, u64 addr, size_t size, enum dma_data_direction direction) { if (dev->dma_ops) dev->dma_ops->unmap_single(dev, addr, size, direction); else dma_unmap_single(dev->dma_device, addr, size, direction); } static inline u64 ib_dma_map_single_attrs(struct ib_device *dev, void *cpu_addr, size_t size, enum dma_data_direction direction, struct dma_attrs *attrs) { return dma_map_single_attrs(dev->dma_device, cpu_addr, size, direction, attrs); } static inline void ib_dma_unmap_single_attrs(struct ib_device *dev, u64 addr, size_t size, enum dma_data_direction direction, struct dma_attrs *attrs) { return dma_unmap_single_attrs(dev->dma_device, addr, size, direction, attrs); } /** * ib_dma_map_page - Map a physical page to DMA address * @dev: The device for which the dma_addr is to be created * @page: The page to be mapped * @offset: The offset within the page * @size: The size of the region in bytes * @direction: The direction of the DMA */ static inline u64 ib_dma_map_page(struct ib_device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction) { if (dev->dma_ops) return dev->dma_ops->map_page(dev, page, offset, size, direction); return dma_map_page(dev->dma_device, page, offset, size, direction); } /** * ib_dma_unmap_page - Destroy a mapping created by ib_dma_map_page() * @dev: The device for which the DMA address was created * @addr: The DMA address * @size: The size of the region in bytes * @direction: The direction of the DMA */ static inline void ib_dma_unmap_page(struct ib_device *dev, u64 addr, size_t size, enum dma_data_direction direction) { if (dev->dma_ops) dev->dma_ops->unmap_page(dev, addr, size, direction); else dma_unmap_page(dev->dma_device, addr, size, direction); } /** * ib_dma_map_sg - Map a scatter/gather list to DMA addresses * @dev: The device for which the DMA addresses are to be created * @sg: The array of scatter/gather entries * @nents: The number of scatter/gather entries * @direction: The direction of the DMA */ static inline int ib_dma_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction) { if (dev->dma_ops) return dev->dma_ops->map_sg(dev, sg, nents, direction); return dma_map_sg(dev->dma_device, sg, nents, direction); } /** * ib_dma_unmap_sg - Unmap a scatter/gather list of DMA addresses * @dev: The device for which the DMA addresses were created * @sg: The array of scatter/gather entries * @nents: The number of scatter/gather entries * @direction: The direction of the DMA */ static inline void ib_dma_unmap_sg(struct ib_device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction) { if (dev->dma_ops) dev->dma_ops->unmap_sg(dev, sg, nents, direction); else dma_unmap_sg(dev->dma_device, sg, nents, direction); } static inline int ib_dma_map_sg_attrs(struct ib_device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction, struct dma_attrs *attrs) { return dma_map_sg_attrs(dev->dma_device, sg, nents, direction, attrs); } static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction, struct dma_attrs *attrs) { dma_unmap_sg_attrs(dev->dma_device, sg, nents, direction, attrs); } /** * ib_sg_dma_address - Return the DMA address from a scatter/gather entry * @dev: The device for which the DMA addresses were created * @sg: The scatter/gather entry */ static inline u64 ib_sg_dma_address(struct ib_device *dev, struct scatterlist *sg) { if (dev->dma_ops) return dev->dma_ops->dma_address(dev, sg); return sg_dma_address(sg); } /** * ib_sg_dma_len - Return the DMA length from a scatter/gather entry * @dev: The device for which the DMA addresses were created * @sg: The scatter/gather entry */ static inline unsigned int ib_sg_dma_len(struct ib_device *dev, struct scatterlist *sg) { if (dev->dma_ops) return dev->dma_ops->dma_len(dev, sg); return sg_dma_len(sg); } /** * ib_dma_sync_single_for_cpu - Prepare DMA region to be accessed by CPU * @dev: The device for which the DMA address was created * @addr: The DMA address * @size: The size of the region in bytes * @dir: The direction of the DMA */ static inline void ib_dma_sync_single_for_cpu(struct ib_device *dev, u64 addr, size_t size, enum dma_data_direction dir) { if (dev->dma_ops) dev->dma_ops->sync_single_for_cpu(dev, addr, size, dir); else dma_sync_single_for_cpu(dev->dma_device, addr, size, dir); } /** * ib_dma_sync_single_for_device - Prepare DMA region to be accessed by device * @dev: The device for which the DMA address was created * @addr: The DMA address * @size: The size of the region in bytes * @dir: The direction of the DMA */ static inline void ib_dma_sync_single_for_device(struct ib_device *dev, u64 addr, size_t size, enum dma_data_direction dir) { if (dev->dma_ops) dev->dma_ops->sync_single_for_device(dev, addr, size, dir); else dma_sync_single_for_device(dev->dma_device, addr, size, dir); } /** * ib_dma_alloc_coherent - Allocate memory and map it for DMA * @dev: The device for which the DMA address is requested * @size: The size of the region to allocate in bytes * @dma_handle: A pointer for returning the DMA address of the region * @flag: memory allocator flags */ static inline void *ib_dma_alloc_coherent(struct ib_device *dev, size_t size, u64 *dma_handle, gfp_t flag) { if (dev->dma_ops) return dev->dma_ops->alloc_coherent(dev, size, dma_handle, flag); else { dma_addr_t handle; void *ret; ret = dma_alloc_coherent(dev->dma_device, size, &handle, flag); *dma_handle = handle; return ret; } } /** * ib_dma_free_coherent - Free memory allocated by ib_dma_alloc_coherent() * @dev: The device for which the DMA addresses were allocated * @size: The size of the region * @cpu_addr: the address returned by ib_dma_alloc_coherent() * @dma_handle: the DMA address returned by ib_dma_alloc_coherent() */ static inline void ib_dma_free_coherent(struct ib_device *dev, size_t size, void *cpu_addr, u64 dma_handle) { if (dev->dma_ops) dev->dma_ops->free_coherent(dev, size, cpu_addr, dma_handle); else dma_free_coherent(dev->dma_device, size, cpu_addr, dma_handle); } /** * ib_reg_phys_mr - Prepares a virtually addressed memory region for use * by an HCA. * @pd: The protection domain associated assigned to the registered region. * @phys_buf_array: Specifies a list of physical buffers to use in the * memory region. * @num_phys_buf: Specifies the size of the phys_buf_array. * @mr_access_flags: Specifies the memory access rights. * @iova_start: The offset of the region's starting I/O virtual address. */ struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd, struct ib_phys_buf *phys_buf_array, int num_phys_buf, int mr_access_flags, u64 *iova_start); /** * ib_rereg_phys_mr - Modifies the attributes of an existing memory region. * Conceptually, this call performs the functions deregister memory region * followed by register physical memory region. Where possible, * resources are reused instead of deallocated and reallocated. * @mr: The memory region to modify. * @mr_rereg_mask: A bit-mask used to indicate which of the following * properties of the memory region are being modified. * @pd: If %IB_MR_REREG_PD is set in mr_rereg_mask, this field specifies * the new protection domain to associated with the memory region, * otherwise, this parameter is ignored. * @phys_buf_array: If %IB_MR_REREG_TRANS is set in mr_rereg_mask, this * field specifies a list of physical buffers to use in the new * translation, otherwise, this parameter is ignored. * @num_phys_buf: If %IB_MR_REREG_TRANS is set in mr_rereg_mask, this * field specifies the size of the phys_buf_array, otherwise, this * parameter is ignored. * @mr_access_flags: If %IB_MR_REREG_ACCESS is set in mr_rereg_mask, this * field specifies the new memory access rights, otherwise, this * parameter is ignored. * @iova_start: The offset of the region's starting I/O virtual address. */ int ib_rereg_phys_mr(struct ib_mr *mr, int mr_rereg_mask, struct ib_pd *pd, struct ib_phys_buf *phys_buf_array, int num_phys_buf, int mr_access_flags, u64 *iova_start); /** * ib_query_mr - Retrieves information about a specific memory region. * @mr: The memory region to retrieve information about. * @mr_attr: The attributes of the specified memory region. */ int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr); /** * ib_dereg_mr - Deregisters a memory region and removes it from the * HCA translation table. * @mr: The memory region to deregister. */ int ib_dereg_mr(struct ib_mr *mr); /** * ib_alloc_fast_reg_mr - Allocates memory region usable with the * IB_WR_FAST_REG_MR send work request. * @pd: The protection domain associated with the region. * @max_page_list_len: requested max physical buffer list length to be * used with fast register work requests for this MR. */ struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len); /** * ib_alloc_fast_reg_page_list - Allocates a page list array * @device - ib device pointer. * @page_list_len - size of the page list array to be allocated. * * This allocates and returns a struct ib_fast_reg_page_list * and a * page_list array that is at least page_list_len in size. The actual * size is returned in max_page_list_len. The caller is responsible * for initializing the contents of the page_list array before posting * a send work request with the IB_WC_FAST_REG_MR opcode. * * The page_list array entries must be translated using one of the * ib_dma_*() functions just like the addresses passed to * ib_map_phys_fmr(). Once the ib_post_send() is issued, the struct * ib_fast_reg_page_list must not be modified by the caller until the * IB_WC_FAST_REG_MR work request completes. */ struct ib_fast_reg_page_list *ib_alloc_fast_reg_page_list( struct ib_device *device, int page_list_len); /** * ib_free_fast_reg_page_list - Deallocates a previously allocated * page list array. * @page_list - struct ib_fast_reg_page_list pointer to be deallocated. */ void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list); /** * ib_update_fast_reg_key - updates the key portion of the fast_reg MR * R_Key and L_Key. * @mr - struct ib_mr pointer to be updated. * @newkey - new key to be used. */ static inline void ib_update_fast_reg_key(struct ib_mr *mr, u8 newkey) { mr->lkey = (mr->lkey & 0xffffff00) | newkey; mr->rkey = (mr->rkey & 0xffffff00) | newkey; } /** * ib_alloc_mw - Allocates a memory window. * @pd: The protection domain associated with the memory window. */ struct ib_mw *ib_alloc_mw(struct ib_pd *pd); /** * ib_bind_mw - Posts a work request to the send queue of the specified * QP, which binds the memory window to the given address range and * remote access attributes. * @qp: QP to post the bind work request on. * @mw: The memory window to bind. * @mw_bind: Specifies information about the memory window, including * its address range, remote access rights, and associated memory region. */ static inline int ib_bind_mw(struct ib_qp *qp, struct ib_mw *mw, struct ib_mw_bind *mw_bind) { /* XXX reference counting in corresponding MR? */ return mw->device->bind_mw ? mw->device->bind_mw(qp, mw, mw_bind) : -ENOSYS; } /** * ib_dealloc_mw - Deallocates a memory window. * @mw: The memory window to deallocate. */ int ib_dealloc_mw(struct ib_mw *mw); /** * ib_alloc_fmr - Allocates a unmapped fast memory region. * @pd: The protection domain associated with the unmapped region. * @mr_access_flags: Specifies the memory access rights. * @fmr_attr: Attributes of the unmapped region. * * A fast memory region must be mapped before it can be used as part of * a work request. */ struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd, int mr_access_flags, struct ib_fmr_attr *fmr_attr); /** * ib_map_phys_fmr - Maps a list of physical pages to a fast memory region. * @fmr: The fast memory region to associate with the pages. * @page_list: An array of physical pages to map to the fast memory region. * @list_len: The number of pages in page_list. * @iova: The I/O virtual address to use with the mapped region. */ static inline int ib_map_phys_fmr(struct ib_fmr *fmr, u64 *page_list, int list_len, u64 iova) { return fmr->device->map_phys_fmr(fmr, page_list, list_len, iova); } /** * ib_unmap_fmr - Removes the mapping from a list of fast memory regions. * @fmr_list: A linked list of fast memory regions to unmap. */ int ib_unmap_fmr(struct list_head *fmr_list); /** * ib_dealloc_fmr - Deallocates a fast memory region. * @fmr: The fast memory region to deallocate. */ int ib_dealloc_fmr(struct ib_fmr *fmr); /** * ib_attach_mcast - Attaches the specified QP to a multicast group. * @qp: QP to attach to the multicast group. The QP must be type * IB_QPT_UD. * @gid: Multicast group GID. * @lid: Multicast group LID in host byte order. * * In order to send and receive multicast packets, subnet * administration must have created the multicast group and configured * the fabric appropriately. The port associated with the specified * QP must also be a member of the multicast group. */ int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid); /** * ib_detach_mcast - Detaches the specified QP from a multicast group. * @qp: QP to detach from the multicast group. * @gid: Multicast group GID. * @lid: Multicast group LID in host byte order. */ int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid); - /** - * ib_dealloc_xrcd - Deallocates an extended reliably connected domain. - * @xrcd: The xrc domain to deallocate. + * ib_alloc_xrcd - Allocates an XRC domain. + * @device: The device on which to allocate the XRC domain. */ -int ib_dealloc_xrcd(struct ib_xrcd *xrcd); +struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device); /** - * ib_alloc_xrcd - Allocates an extended reliably connected domain. - * @device: The device on which to allocate the xrcd. + * ib_dealloc_xrcd - Deallocates an XRC domain. + * @xrcd: The XRC domain to deallocate. */ -struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device); +int ib_dealloc_xrcd(struct ib_xrcd *xrcd); + +int ib_attach_flow(struct ib_qp *qp, struct ib_flow_spec *spec, int priority); +int ib_detach_flow(struct ib_qp *qp, struct ib_flow_spec *spec, int priority); #endif /* IB_VERBS_H */ Index: stable/9/sys/ofed/include/rdma/sdp_socket.h =================================================================== --- stable/9/sys/ofed/include/rdma/sdp_socket.h (revision 273863) +++ stable/9/sys/ofed/include/rdma/sdp_socket.h (revision 273864) @@ -1,21 +1,23 @@ /* Stuff that should go into include/linux/socket.h */ #ifndef SDP_SOCKET_H #define SDP_SOCKET_H +#ifndef __FreeBSD__ #ifndef AF_INET_SDP #define AF_INET_SDP 27 #define PF_INET_SDP AF_INET_SDP +#endif #endif #ifndef SDP_ZCOPY_THRESH #define SDP_ZCOPY_THRESH 80 #endif #ifndef SDP_LAST_BIND_ERR #define SDP_LAST_BIND_ERR 81 #endif /* TODO: AF_INET6_SDP ? */ #endif Index: stable/9/sys/sys/socket.h =================================================================== --- stable/9/sys/sys/socket.h (revision 273863) +++ stable/9/sys/sys/socket.h (revision 273864) @@ -1,685 +1,689 @@ /*- * Copyright (c) 1982, 1985, 1986, 1988, 1993, 1994 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)socket.h 8.4 (Berkeley) 2/21/94 * $FreeBSD$ */ #ifndef _SYS_SOCKET_H_ #define _SYS_SOCKET_H_ #include #include #include #include /* * Definitions related to sockets: types, address families, options. */ /* * Data types. */ #if __BSD_VISIBLE #ifndef _GID_T_DECLARED typedef __gid_t gid_t; #define _GID_T_DECLARED #endif #ifndef _OFF_T_DECLARED typedef __off_t off_t; #define _OFF_T_DECLARED #endif #ifndef _PID_T_DECLARED typedef __pid_t pid_t; #define _PID_T_DECLARED #endif #endif #ifndef _SA_FAMILY_T_DECLARED typedef __sa_family_t sa_family_t; #define _SA_FAMILY_T_DECLARED #endif #ifndef _SOCKLEN_T_DECLARED typedef __socklen_t socklen_t; #define _SOCKLEN_T_DECLARED #endif #ifndef _SSIZE_T_DECLARED typedef __ssize_t ssize_t; #define _SSIZE_T_DECLARED #endif #if __BSD_VISIBLE #ifndef _UID_T_DECLARED typedef __uid_t uid_t; #define _UID_T_DECLARED #endif #endif /* * Types */ #define SOCK_STREAM 1 /* stream socket */ #define SOCK_DGRAM 2 /* datagram socket */ #define SOCK_RAW 3 /* raw-protocol interface */ #if __BSD_VISIBLE #define SOCK_RDM 4 /* reliably-delivered message */ #endif #define SOCK_SEQPACKET 5 /* sequenced packet stream */ /* * Option flags per-socket. */ #define SO_DEBUG 0x0001 /* turn on debugging info recording */ #define SO_ACCEPTCONN 0x0002 /* socket has had listen() */ #define SO_REUSEADDR 0x0004 /* allow local address reuse */ #define SO_KEEPALIVE 0x0008 /* keep connections alive */ #define SO_DONTROUTE 0x0010 /* just use interface addresses */ #define SO_BROADCAST 0x0020 /* permit sending of broadcast msgs */ #if __BSD_VISIBLE #define SO_USELOOPBACK 0x0040 /* bypass hardware when possible */ #endif #define SO_LINGER 0x0080 /* linger on close if data present */ #define SO_OOBINLINE 0x0100 /* leave received OOB data in line */ #if __BSD_VISIBLE #define SO_REUSEPORT 0x0200 /* allow local address & port reuse */ #define SO_TIMESTAMP 0x0400 /* timestamp received dgram traffic */ #define SO_NOSIGPIPE 0x0800 /* no SIGPIPE from EPIPE */ #define SO_ACCEPTFILTER 0x1000 /* there is an accept filter */ #define SO_BINTIME 0x2000 /* timestamp received dgram traffic */ #endif #define SO_NO_OFFLOAD 0x4000 /* socket cannot be offloaded */ #define SO_NO_DDP 0x8000 /* disable direct data placement */ /* * Additional options, not kept in so_options. */ #define SO_SNDBUF 0x1001 /* send buffer size */ #define SO_RCVBUF 0x1002 /* receive buffer size */ #define SO_SNDLOWAT 0x1003 /* send low-water mark */ #define SO_RCVLOWAT 0x1004 /* receive low-water mark */ #define SO_SNDTIMEO 0x1005 /* send timeout */ #define SO_RCVTIMEO 0x1006 /* receive timeout */ #define SO_ERROR 0x1007 /* get error status and clear */ #define SO_TYPE 0x1008 /* get socket type */ #if __BSD_VISIBLE #define SO_LABEL 0x1009 /* socket's MAC label */ #define SO_PEERLABEL 0x1010 /* socket's peer's MAC label */ #define SO_LISTENQLIMIT 0x1011 /* socket's backlog limit */ #define SO_LISTENQLEN 0x1012 /* socket's complete queue length */ #define SO_LISTENINCQLEN 0x1013 /* socket's incomplete queue length */ #define SO_SETFIB 0x1014 /* use this FIB to route */ #define SO_USER_COOKIE 0x1015 /* user cookie (dummynet etc.) */ #define SO_PROTOCOL 0x1016 /* get socket protocol (Linux name) */ #define SO_PROTOTYPE SO_PROTOCOL /* alias for SO_PROTOCOL (SunOS name) */ #endif /* * Space reserved for new socket options added by third-party vendors. * This range applies to all socket option levels. New socket options * in FreeBSD should always use an option value less than SO_VENDOR. */ #if __BSD_VISIBLE #define SO_VENDOR 0x80000000 #endif /* * Structure used for manipulating linger option. */ struct linger { int l_onoff; /* option on/off */ int l_linger; /* linger time */ }; #if __BSD_VISIBLE struct accept_filter_arg { char af_name[16]; char af_arg[256-16]; }; #endif /* * Level number for (get/set)sockopt() to apply to socket itself. */ #define SOL_SOCKET 0xffff /* options for socket level */ /* * Address families. */ #define AF_UNSPEC 0 /* unspecified */ #if __BSD_VISIBLE #define AF_LOCAL AF_UNIX /* local to host (pipes, portals) */ #endif #define AF_UNIX 1 /* standardized name for AF_LOCAL */ #define AF_INET 2 /* internetwork: UDP, TCP, etc. */ #if __BSD_VISIBLE #define AF_IMPLINK 3 /* arpanet imp addresses */ #define AF_PUP 4 /* pup protocols: e.g. BSP */ #define AF_CHAOS 5 /* mit CHAOS protocols */ #define AF_NETBIOS 6 /* SMB protocols */ #define AF_ISO 7 /* ISO protocols */ #define AF_OSI AF_ISO #define AF_ECMA 8 /* European computer manufacturers */ #define AF_DATAKIT 9 /* datakit protocols */ #define AF_CCITT 10 /* CCITT protocols, X.25 etc */ #define AF_SNA 11 /* IBM SNA */ #define AF_DECnet 12 /* DECnet */ #define AF_DLI 13 /* DEC Direct data link interface */ #define AF_LAT 14 /* LAT */ #define AF_HYLINK 15 /* NSC Hyperchannel */ #define AF_APPLETALK 16 /* Apple Talk */ #define AF_ROUTE 17 /* Internal Routing Protocol */ #define AF_LINK 18 /* Link layer interface */ #define pseudo_AF_XTP 19 /* eXpress Transfer Protocol (no AF) */ #define AF_COIP 20 /* connection-oriented IP, aka ST II */ #define AF_CNT 21 /* Computer Network Technology */ #define pseudo_AF_RTIP 22 /* Help Identify RTIP packets */ #define AF_IPX 23 /* Novell Internet Protocol */ #define AF_SIP 24 /* Simple Internet Protocol */ #define pseudo_AF_PIP 25 /* Help Identify PIP packets */ #define AF_ISDN 26 /* Integrated Services Digital Network*/ #define AF_E164 AF_ISDN /* CCITT E.164 recommendation */ #define pseudo_AF_KEY 27 /* Internal key-management function */ #endif #define AF_INET6 28 /* IPv6 */ #if __BSD_VISIBLE #define AF_NATM 29 /* native ATM access */ #define AF_ATM 30 /* ATM */ #define pseudo_AF_HDRCMPLT 31 /* Used by BPF to not rewrite headers * in interface output routine */ #define AF_NETGRAPH 32 /* Netgraph sockets */ #define AF_SLOW 33 /* 802.3ad slow protocol */ #define AF_SCLUSTER 34 /* Sitara cluster protocol */ #define AF_ARP 35 #define AF_BLUETOOTH 36 /* Bluetooth sockets */ #define AF_IEEE80211 37 /* IEEE 802.11 protocol */ -#define AF_MAX 38 +#define AF_INET_SDP 40 /* OFED Socket Direct Protocol ipv4 */ +#define AF_INET6_SDP 42 /* OFED Socket Direct Protocol ipv6 */ +#define AF_MAX 42 /* * When allocating a new AF_ constant, please only allocate * even numbered constants for FreeBSD until 134 as odd numbered AF_ * constants 39-133 are now reserved for vendors. */ #define AF_VENDOR00 39 #define AF_VENDOR01 41 #define AF_VENDOR02 43 #define AF_VENDOR03 45 #define AF_VENDOR04 47 #define AF_VENDOR05 49 #define AF_VENDOR06 51 #define AF_VENDOR07 53 #define AF_VENDOR08 55 #define AF_VENDOR09 57 #define AF_VENDOR10 59 #define AF_VENDOR11 61 #define AF_VENDOR12 63 #define AF_VENDOR13 65 #define AF_VENDOR14 67 #define AF_VENDOR15 69 #define AF_VENDOR16 71 #define AF_VENDOR17 73 #define AF_VENDOR18 75 #define AF_VENDOR19 77 #define AF_VENDOR20 79 #define AF_VENDOR21 81 #define AF_VENDOR22 83 #define AF_VENDOR23 85 #define AF_VENDOR24 87 #define AF_VENDOR25 89 #define AF_VENDOR26 91 #define AF_VENDOR27 93 #define AF_VENDOR28 95 #define AF_VENDOR29 97 #define AF_VENDOR30 99 #define AF_VENDOR31 101 #define AF_VENDOR32 103 #define AF_VENDOR33 105 #define AF_VENDOR34 107 #define AF_VENDOR35 109 #define AF_VENDOR36 111 #define AF_VENDOR37 113 #define AF_VENDOR38 115 #define AF_VENDOR39 117 #define AF_VENDOR40 119 #define AF_VENDOR41 121 #define AF_VENDOR42 123 #define AF_VENDOR43 125 #define AF_VENDOR44 127 #define AF_VENDOR45 129 #define AF_VENDOR46 131 #define AF_VENDOR47 133 #endif /* * Structure used by kernel to store most * addresses. */ struct sockaddr { unsigned char sa_len; /* total length */ sa_family_t sa_family; /* address family */ char sa_data[14]; /* actually longer; address value */ }; #if __BSD_VISIBLE #define SOCK_MAXADDRLEN 255 /* longest possible addresses */ /* * Structure used by kernel to pass protocol * information in raw sockets. */ struct sockproto { unsigned short sp_family; /* address family */ unsigned short sp_protocol; /* protocol */ }; #endif #include #if __BSD_VISIBLE /* * Protocol families, same as address families for now. */ #define PF_UNSPEC AF_UNSPEC #define PF_LOCAL AF_LOCAL #define PF_UNIX PF_LOCAL /* backward compatibility */ #define PF_INET AF_INET #define PF_IMPLINK AF_IMPLINK #define PF_PUP AF_PUP #define PF_CHAOS AF_CHAOS #define PF_NETBIOS AF_NETBIOS #define PF_ISO AF_ISO #define PF_OSI AF_ISO #define PF_ECMA AF_ECMA #define PF_DATAKIT AF_DATAKIT #define PF_CCITT AF_CCITT #define PF_SNA AF_SNA #define PF_DECnet AF_DECnet #define PF_DLI AF_DLI #define PF_LAT AF_LAT #define PF_HYLINK AF_HYLINK #define PF_APPLETALK AF_APPLETALK #define PF_ROUTE AF_ROUTE #define PF_LINK AF_LINK #define PF_XTP pseudo_AF_XTP /* really just proto family, no AF */ #define PF_COIP AF_COIP #define PF_CNT AF_CNT #define PF_SIP AF_SIP #define PF_IPX AF_IPX #define PF_RTIP pseudo_AF_RTIP /* same format as AF_INET */ #define PF_PIP pseudo_AF_PIP #define PF_ISDN AF_ISDN #define PF_KEY pseudo_AF_KEY #define PF_INET6 AF_INET6 #define PF_NATM AF_NATM #define PF_ATM AF_ATM #define PF_NETGRAPH AF_NETGRAPH #define PF_SLOW AF_SLOW #define PF_SCLUSTER AF_SCLUSTER #define PF_ARP AF_ARP #define PF_BLUETOOTH AF_BLUETOOTH +#define PF_INET_SDP AF_INET_SDP +#define PF_INET6_SDP AF_INET6_SDP #define PF_MAX AF_MAX /* * Definitions for network related sysctl, CTL_NET. * * Second level is protocol family. * Third level is protocol number. * * Further levels are defined by the individual families below. */ #define NET_MAXID AF_MAX #define CTL_NET_NAMES { \ { 0, 0 }, \ { "unix", CTLTYPE_NODE }, \ { "inet", CTLTYPE_NODE }, \ { "implink", CTLTYPE_NODE }, \ { "pup", CTLTYPE_NODE }, \ { "chaos", CTLTYPE_NODE }, \ { "xerox_ns", CTLTYPE_NODE }, \ { "iso", CTLTYPE_NODE }, \ { "emca", CTLTYPE_NODE }, \ { "datakit", CTLTYPE_NODE }, \ { "ccitt", CTLTYPE_NODE }, \ { "ibm_sna", CTLTYPE_NODE }, \ { "decnet", CTLTYPE_NODE }, \ { "dec_dli", CTLTYPE_NODE }, \ { "lat", CTLTYPE_NODE }, \ { "hylink", CTLTYPE_NODE }, \ { "appletalk", CTLTYPE_NODE }, \ { "route", CTLTYPE_NODE }, \ { "link_layer", CTLTYPE_NODE }, \ { "xtp", CTLTYPE_NODE }, \ { "coip", CTLTYPE_NODE }, \ { "cnt", CTLTYPE_NODE }, \ { "rtip", CTLTYPE_NODE }, \ { "ipx", CTLTYPE_NODE }, \ { "sip", CTLTYPE_NODE }, \ { "pip", CTLTYPE_NODE }, \ { "isdn", CTLTYPE_NODE }, \ { "key", CTLTYPE_NODE }, \ { "inet6", CTLTYPE_NODE }, \ { "natm", CTLTYPE_NODE }, \ { "atm", CTLTYPE_NODE }, \ { "hdrcomplete", CTLTYPE_NODE }, \ { "netgraph", CTLTYPE_NODE }, \ { "snp", CTLTYPE_NODE }, \ { "scp", CTLTYPE_NODE }, \ } /* * PF_ROUTE - Routing table * * Three additional levels are defined: * Fourth: address family, 0 is wildcard * Fifth: type of info, defined below * Sixth: flag(s) to mask with for NET_RT_FLAGS */ #define NET_RT_DUMP 1 /* dump; may limit to a.f. */ #define NET_RT_FLAGS 2 /* by flags, e.g. RESOLVING */ #define NET_RT_IFLIST 3 /* survey interface list */ #define NET_RT_IFMALIST 4 /* return multicast address list */ #define NET_RT_IFLISTL 5 /* Survey interface list, using 'l'en * versions of msghdr structs. */ #define NET_RT_MAXID 6 #define CTL_NET_RT_NAMES { \ { 0, 0 }, \ { "dump", CTLTYPE_STRUCT }, \ { "flags", CTLTYPE_STRUCT }, \ { "iflist", CTLTYPE_STRUCT }, \ { "ifmalist", CTLTYPE_STRUCT }, \ { "iflistl", CTLTYPE_STRUCT }, \ } #endif /* __BSD_VISIBLE */ /* * Maximum queue length specifiable by listen. */ #define SOMAXCONN 128 /* * Message header for recvmsg and sendmsg calls. * Used value-result for recvmsg, value only for sendmsg. */ struct msghdr { void *msg_name; /* optional address */ socklen_t msg_namelen; /* size of address */ struct iovec *msg_iov; /* scatter/gather array */ int msg_iovlen; /* # elements in msg_iov */ void *msg_control; /* ancillary data, see below */ socklen_t msg_controllen; /* ancillary data buffer len */ int msg_flags; /* flags on received message */ }; #define MSG_OOB 0x1 /* process out-of-band data */ #define MSG_PEEK 0x2 /* peek at incoming message */ #define MSG_DONTROUTE 0x4 /* send without using routing tables */ #define MSG_EOR 0x8 /* data completes record */ #define MSG_TRUNC 0x10 /* data discarded before delivery */ #define MSG_CTRUNC 0x20 /* control data lost before delivery */ #define MSG_WAITALL 0x40 /* wait for full request or error */ #define MSG_NOTIFICATION 0x2000 /* SCTP notification */ #if __BSD_VISIBLE #define MSG_DONTWAIT 0x80 /* this message should be nonblocking */ #define MSG_EOF 0x100 /* data completes connection */ #define MSG_NBIO 0x4000 /* FIONBIO mode, used by fifofs */ #define MSG_COMPAT 0x8000 /* used in sendit() */ #endif #ifdef _KERNEL #define MSG_SOCALLBCK 0x10000 /* for use by socket callbacks - soreceive (TCP) */ #endif #if __BSD_VISIBLE #define MSG_NOSIGNAL 0x20000 /* do not generate SIGPIPE on EOF */ #endif /* * Header for ancillary data objects in msg_control buffer. * Used for additional information with/about a datagram * not expressible by flags. The format is a sequence * of message elements headed by cmsghdr structures. */ struct cmsghdr { socklen_t cmsg_len; /* data byte count, including hdr */ int cmsg_level; /* originating protocol */ int cmsg_type; /* protocol-specific type */ /* followed by u_char cmsg_data[]; */ }; #if __BSD_VISIBLE /* * While we may have more groups than this, the cmsgcred struct must * be able to fit in an mbuf and we have historically supported a * maximum of 16 groups. */ #define CMGROUP_MAX 16 /* * Credentials structure, used to verify the identity of a peer * process that has sent us a message. This is allocated by the * peer process but filled in by the kernel. This prevents the * peer from lying about its identity. (Note that cmcred_groups[0] * is the effective GID.) */ struct cmsgcred { pid_t cmcred_pid; /* PID of sending process */ uid_t cmcred_uid; /* real UID of sending process */ uid_t cmcred_euid; /* effective UID of sending process */ gid_t cmcred_gid; /* real GID of sending process */ short cmcred_ngroups; /* number or groups */ gid_t cmcred_groups[CMGROUP_MAX]; /* groups */ }; /* * Socket credentials. */ struct sockcred { uid_t sc_uid; /* real user id */ uid_t sc_euid; /* effective user id */ gid_t sc_gid; /* real group id */ gid_t sc_egid; /* effective group id */ int sc_ngroups; /* number of supplemental groups */ gid_t sc_groups[1]; /* variable length */ }; /* * Compute size of a sockcred structure with groups. */ #define SOCKCREDSIZE(ngrps) \ (sizeof(struct sockcred) + (sizeof(gid_t) * ((ngrps) - 1))) #endif /* __BSD_VISIBLE */ /* given pointer to struct cmsghdr, return pointer to data */ #define CMSG_DATA(cmsg) ((unsigned char *)(cmsg) + \ _ALIGN(sizeof(struct cmsghdr))) /* given pointer to struct cmsghdr, return pointer to next cmsghdr */ #define CMSG_NXTHDR(mhdr, cmsg) \ ((char *)(cmsg) == NULL ? CMSG_FIRSTHDR(mhdr) : \ ((char *)(cmsg) + _ALIGN(((struct cmsghdr *)(cmsg))->cmsg_len) + \ _ALIGN(sizeof(struct cmsghdr)) > \ (char *)(mhdr)->msg_control + (mhdr)->msg_controllen) ? \ (struct cmsghdr *)0 : \ (struct cmsghdr *)(void *)((char *)(cmsg) + \ _ALIGN(((struct cmsghdr *)(cmsg))->cmsg_len))) /* * RFC 2292 requires to check msg_controllen, in case that the kernel returns * an empty list for some reasons. */ #define CMSG_FIRSTHDR(mhdr) \ ((mhdr)->msg_controllen >= sizeof(struct cmsghdr) ? \ (struct cmsghdr *)(mhdr)->msg_control : \ (struct cmsghdr *)NULL) #if __BSD_VISIBLE /* RFC 2292 additions */ #define CMSG_SPACE(l) (_ALIGN(sizeof(struct cmsghdr)) + _ALIGN(l)) #define CMSG_LEN(l) (_ALIGN(sizeof(struct cmsghdr)) + (l)) #endif #ifdef _KERNEL #define CMSG_ALIGN(n) _ALIGN(n) #endif /* "Socket"-level control message types: */ #define SCM_RIGHTS 0x01 /* access rights (array of int) */ #if __BSD_VISIBLE #define SCM_TIMESTAMP 0x02 /* timestamp (struct timeval) */ #define SCM_CREDS 0x03 /* process creds (struct cmsgcred) */ #define SCM_BINTIME 0x04 /* timestamp (struct bintime) */ #endif #if __BSD_VISIBLE /* * 4.3 compat sockaddr, move to compat file later */ struct osockaddr { unsigned short sa_family; /* address family */ char sa_data[14]; /* up to 14 bytes of direct address */ }; /* * 4.3-compat message header (move to compat file later). */ struct omsghdr { char *msg_name; /* optional address */ int msg_namelen; /* size of address */ struct iovec *msg_iov; /* scatter/gather array */ int msg_iovlen; /* # elements in msg_iov */ char *msg_accrights; /* access rights sent/received */ int msg_accrightslen; }; #endif /* * howto arguments for shutdown(2), specified by Posix.1g. */ #define SHUT_RD 0 /* shut down the reading side */ #define SHUT_WR 1 /* shut down the writing side */ #define SHUT_RDWR 2 /* shut down both sides */ /* we cheat and use the SHUT_XX defines for these */ #define PRU_FLUSH_RD SHUT_RD #define PRU_FLUSH_WR SHUT_WR #define PRU_FLUSH_RDWR SHUT_RDWR #if __BSD_VISIBLE /* * sendfile(2) header/trailer struct */ struct sf_hdtr { struct iovec *headers; /* pointer to an array of header struct iovec's */ int hdr_cnt; /* number of header iovec's */ struct iovec *trailers; /* pointer to an array of trailer struct iovec's */ int trl_cnt; /* number of trailer iovec's */ }; /* * Sendfile-specific flag(s) */ #define SF_NODISKIO 0x00000001 #define SF_MNOWAIT 0x00000002 #define SF_SYNC 0x00000004 #endif #ifndef _KERNEL #include __BEGIN_DECLS int accept(int, struct sockaddr * __restrict, socklen_t * __restrict); int bind(int, const struct sockaddr *, socklen_t); int connect(int, const struct sockaddr *, socklen_t); int getpeername(int, struct sockaddr * __restrict, socklen_t * __restrict); int getsockname(int, struct sockaddr * __restrict, socklen_t * __restrict); int getsockopt(int, int, int, void * __restrict, socklen_t * __restrict); int listen(int, int); ssize_t recv(int, void *, size_t, int); ssize_t recvfrom(int, void *, size_t, int, struct sockaddr * __restrict, socklen_t * __restrict); ssize_t recvmsg(int, struct msghdr *, int); ssize_t send(int, const void *, size_t, int); ssize_t sendto(int, const void *, size_t, int, const struct sockaddr *, socklen_t); ssize_t sendmsg(int, const struct msghdr *, int); #if __BSD_VISIBLE int sendfile(int, int, off_t, size_t, struct sf_hdtr *, off_t *, int); int setfib(int); #endif int setsockopt(int, int, int, const void *, socklen_t); int shutdown(int, int); int sockatmark(int); int socket(int, int, int); int socketpair(int, int, int, int *); __END_DECLS #endif /* !_KERNEL */ #ifdef _KERNEL struct socket; struct tcpcb *so_sototcpcb(struct socket *so); struct inpcb *so_sotoinpcb(struct socket *so); struct sockbuf *so_sockbuf_snd(struct socket *); struct sockbuf *so_sockbuf_rcv(struct socket *); int so_state_get(const struct socket *); void so_state_set(struct socket *, int); int so_options_get(const struct socket *); void so_options_set(struct socket *, int); int so_error_get(const struct socket *); void so_error_set(struct socket *, int); int so_linger_get(const struct socket *); void so_linger_set(struct socket *, int); struct protosw *so_protosw_get(const struct socket *); void so_protosw_set(struct socket *, struct protosw *); void so_sorwakeup_locked(struct socket *so); void so_sowwakeup_locked(struct socket *so); void so_sorwakeup(struct socket *so); void so_sowwakeup(struct socket *so); void so_lock(struct socket *so); void so_unlock(struct socket *so); void so_listeners_apply_all(struct socket *so, void (*func)(struct socket *, void *), void *arg); #endif #endif /* !_SYS_SOCKET_H_ */ Index: stable/9/sys/sys =================================================================== --- stable/9/sys/sys (revision 273863) +++ stable/9/sys/sys (revision 273864) Property changes on: stable/9/sys/sys ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/sys/sys:r230135,247671,247675,254121-254123,254734,255932,255969-255970,255972-255973,256116,256179,256269 Index: stable/9/sys =================================================================== --- stable/9/sys (revision 273863) +++ stable/9/sys (revision 273864) Property changes on: stable/9/sys ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/sys:r230135,247671,247675,254121-254123,254734-254735,255932,255969-255970,255972-255973,256116,256179,256269,256281,256470,256682,257863-257864,257867,258276,259608,268314,269861-269862,271127,272407,272683