Index: stable/11/sys/dev/mlx5/mlx5_core/mlx5_cmd.c =================================================================== --- stable/11/sys/dev/mlx5/mlx5_core/mlx5_cmd.c (revision 353223) +++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_cmd.c (revision 353224) @@ -1,1610 +1,1617 @@ /*- * Copyright (c) 2013-2019, Mellanox Technologies, Ltd. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include #include #include #include #include #include #include #include #include #include "mlx5_core.h" static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size); static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg); static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg); enum { CMD_IF_REV = 5, }; enum { NUM_LONG_LISTS = 2, NUM_MED_LISTS = 64, LONG_LIST_SIZE = (2ULL * 1024 * 1024 * 1024 / PAGE_SIZE) * 8 + 16 + MLX5_CMD_DATA_BLOCK_SIZE, MED_LIST_SIZE = 16 + MLX5_CMD_DATA_BLOCK_SIZE, }; enum { MLX5_CMD_DELIVERY_STAT_OK = 0x0, MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR = 0x1, MLX5_CMD_DELIVERY_STAT_TOK_ERR = 0x2, MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR = 0x3, MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR = 0x4, MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR = 0x5, MLX5_CMD_DELIVERY_STAT_FW_ERR = 0x6, MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR = 0x7, MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR = 0x8, MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR = 0x9, MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10, }; struct mlx5_ifc_mbox_out_bits { u8 status[0x8]; u8 reserved_at_8[0x18]; u8 syndrome[0x20]; u8 reserved_at_40[0x40]; }; struct mlx5_ifc_mbox_in_bits { u8 opcode[0x10]; u8 reserved_at_10[0x10]; u8 reserved_at_20[0x10]; u8 op_mod[0x10]; u8 reserved_at_40[0x40]; }; static struct mlx5_cmd_work_ent *alloc_cmd(struct mlx5_cmd *cmd, struct mlx5_cmd_msg *in, int uin_size, struct mlx5_cmd_msg *out, void *uout, int uout_size, mlx5_cmd_cbk_t cbk, void *context, int page_queue) { gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL; struct mlx5_cmd_work_ent *ent; ent = kzalloc(sizeof(*ent), alloc_flags); if (!ent) return ERR_PTR(-ENOMEM); ent->in = in; ent->uin_size = uin_size; ent->out = out; ent->uout = uout; ent->uout_size = uout_size; ent->callback = cbk; ent->context = context; ent->cmd = cmd; ent->page_queue = page_queue; return ent; } static u8 alloc_token(struct mlx5_cmd *cmd) { u8 token; spin_lock(&cmd->token_lock); cmd->token++; if (cmd->token == 0) cmd->token++; token = cmd->token; spin_unlock(&cmd->token_lock); return token; } static int alloc_ent(struct mlx5_cmd_work_ent *ent) { unsigned long flags; struct mlx5_cmd *cmd = ent->cmd; struct mlx5_core_dev *dev = container_of(cmd, struct mlx5_core_dev, cmd); int ret = cmd->max_reg_cmds; spin_lock_irqsave(&cmd->alloc_lock, flags); if (!ent->page_queue) { ret = find_first_bit(&cmd->bitmask, cmd->max_reg_cmds); if (ret >= cmd->max_reg_cmds) ret = -1; } if (dev->state != MLX5_DEVICE_STATE_UP) ret = -1; if (ret != -1) { ent->busy = 1; ent->idx = ret; clear_bit(ent->idx, &cmd->bitmask); cmd->ent_mode[ent->idx] = ent->polling ? MLX5_CMD_MODE_POLLING : MLX5_CMD_MODE_EVENTS; cmd->ent_arr[ent->idx] = ent; } spin_unlock_irqrestore(&cmd->alloc_lock, flags); return ret; } static void free_ent(struct mlx5_cmd *cmd, int idx) { unsigned long flags; spin_lock_irqsave(&cmd->alloc_lock, flags); cmd->ent_arr[idx] = NULL; /* safety clear */ cmd->ent_mode[idx] = MLX5_CMD_MODE_POLLING; /* reset mode */ set_bit(idx, &cmd->bitmask); spin_unlock_irqrestore(&cmd->alloc_lock, flags); } static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx) { return cmd->cmd_buf + (idx << cmd->log_stride); } static u8 xor8_buf(void *buf, int len) { u8 *ptr = buf; u8 sum = 0; int i; for (i = 0; i < len; i++) sum ^= ptr[i]; return sum; } static int verify_block_sig(struct mlx5_cmd_prot_block *block) { if (xor8_buf(block->rsvd0, sizeof(*block) - sizeof(block->data) - 1) != 0xff) return -EINVAL; if (xor8_buf(block, sizeof(*block)) != 0xff) return -EINVAL; return 0; } static void calc_block_sig(struct mlx5_cmd_prot_block *block, u8 token, int csum) { block->token = token; if (csum) { block->ctrl_sig = ~xor8_buf(block->rsvd0, sizeof(*block) - sizeof(block->data) - 2); block->sig = ~xor8_buf(block, sizeof(*block) - 1); } } static void calc_chain_sig(struct mlx5_cmd_msg *msg, u8 token, int csum) { size_t i; for (i = 0; i != (msg->numpages * MLX5_NUM_CMDS_IN_ADAPTER_PAGE); i++) { struct mlx5_cmd_prot_block *block; block = mlx5_fwp_get_virt(msg, i * MLX5_CMD_MBOX_SIZE); /* compute signature */ calc_block_sig(block, token, csum); /* check for last block */ if (block->next == 0) break; } /* make sure data gets written to RAM */ mlx5_fwp_flush(msg); } static void set_signature(struct mlx5_cmd_work_ent *ent, int csum) { ent->lay->sig = ~xor8_buf(ent->lay, sizeof(*ent->lay)); calc_chain_sig(ent->in, ent->token, csum); calc_chain_sig(ent->out, ent->token, csum); } static void poll_timeout(struct mlx5_cmd_work_ent *ent) { struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd); int poll_end = jiffies + msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC + 1000); u8 own; do { own = ent->lay->status_own; if (!(own & CMD_OWNER_HW) || dev->state != MLX5_DEVICE_STATE_UP) { ent->ret = 0; return; } usleep_range(5000, 10000); } while (time_before(jiffies, poll_end)); ent->ret = -ETIMEDOUT; } static void free_cmd(struct mlx5_cmd_work_ent *ent) { cancel_delayed_work_sync(&ent->cb_timeout_work); kfree(ent); } static int verify_signature(struct mlx5_cmd_work_ent *ent) { struct mlx5_cmd_msg *msg = ent->out; size_t i; int err; u8 sig; sig = xor8_buf(ent->lay, sizeof(*ent->lay)); if (sig != 0xff) return -EINVAL; for (i = 0; i != (msg->numpages * MLX5_NUM_CMDS_IN_ADAPTER_PAGE); i++) { struct mlx5_cmd_prot_block *block; block = mlx5_fwp_get_virt(msg, i * MLX5_CMD_MBOX_SIZE); /* compute signature */ err = verify_block_sig(block); if (err != 0) return (err); /* check for last block */ if (block->next == 0) break; } return (0); } static void dump_buf(void *buf, int size, int data_only, int offset) { __be32 *p = buf; int i; for (i = 0; i < size; i += 16) { pr_debug("%03x: %08x %08x %08x %08x\n", offset, be32_to_cpu(p[0]), be32_to_cpu(p[1]), be32_to_cpu(p[2]), be32_to_cpu(p[3])); p += 4; offset += 16; } if (!data_only) pr_debug("\n"); } enum { MLX5_DRIVER_STATUS_ABORTED = 0xfe, MLX5_DRIVER_SYND = 0xbadd00de, }; static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, u32 *synd, u8 *status) { *synd = 0; *status = 0; switch (op) { case MLX5_CMD_OP_TEARDOWN_HCA: case MLX5_CMD_OP_DISABLE_HCA: case MLX5_CMD_OP_MANAGE_PAGES: case MLX5_CMD_OP_DESTROY_MKEY: case MLX5_CMD_OP_DESTROY_EQ: case MLX5_CMD_OP_DESTROY_CQ: case MLX5_CMD_OP_DESTROY_QP: case MLX5_CMD_OP_DESTROY_PSV: case MLX5_CMD_OP_DESTROY_SRQ: case MLX5_CMD_OP_DESTROY_XRC_SRQ: case MLX5_CMD_OP_DESTROY_DCT: case MLX5_CMD_OP_DEALLOC_Q_COUNTER: case MLX5_CMD_OP_DEALLOC_PD: case MLX5_CMD_OP_DEALLOC_UAR: case MLX5_CMD_OP_DETACH_FROM_MCG: case MLX5_CMD_OP_DEALLOC_XRCD: case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN: case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT: case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY: case MLX5_CMD_OP_DESTROY_TIR: case MLX5_CMD_OP_DESTROY_SQ: case MLX5_CMD_OP_DESTROY_RQ: case MLX5_CMD_OP_DESTROY_RMP: case MLX5_CMD_OP_DESTROY_TIS: case MLX5_CMD_OP_DESTROY_RQT: case MLX5_CMD_OP_DESTROY_FLOW_TABLE: case MLX5_CMD_OP_DESTROY_FLOW_GROUP: case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY: case MLX5_CMD_OP_2ERR_QP: case MLX5_CMD_OP_2RST_QP: case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT: case MLX5_CMD_OP_MODIFY_FLOW_TABLE: case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: case MLX5_CMD_OP_SET_FLOW_TABLE_ROOT: return MLX5_CMD_STAT_OK; case MLX5_CMD_OP_QUERY_HCA_CAP: case MLX5_CMD_OP_QUERY_ADAPTER: case MLX5_CMD_OP_INIT_HCA: case MLX5_CMD_OP_ENABLE_HCA: case MLX5_CMD_OP_QUERY_PAGES: case MLX5_CMD_OP_SET_HCA_CAP: case MLX5_CMD_OP_QUERY_ISSI: case MLX5_CMD_OP_SET_ISSI: case MLX5_CMD_OP_CREATE_MKEY: case MLX5_CMD_OP_QUERY_MKEY: case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS: case MLX5_CMD_OP_PAGE_FAULT_RESUME: case MLX5_CMD_OP_CREATE_EQ: case MLX5_CMD_OP_QUERY_EQ: case MLX5_CMD_OP_GEN_EQE: case MLX5_CMD_OP_CREATE_CQ: case MLX5_CMD_OP_QUERY_CQ: case MLX5_CMD_OP_MODIFY_CQ: case MLX5_CMD_OP_CREATE_QP: case MLX5_CMD_OP_RST2INIT_QP: case MLX5_CMD_OP_INIT2RTR_QP: case MLX5_CMD_OP_RTR2RTS_QP: case MLX5_CMD_OP_RTS2RTS_QP: case MLX5_CMD_OP_SQERR2RTS_QP: case MLX5_CMD_OP_QUERY_QP: case MLX5_CMD_OP_SQD_RTS_QP: case MLX5_CMD_OP_INIT2INIT_QP: case MLX5_CMD_OP_CREATE_PSV: case MLX5_CMD_OP_CREATE_SRQ: case MLX5_CMD_OP_QUERY_SRQ: case MLX5_CMD_OP_ARM_RQ: case MLX5_CMD_OP_CREATE_XRC_SRQ: case MLX5_CMD_OP_QUERY_XRC_SRQ: case MLX5_CMD_OP_ARM_XRC_SRQ: case MLX5_CMD_OP_CREATE_DCT: case MLX5_CMD_OP_DRAIN_DCT: case MLX5_CMD_OP_QUERY_DCT: case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION: case MLX5_CMD_OP_QUERY_VPORT_STATE: case MLX5_CMD_OP_MODIFY_VPORT_STATE: case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT: case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT: case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT: case MLX5_CMD_OP_QUERY_ROCE_ADDRESS: case MLX5_CMD_OP_SET_ROCE_ADDRESS: case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT: case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT: case MLX5_CMD_OP_QUERY_HCA_VPORT_GID: case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY: case MLX5_CMD_OP_QUERY_VNIC_ENV: case MLX5_CMD_OP_QUERY_VPORT_COUNTER: case MLX5_CMD_OP_ALLOC_Q_COUNTER: case MLX5_CMD_OP_QUERY_Q_COUNTER: case MLX5_CMD_OP_ALLOC_PD: case MLX5_CMD_OP_ALLOC_UAR: case MLX5_CMD_OP_CONFIG_INT_MODERATION: case MLX5_CMD_OP_ACCESS_REG: case MLX5_CMD_OP_ATTACH_TO_MCG: case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG: case MLX5_CMD_OP_MAD_IFC: case MLX5_CMD_OP_QUERY_MAD_DEMUX: case MLX5_CMD_OP_SET_MAD_DEMUX: case MLX5_CMD_OP_NOP: case MLX5_CMD_OP_ALLOC_XRCD: case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN: case MLX5_CMD_OP_QUERY_CONG_STATUS: case MLX5_CMD_OP_MODIFY_CONG_STATUS: case MLX5_CMD_OP_QUERY_CONG_PARAMS: case MLX5_CMD_OP_MODIFY_CONG_PARAMS: case MLX5_CMD_OP_QUERY_CONG_STATISTICS: case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY: case MLX5_CMD_OP_CREATE_TIR: case MLX5_CMD_OP_MODIFY_TIR: case MLX5_CMD_OP_QUERY_TIR: case MLX5_CMD_OP_CREATE_SQ: case MLX5_CMD_OP_MODIFY_SQ: case MLX5_CMD_OP_QUERY_SQ: case MLX5_CMD_OP_CREATE_RQ: case MLX5_CMD_OP_MODIFY_RQ: case MLX5_CMD_OP_QUERY_RQ: case MLX5_CMD_OP_CREATE_RMP: case MLX5_CMD_OP_MODIFY_RMP: case MLX5_CMD_OP_QUERY_RMP: case MLX5_CMD_OP_CREATE_TIS: case MLX5_CMD_OP_MODIFY_TIS: case MLX5_CMD_OP_QUERY_TIS: case MLX5_CMD_OP_CREATE_RQT: case MLX5_CMD_OP_MODIFY_RQT: case MLX5_CMD_OP_QUERY_RQT: case MLX5_CMD_OP_CREATE_FLOW_TABLE: case MLX5_CMD_OP_QUERY_FLOW_TABLE: case MLX5_CMD_OP_CREATE_FLOW_GROUP: case MLX5_CMD_OP_QUERY_FLOW_GROUP: case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY: *status = MLX5_DRIVER_STATUS_ABORTED; *synd = MLX5_DRIVER_SYND; return -EIO; default: mlx5_core_err(dev, "Unknown FW command (%d)\n", op); return -EINVAL; } } const char *mlx5_command_str(int command) { #define MLX5_COMMAND_STR_CASE(__cmd) case MLX5_CMD_OP_ ## __cmd: return #__cmd switch (command) { MLX5_COMMAND_STR_CASE(QUERY_HCA_CAP); MLX5_COMMAND_STR_CASE(SET_HCA_CAP); MLX5_COMMAND_STR_CASE(QUERY_ADAPTER); MLX5_COMMAND_STR_CASE(INIT_HCA); MLX5_COMMAND_STR_CASE(TEARDOWN_HCA); MLX5_COMMAND_STR_CASE(ENABLE_HCA); MLX5_COMMAND_STR_CASE(DISABLE_HCA); MLX5_COMMAND_STR_CASE(QUERY_PAGES); MLX5_COMMAND_STR_CASE(MANAGE_PAGES); MLX5_COMMAND_STR_CASE(QUERY_ISSI); MLX5_COMMAND_STR_CASE(SET_ISSI); MLX5_COMMAND_STR_CASE(CREATE_MKEY); MLX5_COMMAND_STR_CASE(QUERY_MKEY); MLX5_COMMAND_STR_CASE(DESTROY_MKEY); MLX5_COMMAND_STR_CASE(QUERY_SPECIAL_CONTEXTS); MLX5_COMMAND_STR_CASE(PAGE_FAULT_RESUME); MLX5_COMMAND_STR_CASE(CREATE_EQ); MLX5_COMMAND_STR_CASE(DESTROY_EQ); MLX5_COMMAND_STR_CASE(QUERY_EQ); MLX5_COMMAND_STR_CASE(GEN_EQE); MLX5_COMMAND_STR_CASE(CREATE_CQ); MLX5_COMMAND_STR_CASE(DESTROY_CQ); MLX5_COMMAND_STR_CASE(QUERY_CQ); MLX5_COMMAND_STR_CASE(MODIFY_CQ); MLX5_COMMAND_STR_CASE(CREATE_QP); MLX5_COMMAND_STR_CASE(DESTROY_QP); MLX5_COMMAND_STR_CASE(RST2INIT_QP); MLX5_COMMAND_STR_CASE(INIT2RTR_QP); MLX5_COMMAND_STR_CASE(RTR2RTS_QP); MLX5_COMMAND_STR_CASE(RTS2RTS_QP); MLX5_COMMAND_STR_CASE(SQERR2RTS_QP); MLX5_COMMAND_STR_CASE(2ERR_QP); MLX5_COMMAND_STR_CASE(2RST_QP); MLX5_COMMAND_STR_CASE(QUERY_QP); MLX5_COMMAND_STR_CASE(SQD_RTS_QP); MLX5_COMMAND_STR_CASE(MAD_IFC); MLX5_COMMAND_STR_CASE(INIT2INIT_QP); MLX5_COMMAND_STR_CASE(CREATE_PSV); MLX5_COMMAND_STR_CASE(DESTROY_PSV); MLX5_COMMAND_STR_CASE(CREATE_SRQ); MLX5_COMMAND_STR_CASE(DESTROY_SRQ); MLX5_COMMAND_STR_CASE(QUERY_SRQ); MLX5_COMMAND_STR_CASE(ARM_RQ); MLX5_COMMAND_STR_CASE(CREATE_XRC_SRQ); MLX5_COMMAND_STR_CASE(DESTROY_XRC_SRQ); MLX5_COMMAND_STR_CASE(QUERY_XRC_SRQ); MLX5_COMMAND_STR_CASE(ARM_XRC_SRQ); MLX5_COMMAND_STR_CASE(CREATE_DCT); MLX5_COMMAND_STR_CASE(SET_DC_CNAK_TRACE); MLX5_COMMAND_STR_CASE(DESTROY_DCT); MLX5_COMMAND_STR_CASE(DRAIN_DCT); MLX5_COMMAND_STR_CASE(QUERY_DCT); MLX5_COMMAND_STR_CASE(ARM_DCT_FOR_KEY_VIOLATION); MLX5_COMMAND_STR_CASE(QUERY_VPORT_STATE); MLX5_COMMAND_STR_CASE(MODIFY_VPORT_STATE); MLX5_COMMAND_STR_CASE(QUERY_ESW_VPORT_CONTEXT); MLX5_COMMAND_STR_CASE(MODIFY_ESW_VPORT_CONTEXT); MLX5_COMMAND_STR_CASE(QUERY_NIC_VPORT_CONTEXT); MLX5_COMMAND_STR_CASE(MODIFY_NIC_VPORT_CONTEXT); MLX5_COMMAND_STR_CASE(QUERY_ROCE_ADDRESS); MLX5_COMMAND_STR_CASE(SET_ROCE_ADDRESS); MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_CONTEXT); MLX5_COMMAND_STR_CASE(MODIFY_HCA_VPORT_CONTEXT); MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_GID); MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_PKEY); MLX5_COMMAND_STR_CASE(QUERY_VNIC_ENV); MLX5_COMMAND_STR_CASE(QUERY_VPORT_COUNTER); MLX5_COMMAND_STR_CASE(SET_WOL_ROL); MLX5_COMMAND_STR_CASE(QUERY_WOL_ROL); MLX5_COMMAND_STR_CASE(ALLOC_Q_COUNTER); MLX5_COMMAND_STR_CASE(DEALLOC_Q_COUNTER); MLX5_COMMAND_STR_CASE(QUERY_Q_COUNTER); MLX5_COMMAND_STR_CASE(ALLOC_PD); MLX5_COMMAND_STR_CASE(DEALLOC_PD); MLX5_COMMAND_STR_CASE(ALLOC_UAR); MLX5_COMMAND_STR_CASE(DEALLOC_UAR); MLX5_COMMAND_STR_CASE(CONFIG_INT_MODERATION); MLX5_COMMAND_STR_CASE(ATTACH_TO_MCG); MLX5_COMMAND_STR_CASE(DETACH_FROM_MCG); MLX5_COMMAND_STR_CASE(GET_DROPPED_PACKET_LOG); MLX5_COMMAND_STR_CASE(QUERY_MAD_DEMUX); MLX5_COMMAND_STR_CASE(SET_MAD_DEMUX); MLX5_COMMAND_STR_CASE(NOP); MLX5_COMMAND_STR_CASE(ALLOC_XRCD); MLX5_COMMAND_STR_CASE(DEALLOC_XRCD); MLX5_COMMAND_STR_CASE(ALLOC_TRANSPORT_DOMAIN); MLX5_COMMAND_STR_CASE(DEALLOC_TRANSPORT_DOMAIN); MLX5_COMMAND_STR_CASE(QUERY_CONG_STATUS); MLX5_COMMAND_STR_CASE(MODIFY_CONG_STATUS); MLX5_COMMAND_STR_CASE(QUERY_CONG_PARAMS); MLX5_COMMAND_STR_CASE(MODIFY_CONG_PARAMS); MLX5_COMMAND_STR_CASE(QUERY_CONG_STATISTICS); MLX5_COMMAND_STR_CASE(ADD_VXLAN_UDP_DPORT); MLX5_COMMAND_STR_CASE(DELETE_VXLAN_UDP_DPORT); MLX5_COMMAND_STR_CASE(SET_L2_TABLE_ENTRY); MLX5_COMMAND_STR_CASE(QUERY_L2_TABLE_ENTRY); MLX5_COMMAND_STR_CASE(DELETE_L2_TABLE_ENTRY); MLX5_COMMAND_STR_CASE(CREATE_RMP); MLX5_COMMAND_STR_CASE(MODIFY_RMP); MLX5_COMMAND_STR_CASE(DESTROY_RMP); MLX5_COMMAND_STR_CASE(QUERY_RMP); MLX5_COMMAND_STR_CASE(CREATE_RQT); MLX5_COMMAND_STR_CASE(MODIFY_RQT); MLX5_COMMAND_STR_CASE(DESTROY_RQT); MLX5_COMMAND_STR_CASE(QUERY_RQT); MLX5_COMMAND_STR_CASE(ACCESS_REG); MLX5_COMMAND_STR_CASE(CREATE_SQ); MLX5_COMMAND_STR_CASE(MODIFY_SQ); MLX5_COMMAND_STR_CASE(DESTROY_SQ); MLX5_COMMAND_STR_CASE(QUERY_SQ); MLX5_COMMAND_STR_CASE(CREATE_RQ); MLX5_COMMAND_STR_CASE(MODIFY_RQ); MLX5_COMMAND_STR_CASE(DESTROY_RQ); MLX5_COMMAND_STR_CASE(QUERY_RQ); MLX5_COMMAND_STR_CASE(CREATE_TIR); MLX5_COMMAND_STR_CASE(MODIFY_TIR); MLX5_COMMAND_STR_CASE(DESTROY_TIR); MLX5_COMMAND_STR_CASE(QUERY_TIR); MLX5_COMMAND_STR_CASE(CREATE_TIS); MLX5_COMMAND_STR_CASE(MODIFY_TIS); MLX5_COMMAND_STR_CASE(DESTROY_TIS); MLX5_COMMAND_STR_CASE(QUERY_TIS); MLX5_COMMAND_STR_CASE(CREATE_FLOW_TABLE); MLX5_COMMAND_STR_CASE(DESTROY_FLOW_TABLE); MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE); MLX5_COMMAND_STR_CASE(CREATE_FLOW_GROUP); MLX5_COMMAND_STR_CASE(DESTROY_FLOW_GROUP); MLX5_COMMAND_STR_CASE(QUERY_FLOW_GROUP); MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ENTRY); MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE_ENTRY); MLX5_COMMAND_STR_CASE(DELETE_FLOW_TABLE_ENTRY); MLX5_COMMAND_STR_CASE(SET_DIAGNOSTICS); MLX5_COMMAND_STR_CASE(QUERY_DIAGNOSTICS); default: return "unknown command opcode"; } } static const char *cmd_status_str(u8 status) { switch (status) { case MLX5_CMD_STAT_OK: return "OK"; case MLX5_CMD_STAT_INT_ERR: return "internal error"; case MLX5_CMD_STAT_BAD_OP_ERR: return "bad operation"; case MLX5_CMD_STAT_BAD_PARAM_ERR: return "bad parameter"; case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: return "bad system state"; case MLX5_CMD_STAT_BAD_RES_ERR: return "bad resource"; case MLX5_CMD_STAT_RES_BUSY: return "resource busy"; case MLX5_CMD_STAT_LIM_ERR: return "limits exceeded"; case MLX5_CMD_STAT_BAD_RES_STATE_ERR: return "bad resource state"; case MLX5_CMD_STAT_IX_ERR: return "bad index"; case MLX5_CMD_STAT_NO_RES_ERR: return "no resources"; case MLX5_CMD_STAT_BAD_INP_LEN_ERR: return "bad input length"; case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: return "bad output length"; case MLX5_CMD_STAT_BAD_QP_STATE_ERR: return "bad QP state"; case MLX5_CMD_STAT_BAD_PKT_ERR: return "bad packet (discarded)"; case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return "bad size too many outstanding CQEs"; default: return "unknown status"; } } static int cmd_status_to_err_helper(u8 status) { switch (status) { case MLX5_CMD_STAT_OK: return 0; case MLX5_CMD_STAT_INT_ERR: return -EIO; case MLX5_CMD_STAT_BAD_OP_ERR: return -EINVAL; case MLX5_CMD_STAT_BAD_PARAM_ERR: return -EINVAL; case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: return -EIO; case MLX5_CMD_STAT_BAD_RES_ERR: return -EINVAL; case MLX5_CMD_STAT_RES_BUSY: return -EBUSY; case MLX5_CMD_STAT_LIM_ERR: return -ENOMEM; case MLX5_CMD_STAT_BAD_RES_STATE_ERR: return -EINVAL; case MLX5_CMD_STAT_IX_ERR: return -EINVAL; case MLX5_CMD_STAT_NO_RES_ERR: return -EAGAIN; case MLX5_CMD_STAT_BAD_INP_LEN_ERR: return -EIO; case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: return -EIO; case MLX5_CMD_STAT_BAD_QP_STATE_ERR: return -EINVAL; case MLX5_CMD_STAT_BAD_PKT_ERR: return -EINVAL; case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return -EINVAL; default: return -EIO; } } void mlx5_cmd_mbox_status(void *out, u8 *status, u32 *syndrome) { *status = MLX5_GET(mbox_out, out, status); *syndrome = MLX5_GET(mbox_out, out, syndrome); } static int mlx5_cmd_check(struct mlx5_core_dev *dev, void *in, void *out) { u32 syndrome; u8 status; u16 opcode; u16 op_mod; mlx5_cmd_mbox_status(out, &status, &syndrome); if (!status) return 0; opcode = MLX5_GET(mbox_in, in, opcode); op_mod = MLX5_GET(mbox_in, in, op_mod); mlx5_core_err(dev, "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x)\n", mlx5_command_str(opcode), opcode, op_mod, cmd_status_str(status), status, syndrome); return cmd_status_to_err_helper(status); } static void dump_command(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent, int input) { struct mlx5_cmd_msg *msg = input ? ent->in : ent->out; u16 op = MLX5_GET(mbox_in, ent->lay->in, opcode); size_t i; int data_only; int offset = 0; int msg_len = input ? ent->uin_size : ent->uout_size; int dump_len; data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA)); if (data_only) mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA, "dump command data %s(0x%x) %s\n", mlx5_command_str(op), op, input ? "INPUT" : "OUTPUT"); else mlx5_core_dbg(dev, "dump command %s(0x%x) %s\n", mlx5_command_str(op), op, input ? "INPUT" : "OUTPUT"); if (data_only) { if (input) { dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset); offset += sizeof(ent->lay->in); } else { dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset); offset += sizeof(ent->lay->out); } } else { dump_buf(ent->lay, sizeof(*ent->lay), 0, offset); offset += sizeof(*ent->lay); } for (i = 0; i != (msg->numpages * MLX5_NUM_CMDS_IN_ADAPTER_PAGE); i++) { struct mlx5_cmd_prot_block *block; block = mlx5_fwp_get_virt(msg, i * MLX5_CMD_MBOX_SIZE); if (data_only) { if (offset >= msg_len) break; dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg_len - offset); dump_buf(block->data, dump_len, 1, offset); offset += MLX5_CMD_DATA_BLOCK_SIZE; } else { mlx5_core_dbg(dev, "command block:\n"); dump_buf(block, sizeof(*block), 0, offset); offset += sizeof(*block); } /* check for last block */ if (block->next == 0) break; } if (data_only) pr_debug("\n"); } static u16 msg_to_opcode(struct mlx5_cmd_msg *in) { return MLX5_GET(mbox_in, in->first.data, opcode); } static void cb_timeout_handler(struct work_struct *work) { struct delayed_work *dwork = container_of(work, struct delayed_work, work); struct mlx5_cmd_work_ent *ent = container_of(dwork, struct mlx5_cmd_work_ent, cb_timeout_work); struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd); ent->ret = -ETIMEDOUT; mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n", mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); mlx5_cmd_comp_handler(dev, 1UL << ent->idx, MLX5_CMD_MODE_EVENTS); } static void complete_command(struct mlx5_cmd_work_ent *ent) { struct mlx5_cmd *cmd = ent->cmd; struct mlx5_core_dev *dev = container_of(cmd, struct mlx5_core_dev, cmd); mlx5_cmd_cbk_t callback; void *context; s64 ds; struct mlx5_cmd_stats *stats; unsigned long flags; int err; struct semaphore *sem; if (ent->page_queue) sem = &cmd->pages_sem; else sem = &cmd->sem; if (dev->state != MLX5_DEVICE_STATE_UP) { u8 status = 0; u32 drv_synd; ent->ret = mlx5_internal_err_ret_value(dev, msg_to_opcode(ent->in), &drv_synd, &status); MLX5_SET(mbox_out, ent->out, status, status); MLX5_SET(mbox_out, ent->out, syndrome, drv_synd); } if (ent->callback) { ds = ent->ts2 - ent->ts1; if (ent->op < ARRAY_SIZE(cmd->stats)) { stats = &cmd->stats[ent->op]; spin_lock_irqsave(&stats->lock, flags); stats->sum += ds; ++stats->n; spin_unlock_irqrestore(&stats->lock, flags); } callback = ent->callback; context = ent->context; err = ent->ret; if (!err) { err = mlx5_copy_from_msg(ent->uout, ent->out, ent->uout_size); err = err ? err : mlx5_cmd_check(dev, ent->in->first.data, ent->uout); } mlx5_free_cmd_msg(dev, ent->out); free_msg(dev, ent->in); err = err ? err : ent->status; free_cmd(ent); callback(err, context); } else { complete(&ent->done); } up(sem); } static void cmd_work_handler(struct work_struct *work) { struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work); struct mlx5_cmd *cmd = ent->cmd; struct mlx5_core_dev *dev = container_of(cmd, struct mlx5_core_dev, cmd); unsigned long cb_timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC); struct mlx5_cmd_layout *lay; struct semaphore *sem; bool poll_cmd = ent->polling; sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem; down(sem); if (alloc_ent(ent) < 0) { complete_command(ent); return; } ent->token = alloc_token(cmd); lay = get_inst(cmd, ent->idx); ent->lay = lay; memset(lay, 0, sizeof(*lay)); memcpy(lay->in, ent->in->first.data, sizeof(lay->in)); ent->op = be32_to_cpu(lay->in[0]) >> 16; if (ent->in->numpages != 0) lay->in_ptr = cpu_to_be64(mlx5_fwp_get_dma(ent->in, 0)); if (ent->out->numpages != 0) lay->out_ptr = cpu_to_be64(mlx5_fwp_get_dma(ent->out, 0)); lay->inlen = cpu_to_be32(ent->uin_size); lay->outlen = cpu_to_be32(ent->uout_size); lay->type = MLX5_PCI_CMD_XPORT; lay->token = ent->token; lay->status_own = CMD_OWNER_HW; set_signature(ent, !cmd->checksum_disabled); dump_command(dev, ent, 1); ent->ts1 = ktime_get_ns(); ent->busy = 0; if (ent->callback) schedule_delayed_work(&ent->cb_timeout_work, cb_timeout); /* ring doorbell after the descriptor is valid */ mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx); /* make sure data is written to RAM */ mlx5_fwp_flush(cmd->cmd_page); iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell); mmiowb(); /* if not in polling don't use ent after this point */ if (poll_cmd) { poll_timeout(ent); /* make sure we read the descriptor after ownership is SW */ mlx5_cmd_comp_handler(dev, 1U << ent->idx, MLX5_CMD_MODE_POLLING); } } static const char *deliv_status_to_str(u8 status) { switch (status) { case MLX5_CMD_DELIVERY_STAT_OK: return "no errors"; case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR: return "signature error"; case MLX5_CMD_DELIVERY_STAT_TOK_ERR: return "token error"; case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR: return "bad block number"; case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR: return "output pointer not aligned to block size"; case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR: return "input pointer not aligned to block size"; case MLX5_CMD_DELIVERY_STAT_FW_ERR: return "firmware internal error"; case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR: return "command input length error"; case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR: return "command ouput length error"; case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR: return "reserved fields not cleared"; case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR: return "bad command descriptor type"; default: return "unknown status code"; } } static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent) { int timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC); int err; if (ent->polling) { wait_for_completion(&ent->done); } else if (!wait_for_completion_timeout(&ent->done, timeout)) { ent->ret = -ETIMEDOUT; mlx5_cmd_comp_handler(dev, 1UL << ent->idx, MLX5_CMD_MODE_EVENTS); } err = ent->ret; if (err == -ETIMEDOUT) { mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n", mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); } mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n", err, deliv_status_to_str(ent->status), ent->status); return err; } /* Notes: * 1. Callback functions may not sleep * 2. page queue commands do not support asynchrous completion */ static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in, int uin_size, struct mlx5_cmd_msg *out, void *uout, int uout_size, mlx5_cmd_cbk_t callback, void *context, int page_queue, u8 *status, bool force_polling) { struct mlx5_cmd *cmd = &dev->cmd; struct mlx5_cmd_work_ent *ent; struct mlx5_cmd_stats *stats; int err = 0; s64 ds; u16 op; if (callback && page_queue) return -EINVAL; ent = alloc_cmd(cmd, in, uin_size, out, uout, uout_size, callback, context, page_queue); if (IS_ERR(ent)) return PTR_ERR(ent); ent->polling = force_polling || (cmd->mode == MLX5_CMD_MODE_POLLING); if (!callback) init_completion(&ent->done); INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler); INIT_WORK(&ent->work, cmd_work_handler); if (page_queue) { cmd_work_handler(&ent->work); } else if (!queue_work(dev->priv.health.wq_cmd, &ent->work)) { mlx5_core_warn(dev, "failed to queue work\n"); err = -ENOMEM; goto out_free; } if (callback) goto out; err = wait_func(dev, ent); if (err == -ETIMEDOUT) goto out; ds = ent->ts2 - ent->ts1; op = MLX5_GET(mbox_in, in->first.data, opcode); if (op < ARRAY_SIZE(cmd->stats)) { stats = &cmd->stats[op]; spin_lock_irq(&stats->lock); stats->sum += ds; ++stats->n; spin_unlock_irq(&stats->lock); } mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME, "fw exec time for %s is %lld nsec\n", mlx5_command_str(op), (long long)ds); *status = ent->status; free_cmd(ent); return err; out_free: free_cmd(ent); out: return err; } static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, size_t size) { size_t delta; size_t i; if (to == NULL || from == NULL) return (-ENOMEM); delta = min_t(size_t, size, sizeof(to->first.data)); memcpy(to->first.data, from, delta); from = (char *)from + delta; size -= delta; for (i = 0; size != 0; i++) { struct mlx5_cmd_prot_block *block; block = mlx5_fwp_get_virt(to, i * MLX5_CMD_MBOX_SIZE); delta = min_t(size_t, size, MLX5_CMD_DATA_BLOCK_SIZE); memcpy(block->data, from, delta); from = (char *)from + delta; size -= delta; } return (0); } static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size) { size_t delta; size_t i; if (to == NULL || from == NULL) return (-ENOMEM); delta = min_t(size_t, size, sizeof(from->first.data)); memcpy(to, from->first.data, delta); to = (char *)to + delta; size -= delta; for (i = 0; size != 0; i++) { struct mlx5_cmd_prot_block *block; block = mlx5_fwp_get_virt(from, i * MLX5_CMD_MBOX_SIZE); delta = min_t(size_t, size, MLX5_CMD_DATA_BLOCK_SIZE); memcpy(to, block->data, delta); to = (char *)to + delta; size -= delta; } return (0); } static struct mlx5_cmd_msg * mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev, gfp_t flags, size_t size) { struct mlx5_cmd_msg *msg; size_t blen; size_t n; size_t i; blen = size - min_t(size_t, sizeof(msg->first.data), size); n = howmany(blen, MLX5_CMD_DATA_BLOCK_SIZE); msg = mlx5_fwp_alloc(dev, flags, howmany(n, MLX5_NUM_CMDS_IN_ADAPTER_PAGE)); if (msg == NULL) return (ERR_PTR(-ENOMEM)); for (i = 0; i != n; i++) { struct mlx5_cmd_prot_block *block; block = mlx5_fwp_get_virt(msg, i * MLX5_CMD_MBOX_SIZE); memset(block, 0, MLX5_CMD_MBOX_SIZE); if (i != (n - 1)) { u64 dma = mlx5_fwp_get_dma(msg, (i + 1) * MLX5_CMD_MBOX_SIZE); block->next = cpu_to_be64(dma); } block->block_num = cpu_to_be32(i); } /* make sure initial data is written to RAM */ mlx5_fwp_flush(msg); return (msg); } static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg) { mlx5_fwp_free(msg); } static void clean_debug_files(struct mlx5_core_dev *dev) { } static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode) { struct mlx5_cmd *cmd = &dev->cmd; int i; if (cmd->mode == mode) return; for (i = 0; i < cmd->max_reg_cmds; i++) down(&cmd->sem); down(&cmd->pages_sem); cmd->mode = mode; up(&cmd->pages_sem); for (i = 0; i < cmd->max_reg_cmds; i++) up(&cmd->sem); } void mlx5_cmd_use_events(struct mlx5_core_dev *dev) { mlx5_cmd_change_mod(dev, MLX5_CMD_MODE_EVENTS); } void mlx5_cmd_use_polling(struct mlx5_core_dev *dev) { mlx5_cmd_change_mod(dev, MLX5_CMD_MODE_POLLING); } static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg) { unsigned long flags; if (msg->cache) { spin_lock_irqsave(&msg->cache->lock, flags); list_add_tail(&msg->list, &msg->cache->head); spin_unlock_irqrestore(&msg->cache->lock, flags); } else { mlx5_free_cmd_msg(dev, msg); } } void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vector_flags, enum mlx5_cmd_mode cmd_mode) { struct mlx5_cmd *cmd = &dev->cmd; struct mlx5_cmd_work_ent *ent; bool triggered = (vector_flags & MLX5_TRIGGERED_CMD_COMP) ? 1 : 0; u32 vector = vector_flags; /* discard flags in the upper dword */ int i; /* make sure data gets read from RAM */ mlx5_fwp_invalidate(cmd->cmd_page); while (vector != 0) { i = ffs(vector) - 1; vector &= ~(1U << i); /* check command mode */ if (cmd->ent_mode[i] != cmd_mode) continue; ent = cmd->ent_arr[i]; /* check if command was already handled */ if (ent == NULL) continue; if (ent->callback) cancel_delayed_work(&ent->cb_timeout_work); ent->ts2 = ktime_get_ns(); memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out)); /* make sure data gets read from RAM */ mlx5_fwp_invalidate(ent->out); dump_command(dev, ent, 0); if (!ent->ret) { if (!cmd->checksum_disabled) ent->ret = verify_signature(ent); else ent->ret = 0; if (triggered) ent->status = MLX5_DRIVER_STATUS_ABORTED; else ent->status = ent->lay->status_own >> 1; mlx5_core_dbg(dev, "FW command ret 0x%x, status %s(0x%x)\n", ent->ret, deliv_status_to_str(ent->status), ent->status); } free_ent(cmd, ent->idx); complete_command(ent); } } EXPORT_SYMBOL(mlx5_cmd_comp_handler); static int status_to_err(u8 status) { return status ? -EIO : 0; /* TBD more meaningful codes */ } static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size, gfp_t gfp) { struct mlx5_cmd_msg *msg = ERR_PTR(-ENOMEM); struct mlx5_cmd *cmd = &dev->cmd; struct cache_ent *ent = NULL; if (in_size > MED_LIST_SIZE && in_size <= LONG_LIST_SIZE) ent = &cmd->cache.large; else if (in_size > 16 && in_size <= MED_LIST_SIZE) ent = &cmd->cache.med; if (ent) { spin_lock_irq(&ent->lock); if (!list_empty(&ent->head)) { msg = list_entry(ent->head.next, struct mlx5_cmd_msg, list); list_del(&msg->list); } spin_unlock_irq(&ent->lock); } if (IS_ERR(msg)) msg = mlx5_alloc_cmd_msg(dev, gfp, in_size); return msg; } static int is_manage_pages(void *in) { return MLX5_GET(mbox_in, in, opcode) == MLX5_CMD_OP_MANAGE_PAGES; } static int cmd_exec_helper(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size, mlx5_cmd_cbk_t callback, void *context, bool force_polling) { struct mlx5_cmd_msg *inb; struct mlx5_cmd_msg *outb; int pages_queue; const gfp_t gfp = GFP_KERNEL; int err; u8 status = 0; u32 drv_synd; if (pci_channel_offline(dev->pdev) || dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { u16 opcode = MLX5_GET(mbox_in, in, opcode); err = mlx5_internal_err_ret_value(dev, opcode, &drv_synd, &status); MLX5_SET(mbox_out, out, status, status); MLX5_SET(mbox_out, out, syndrome, drv_synd); return err; } pages_queue = is_manage_pages(in); inb = alloc_msg(dev, in_size, gfp); if (IS_ERR(inb)) { err = PTR_ERR(inb); return err; } err = mlx5_copy_to_msg(inb, in, in_size); if (err) { mlx5_core_warn(dev, "err %d\n", err); goto out_in; } outb = mlx5_alloc_cmd_msg(dev, gfp, out_size); if (IS_ERR(outb)) { err = PTR_ERR(outb); goto out_in; } err = mlx5_cmd_invoke(dev, inb, in_size, outb, out, out_size, callback, context, pages_queue, &status, force_polling); if (err) { if (err == -ETIMEDOUT) return err; goto out_out; } mlx5_core_dbg(dev, "err %d, status %d\n", err, status); if (status) { err = status_to_err(status); goto out_out; } if (callback) return err; err = mlx5_copy_from_msg(out, outb, out_size); out_out: mlx5_free_cmd_msg(dev, outb); out_in: free_msg(dev, inb); return err; } int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size) { int err; err = cmd_exec_helper(dev, in, in_size, out, out_size, NULL, NULL, false); return err ? : mlx5_cmd_check(dev, in, out); } EXPORT_SYMBOL(mlx5_cmd_exec); int mlx5_cmd_exec_cb(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size, mlx5_cmd_cbk_t callback, void *context) { return cmd_exec_helper(dev, in, in_size, out, out_size, callback, context, false); } EXPORT_SYMBOL(mlx5_cmd_exec_cb); int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size) { int err; err = cmd_exec_helper(dev, in, in_size, out, out_size, NULL, NULL, true); return err ? : mlx5_cmd_check(dev, in, out); } EXPORT_SYMBOL(mlx5_cmd_exec_polling); static void destroy_msg_cache(struct mlx5_core_dev *dev) { struct mlx5_cmd *cmd = &dev->cmd; struct mlx5_cmd_msg *msg; struct mlx5_cmd_msg *n; list_for_each_entry_safe(msg, n, &cmd->cache.large.head, list) { list_del(&msg->list); mlx5_free_cmd_msg(dev, msg); } list_for_each_entry_safe(msg, n, &cmd->cache.med.head, list) { list_del(&msg->list); mlx5_free_cmd_msg(dev, msg); } } static int create_msg_cache(struct mlx5_core_dev *dev) { struct mlx5_cmd *cmd = &dev->cmd; struct mlx5_cmd_msg *msg; int err; int i; spin_lock_init(&cmd->cache.large.lock); INIT_LIST_HEAD(&cmd->cache.large.head); spin_lock_init(&cmd->cache.med.lock); INIT_LIST_HEAD(&cmd->cache.med.head); for (i = 0; i < NUM_LONG_LISTS; i++) { msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL, LONG_LIST_SIZE); if (IS_ERR(msg)) { err = PTR_ERR(msg); goto ex_err; } msg->cache = &cmd->cache.large; list_add_tail(&msg->list, &cmd->cache.large.head); } for (i = 0; i < NUM_MED_LISTS; i++) { msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL, MED_LIST_SIZE); if (IS_ERR(msg)) { err = PTR_ERR(msg); goto ex_err; } msg->cache = &cmd->cache.med; list_add_tail(&msg->list, &cmd->cache.med.head); } return 0; ex_err: destroy_msg_cache(dev); return err; } static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) { int err; sx_init(&cmd->dma_sx, "MLX5-DMA-SX"); mtx_init(&cmd->dma_mtx, "MLX5-DMA-MTX", NULL, MTX_DEF); cv_init(&cmd->dma_cv, "MLX5-DMA-CV"); /* * Create global DMA descriptor tag for allocating * 4K firmware pages: */ err = -bus_dma_tag_create( bus_get_dma_tag(dev->pdev->dev.bsddev), MLX5_ADAPTER_PAGE_SIZE, /* alignment */ 0, /* no boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ MLX5_ADAPTER_PAGE_SIZE, /* maxsize */ 1, /* nsegments */ MLX5_ADAPTER_PAGE_SIZE, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &cmd->dma_tag); if (err != 0) goto failure_destroy_sx; cmd->cmd_page = mlx5_fwp_alloc(dev, GFP_KERNEL, 1); if (cmd->cmd_page == NULL) { err = -ENOMEM; goto failure_alloc_page; } cmd->dma = mlx5_fwp_get_dma(cmd->cmd_page, 0); cmd->cmd_buf = mlx5_fwp_get_virt(cmd->cmd_page, 0); return (0); failure_alloc_page: bus_dma_tag_destroy(cmd->dma_tag); failure_destroy_sx: cv_destroy(&cmd->dma_cv); mtx_destroy(&cmd->dma_mtx); sx_destroy(&cmd->dma_sx); return (err); } static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) { mlx5_fwp_free(cmd->cmd_page); bus_dma_tag_destroy(cmd->dma_tag); cv_destroy(&cmd->dma_cv); mtx_destroy(&cmd->dma_mtx); sx_destroy(&cmd->dma_sx); } int mlx5_cmd_init(struct mlx5_core_dev *dev) { struct mlx5_cmd *cmd = &dev->cmd; u32 cmd_h, cmd_l; u16 cmd_if_rev; int err; int i; memset(cmd, 0, sizeof(*cmd)); cmd_if_rev = cmdif_rev_get(dev); if (cmd_if_rev != CMD_IF_REV) { - device_printf((&dev->pdev->dev)->bsddev, "ERR: ""Driver cmdif rev(%d) differs from firmware's(%d)\n", CMD_IF_REV, cmd_if_rev); + mlx5_core_err(dev, + "Driver cmdif rev(%d) differs from firmware's(%d)\n", + CMD_IF_REV, cmd_if_rev); return -EINVAL; } err = alloc_cmd_page(dev, cmd); if (err) goto err_free_pool; cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff; cmd->log_sz = cmd_l >> 4 & 0xf; cmd->log_stride = cmd_l & 0xf; if (1 << cmd->log_sz > MLX5_MAX_COMMANDS) { - device_printf((&dev->pdev->dev)->bsddev, "ERR: ""firmware reports too many outstanding commands %d\n", 1 << cmd->log_sz); + mlx5_core_err(dev, + "firmware reports too many outstanding commands %d\n", + 1 << cmd->log_sz); err = -EINVAL; goto err_free_page; } if (cmd->log_sz + cmd->log_stride > MLX5_ADAPTER_PAGE_SHIFT) { - device_printf((&dev->pdev->dev)->bsddev, "ERR: ""command queue size overflow\n"); + mlx5_core_err(dev, + "command queue size overflow\n"); err = -EINVAL; goto err_free_page; } cmd->checksum_disabled = 1; cmd->max_reg_cmds = (1 << cmd->log_sz) - 1; cmd->bitmask = (1 << cmd->max_reg_cmds) - 1; cmd->cmdif_rev = ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16; if (cmd->cmdif_rev > CMD_IF_REV) { - device_printf((&dev->pdev->dev)->bsddev, "ERR: ""driver does not support command interface version. driver %d, firmware %d\n", CMD_IF_REV, cmd->cmdif_rev); + mlx5_core_err(dev, + "driver does not support command interface version. driver %d, firmware %d\n", + CMD_IF_REV, cmd->cmdif_rev); err = -ENOTSUPP; goto err_free_page; } spin_lock_init(&cmd->alloc_lock); spin_lock_init(&cmd->token_lock); for (i = 0; i < ARRAY_SIZE(cmd->stats); i++) spin_lock_init(&cmd->stats[i].lock); sema_init(&cmd->sem, cmd->max_reg_cmds); sema_init(&cmd->pages_sem, 1); cmd_h = (u32)((u64)(cmd->dma) >> 32); cmd_l = (u32)(cmd->dma); if (cmd_l & 0xfff) { - device_printf((&dev->pdev->dev)->bsddev, "ERR: ""invalid command queue address\n"); + mlx5_core_err(dev, "invalid command queue address\n"); err = -ENOMEM; goto err_free_page; } iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h); iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz); /* Make sure firmware sees the complete address before we proceed */ wmb(); mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma)); cmd->mode = MLX5_CMD_MODE_POLLING; err = create_msg_cache(dev); if (err) { - device_printf((&dev->pdev->dev)->bsddev, "ERR: ""failed to create command cache\n"); + mlx5_core_err(dev, "failed to create command cache\n"); goto err_free_page; } return 0; err_free_page: free_cmd_page(dev, cmd); err_free_pool: return err; } EXPORT_SYMBOL(mlx5_cmd_init); void mlx5_cmd_cleanup(struct mlx5_core_dev *dev) { struct mlx5_cmd *cmd = &dev->cmd; clean_debug_files(dev); flush_workqueue(dev->priv.health.wq_cmd); destroy_msg_cache(dev); free_cmd_page(dev, cmd); } EXPORT_SYMBOL(mlx5_cmd_cleanup); int mlx5_cmd_query_cong_counter(struct mlx5_core_dev *dev, bool reset, void *out, int out_size) { u32 in[MLX5_ST_SZ_DW(query_cong_statistics_in)] = { }; MLX5_SET(query_cong_statistics_in, in, opcode, MLX5_CMD_OP_QUERY_CONG_STATISTICS); MLX5_SET(query_cong_statistics_in, in, clear, reset); return mlx5_cmd_exec(dev, in, sizeof(in), out, out_size); } EXPORT_SYMBOL(mlx5_cmd_query_cong_counter); int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point, void *out, int out_size) { u32 in[MLX5_ST_SZ_DW(query_cong_params_in)] = { }; MLX5_SET(query_cong_params_in, in, opcode, MLX5_CMD_OP_QUERY_CONG_PARAMS); MLX5_SET(query_cong_params_in, in, cong_protocol, cong_point); return mlx5_cmd_exec(dev, in, sizeof(in), out, out_size); } EXPORT_SYMBOL(mlx5_cmd_query_cong_params); int mlx5_cmd_modify_cong_params(struct mlx5_core_dev *dev, void *in, int in_size) { u32 out[MLX5_ST_SZ_DW(modify_cong_params_out)] = { }; return mlx5_cmd_exec(dev, in, in_size, out, sizeof(out)); } EXPORT_SYMBOL(mlx5_cmd_modify_cong_params); Index: stable/11/sys/dev/mlx5/mlx5_core/mlx5_core.h =================================================================== --- stable/11/sys/dev/mlx5/mlx5_core/mlx5_core.h (revision 353223) +++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_core.h (revision 353224) @@ -1,131 +1,136 @@ /*- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __MLX5_CORE_H__ #define __MLX5_CORE_H__ #include #include #include #include #define DRIVER_NAME "mlx5_core" #ifndef DRIVER_VERSION #define DRIVER_VERSION "3.5.1" #endif #define DRIVER_RELDATE "April 2019" extern int mlx5_core_debug_mask; #define mlx5_core_dbg(dev, format, ...) \ pr_debug("%s:%s:%d:(pid %d): " format, \ (dev)->priv.name, __func__, __LINE__, curthread->td_proc->p_pid, \ ##__VA_ARGS__) #define mlx5_core_dbg_mask(dev, mask, format, ...) \ do { \ if ((mask) & mlx5_core_debug_mask) \ mlx5_core_dbg(dev, format, ##__VA_ARGS__); \ } while (0) -#define mlx5_core_err(_dev, format, ...) \ - device_printf((&(_dev)->pdev->dev)->bsddev, "ERR: ""%s:%d:(pid %d): " format, \ +#define mlx5_core_err(_dev, format, ...) \ + device_printf((_dev)->pdev->dev.bsddev, "ERR: ""%s:%d:(pid %d): " format, \ __func__, __LINE__, curthread->td_proc->p_pid, \ ##__VA_ARGS__) -#define mlx5_core_warn(_dev, format, ...) \ - device_printf((&(_dev)->pdev->dev)->bsddev, "WARN: ""%s:%d:(pid %d): " format, \ +#define mlx5_core_warn(_dev, format, ...) \ + device_printf((_dev)->pdev->dev.bsddev, "WARN: ""%s:%d:(pid %d): " format, \ + __func__, __LINE__, curthread->td_proc->p_pid, \ + ##__VA_ARGS__) + +#define mlx5_core_info(_dev, format, ...) \ + device_printf((_dev)->pdev->dev.bsddev, "INFO: ""%s:%d:(pid %d): " format, \ __func__, __LINE__, curthread->td_proc->p_pid, \ ##__VA_ARGS__) enum { MLX5_CMD_DATA, /* print command payload only */ MLX5_CMD_TIME, /* print command execution time */ }; enum mlx5_semaphore_space_address { MLX5_SEMAPHORE_SW_RESET = 0x20, }; struct mlx5_core_dev; int mlx5_query_hca_caps(struct mlx5_core_dev *dev); int mlx5_query_board_id(struct mlx5_core_dev *dev); int mlx5_query_qcam_reg(struct mlx5_core_dev *mdev, u32 *qcam, u8 feature_group, u8 access_reg_group); int mlx5_query_pcam_reg(struct mlx5_core_dev *dev, u32 *pcam, u8 feature_group, u8 access_reg_group); int mlx5_query_mcam_reg(struct mlx5_core_dev *dev, u32 *mcap, u8 feature_group, u8 access_reg_group); int mlx5_query_mfrl_reg(struct mlx5_core_dev *mdev, u8 *reset_level); int mlx5_set_mfrl_reg(struct mlx5_core_dev *mdev, u8 reset_level); int mlx5_cmd_init_hca(struct mlx5_core_dev *dev); int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev); int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *dev); int mlx5_cmd_fast_teardown_hca(struct mlx5_core_dev *dev); void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event, unsigned long param); void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force); void mlx5_disable_device(struct mlx5_core_dev *dev); void mlx5_recover_device(struct mlx5_core_dev *dev); int mlx5_register_device(struct mlx5_core_dev *dev); void mlx5_unregister_device(struct mlx5_core_dev *dev); int mlx5_firmware_flash(struct mlx5_core_dev *dev, const struct firmware *fw); void mlx5e_init(void); void mlx5e_cleanup(void); int mlx5_ctl_init(void); void mlx5_ctl_fini(void); void mlx5_fwdump_prep(struct mlx5_core_dev *mdev); void mlx5_fwdump(struct mlx5_core_dev *mdev); void mlx5_fwdump_clean(struct mlx5_core_dev *mdev); struct mlx5_crspace_regmap { uint32_t addr; unsigned cnt; }; extern struct pci_driver mlx5_core_driver; SYSCTL_DECL(_hw_mlx5); enum { MLX5_NIC_IFC_FULL = 0, MLX5_NIC_IFC_DISABLED = 1, MLX5_NIC_IFC_NO_DRAM_NIC = 2, MLX5_NIC_IFC_INVALID = 3, MLX5_NIC_IFC_SW_RESET = 7, }; u8 mlx5_get_nic_state(struct mlx5_core_dev *dev); void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state); #endif /* __MLX5_CORE_H__ */ Index: stable/11/sys/dev/mlx5/mlx5_core/mlx5_eq.c =================================================================== --- stable/11/sys/dev/mlx5/mlx5_core/mlx5_eq.c (revision 353223) +++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_eq.c (revision 353224) @@ -1,741 +1,740 @@ /*- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include #include #include "mlx5_core.h" #include "opt_rss.h" #ifdef RSS #include #include #endif enum { MLX5_EQE_SIZE = sizeof(struct mlx5_eqe), MLX5_EQE_OWNER_INIT_VAL = 0x1, }; enum { MLX5_NUM_SPARE_EQE = 0x80, MLX5_NUM_ASYNC_EQE = 0x100, MLX5_NUM_CMD_EQE = 32, }; enum { MLX5_EQ_DOORBEL_OFFSET = 0x40, }; #define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG) | \ (1ull << MLX5_EVENT_TYPE_COMM_EST) | \ (1ull << MLX5_EVENT_TYPE_SQ_DRAINED) | \ (1ull << MLX5_EVENT_TYPE_CQ_ERROR) | \ (1ull << MLX5_EVENT_TYPE_WQ_CATAS_ERROR) | \ (1ull << MLX5_EVENT_TYPE_PATH_MIG_FAILED) | \ (1ull << MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \ (1ull << MLX5_EVENT_TYPE_WQ_ACCESS_ERROR) | \ (1ull << MLX5_EVENT_TYPE_PORT_CHANGE) | \ (1ull << MLX5_EVENT_TYPE_SRQ_CATAS_ERROR) | \ (1ull << MLX5_EVENT_TYPE_SRQ_LAST_WQE) | \ (1ull << MLX5_EVENT_TYPE_SRQ_RQ_LIMIT)) struct map_eq_in { u64 mask; u32 reserved; u32 unmap_eqn; }; struct cre_des_eq { u8 reserved[15]; u8 eqn; }; /*Function prototype*/ static void mlx5_port_module_event(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe); static void mlx5_port_general_notification_event(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe); static int mlx5_cmd_destroy_eq(struct mlx5_core_dev *dev, u8 eqn) { u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {0}; u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {0}; MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ); MLX5_SET(destroy_eq_in, in, eq_number, eqn); return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } static struct mlx5_eqe *get_eqe(struct mlx5_eq *eq, u32 entry) { return mlx5_buf_offset(&eq->buf, entry * MLX5_EQE_SIZE); } static struct mlx5_eqe *next_eqe_sw(struct mlx5_eq *eq) { struct mlx5_eqe *eqe = get_eqe(eq, eq->cons_index & (eq->nent - 1)); return ((eqe->owner & 1) ^ !!(eq->cons_index & eq->nent)) ? NULL : eqe; } static const char *eqe_type_str(u8 type) { switch (type) { case MLX5_EVENT_TYPE_COMP: return "MLX5_EVENT_TYPE_COMP"; case MLX5_EVENT_TYPE_PATH_MIG: return "MLX5_EVENT_TYPE_PATH_MIG"; case MLX5_EVENT_TYPE_COMM_EST: return "MLX5_EVENT_TYPE_COMM_EST"; case MLX5_EVENT_TYPE_SQ_DRAINED: return "MLX5_EVENT_TYPE_SQ_DRAINED"; case MLX5_EVENT_TYPE_SRQ_LAST_WQE: return "MLX5_EVENT_TYPE_SRQ_LAST_WQE"; case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: return "MLX5_EVENT_TYPE_SRQ_RQ_LIMIT"; case MLX5_EVENT_TYPE_CQ_ERROR: return "MLX5_EVENT_TYPE_CQ_ERROR"; case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: return "MLX5_EVENT_TYPE_WQ_CATAS_ERROR"; case MLX5_EVENT_TYPE_PATH_MIG_FAILED: return "MLX5_EVENT_TYPE_PATH_MIG_FAILED"; case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: return "MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR"; case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: return "MLX5_EVENT_TYPE_WQ_ACCESS_ERROR"; case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: return "MLX5_EVENT_TYPE_SRQ_CATAS_ERROR"; case MLX5_EVENT_TYPE_INTERNAL_ERROR: return "MLX5_EVENT_TYPE_INTERNAL_ERROR"; case MLX5_EVENT_TYPE_PORT_CHANGE: return "MLX5_EVENT_TYPE_PORT_CHANGE"; case MLX5_EVENT_TYPE_GPIO_EVENT: return "MLX5_EVENT_TYPE_GPIO_EVENT"; case MLX5_EVENT_TYPE_CODING_PORT_MODULE_EVENT: return "MLX5_EVENT_TYPE_PORT_MODULE_EVENT"; case MLX5_EVENT_TYPE_TEMP_WARN_EVENT: return "MLX5_EVENT_TYPE_TEMP_WARN_EVENT"; case MLX5_EVENT_TYPE_REMOTE_CONFIG: return "MLX5_EVENT_TYPE_REMOTE_CONFIG"; case MLX5_EVENT_TYPE_DB_BF_CONGESTION: return "MLX5_EVENT_TYPE_DB_BF_CONGESTION"; case MLX5_EVENT_TYPE_STALL_EVENT: return "MLX5_EVENT_TYPE_STALL_EVENT"; case MLX5_EVENT_TYPE_CMD: return "MLX5_EVENT_TYPE_CMD"; case MLX5_EVENT_TYPE_PAGE_REQUEST: return "MLX5_EVENT_TYPE_PAGE_REQUEST"; case MLX5_EVENT_TYPE_NIC_VPORT_CHANGE: return "MLX5_EVENT_TYPE_NIC_VPORT_CHANGE"; case MLX5_EVENT_TYPE_FPGA_ERROR: return "MLX5_EVENT_TYPE_FPGA_ERROR"; case MLX5_EVENT_TYPE_FPGA_QP_ERROR: return "MLX5_EVENT_TYPE_FPGA_QP_ERROR"; case MLX5_EVENT_TYPE_CODING_DCBX_CHANGE_EVENT: return "MLX5_EVENT_TYPE_CODING_DCBX_CHANGE_EVENT"; case MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT: return "MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT"; default: return "Unrecognized event"; } } static enum mlx5_dev_event port_subtype_event(u8 subtype) { switch (subtype) { case MLX5_PORT_CHANGE_SUBTYPE_DOWN: return MLX5_DEV_EVENT_PORT_DOWN; case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: return MLX5_DEV_EVENT_PORT_UP; case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED: return MLX5_DEV_EVENT_PORT_INITIALIZED; case MLX5_PORT_CHANGE_SUBTYPE_LID: return MLX5_DEV_EVENT_LID_CHANGE; case MLX5_PORT_CHANGE_SUBTYPE_PKEY: return MLX5_DEV_EVENT_PKEY_CHANGE; case MLX5_PORT_CHANGE_SUBTYPE_GUID: return MLX5_DEV_EVENT_GUID_CHANGE; case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG: return MLX5_DEV_EVENT_CLIENT_REREG; } return -1; } static enum mlx5_dev_event dcbx_subevent(u8 subtype) { switch (subtype) { case MLX5_DCBX_EVENT_SUBTYPE_ERROR_STATE_DCBX: return MLX5_DEV_EVENT_ERROR_STATE_DCBX; case MLX5_DCBX_EVENT_SUBTYPE_REMOTE_CONFIG_CHANGE: return MLX5_DEV_EVENT_REMOTE_CONFIG_CHANGE; case MLX5_DCBX_EVENT_SUBTYPE_LOCAL_OPER_CHANGE: return MLX5_DEV_EVENT_LOCAL_OPER_CHANGE; case MLX5_DCBX_EVENT_SUBTYPE_REMOTE_CONFIG_APP_PRIORITY_CHANGE: return MLX5_DEV_EVENT_REMOTE_CONFIG_APPLICATION_PRIORITY_CHANGE; } return -1; } static void eq_update_ci(struct mlx5_eq *eq, int arm) { __be32 __iomem *addr = eq->doorbell + (arm ? 0 : 2); u32 val = (eq->cons_index & 0xffffff) | (eq->eqn << 24); __raw_writel((__force u32) cpu_to_be32(val), addr); /* We still want ordering, just not swabbing, so add a barrier */ mb(); } static void mlx5_temp_warning_event(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe) { mlx5_core_warn(dev, "High temperature on sensors with bit set %#jx %#jx\n", (uintmax_t)be64_to_cpu(eqe->data.temp_warning.sensor_warning_msb), (uintmax_t)be64_to_cpu(eqe->data.temp_warning.sensor_warning_lsb)); } static int mlx5_eq_int(struct mlx5_core_dev *dev, struct mlx5_eq *eq) { struct mlx5_eqe *eqe; int eqes_found = 0; int set_ci = 0; u32 cqn; u32 rsn; u8 port; while ((eqe = next_eqe_sw(eq))) { /* * Make sure we read EQ entry contents after we've * checked the ownership bit. */ rmb(); mlx5_core_dbg(eq->dev, "eqn %d, eqe type %s\n", eq->eqn, eqe_type_str(eqe->type)); switch (eqe->type) { case MLX5_EVENT_TYPE_COMP: cqn = be32_to_cpu(eqe->data.comp.cqn) & 0xffffff; mlx5_cq_completion(dev, cqn); break; case MLX5_EVENT_TYPE_PATH_MIG: case MLX5_EVENT_TYPE_COMM_EST: case MLX5_EVENT_TYPE_SQ_DRAINED: case MLX5_EVENT_TYPE_SRQ_LAST_WQE: case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: case MLX5_EVENT_TYPE_PATH_MIG_FAILED: case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff; mlx5_core_dbg(dev, "event %s(%d) arrived on resource 0x%x\n", eqe_type_str(eqe->type), eqe->type, rsn); mlx5_rsc_event(dev, rsn, eqe->type); break; case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff; mlx5_core_dbg(dev, "SRQ event %s(%d): srqn 0x%x\n", eqe_type_str(eqe->type), eqe->type, rsn); mlx5_srq_event(dev, rsn, eqe->type); break; case MLX5_EVENT_TYPE_CMD: if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) { mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector), MLX5_CMD_MODE_EVENTS); } break; case MLX5_EVENT_TYPE_PORT_CHANGE: port = (eqe->data.port.port >> 4) & 0xf; switch (eqe->sub_type) { case MLX5_PORT_CHANGE_SUBTYPE_DOWN: case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: case MLX5_PORT_CHANGE_SUBTYPE_LID: case MLX5_PORT_CHANGE_SUBTYPE_PKEY: case MLX5_PORT_CHANGE_SUBTYPE_GUID: case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG: case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED: if (dev->event) dev->event(dev, port_subtype_event(eqe->sub_type), (unsigned long)port); break; default: mlx5_core_warn(dev, "Port event with unrecognized subtype: port %d, sub_type %d\n", port, eqe->sub_type); } break; case MLX5_EVENT_TYPE_CODING_DCBX_CHANGE_EVENT: port = (eqe->data.port.port >> 4) & 0xf; switch (eqe->sub_type) { case MLX5_DCBX_EVENT_SUBTYPE_ERROR_STATE_DCBX: case MLX5_DCBX_EVENT_SUBTYPE_REMOTE_CONFIG_CHANGE: case MLX5_DCBX_EVENT_SUBTYPE_LOCAL_OPER_CHANGE: case MLX5_DCBX_EVENT_SUBTYPE_REMOTE_CONFIG_APP_PRIORITY_CHANGE: if (dev->event) dev->event(dev, dcbx_subevent(eqe->sub_type), 0); break; default: mlx5_core_warn(dev, "dcbx event with unrecognized subtype: port %d, sub_type %d\n", port, eqe->sub_type); } break; case MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT: mlx5_port_general_notification_event(dev, eqe); break; case MLX5_EVENT_TYPE_CQ_ERROR: cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff; mlx5_core_warn(dev, "CQ error on CQN 0x%x, syndrom 0x%x\n", cqn, eqe->data.cq_err.syndrome); mlx5_cq_event(dev, cqn, eqe->type); break; case MLX5_EVENT_TYPE_PAGE_REQUEST: { u16 func_id = be16_to_cpu(eqe->data.req_pages.func_id); s32 npages = be32_to_cpu(eqe->data.req_pages.num_pages); mlx5_core_dbg(dev, "page request for func 0x%x, npages %d\n", func_id, npages); mlx5_core_req_pages_handler(dev, func_id, npages); } break; case MLX5_EVENT_TYPE_CODING_PORT_MODULE_EVENT: mlx5_port_module_event(dev, eqe); break; case MLX5_EVENT_TYPE_NIC_VPORT_CHANGE: { struct mlx5_eqe_vport_change *vc_eqe = &eqe->data.vport_change; u16 vport_num = be16_to_cpu(vc_eqe->vport_num); if (dev->event) dev->event(dev, MLX5_DEV_EVENT_VPORT_CHANGE, (unsigned long)vport_num); } break; case MLX5_EVENT_TYPE_FPGA_ERROR: case MLX5_EVENT_TYPE_FPGA_QP_ERROR: mlx5_fpga_event(dev, eqe->type, &eqe->data.raw); break; case MLX5_EVENT_TYPE_TEMP_WARN_EVENT: mlx5_temp_warning_event(dev, eqe); break; default: mlx5_core_warn(dev, "Unhandled event 0x%x on EQ 0x%x\n", eqe->type, eq->eqn); break; } ++eq->cons_index; eqes_found = 1; ++set_ci; /* The HCA will think the queue has overflowed if we * don't tell it we've been processing events. We * create our EQs with MLX5_NUM_SPARE_EQE extra * entries, so we must update our consumer index at * least that often. */ if (unlikely(set_ci >= MLX5_NUM_SPARE_EQE)) { eq_update_ci(eq, 0); set_ci = 0; } } eq_update_ci(eq, 1); return eqes_found; } static irqreturn_t mlx5_msix_handler(int irq, void *eq_ptr) { struct mlx5_eq *eq = eq_ptr; struct mlx5_core_dev *dev = eq->dev; /* check if IRQs are not disabled */ if (likely(dev->priv.disable_irqs == 0)) mlx5_eq_int(dev, eq); /* MSI-X vectors always belong to us */ return IRQ_HANDLED; } static void init_eq_buf(struct mlx5_eq *eq) { struct mlx5_eqe *eqe; int i; for (i = 0; i < eq->nent; i++) { eqe = get_eqe(eq, i); eqe->owner = MLX5_EQE_OWNER_INIT_VAL; } } int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx, int nent, u64 mask, struct mlx5_uar *uar) { u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0}; struct mlx5_priv *priv = &dev->priv; __be64 *pas; void *eqc; int inlen; u32 *in; int err; eq->nent = roundup_pow_of_two(nent + MLX5_NUM_SPARE_EQE); eq->cons_index = 0; err = mlx5_buf_alloc(dev, eq->nent * MLX5_EQE_SIZE, 2 * PAGE_SIZE, &eq->buf); if (err) return err; init_eq_buf(eq); inlen = MLX5_ST_SZ_BYTES(create_eq_in) + MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->buf.npages; in = mlx5_vzalloc(inlen); if (!in) { err = -ENOMEM; goto err_buf; } pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas); mlx5_fill_page_array(&eq->buf, pas); MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ); MLX5_SET64(create_eq_in, in, event_bitmask, mask); eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry); MLX5_SET(eqc, eqc, log_eq_size, ilog2(eq->nent)); MLX5_SET(eqc, eqc, uar_page, uar->index); MLX5_SET(eqc, eqc, intr, vecidx); MLX5_SET(eqc, eqc, log_page_size, eq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); if (err) goto err_in; eq->eqn = MLX5_GET(create_eq_out, out, eq_number); eq->irqn = vecidx; eq->dev = dev; eq->doorbell = uar->map + MLX5_EQ_DOORBEL_OFFSET; err = request_irq(priv->msix_arr[vecidx].vector, mlx5_msix_handler, 0, "mlx5_core", eq); if (err) goto err_eq; #ifdef RSS if (vecidx >= MLX5_EQ_VEC_COMP_BASE) { u8 bucket = vecidx - MLX5_EQ_VEC_COMP_BASE; err = bind_irq_to_cpu(priv->msix_arr[vecidx].vector, rss_getcpu(bucket % rss_getnumbuckets())); if (err) goto err_irq; } #else if (0) goto err_irq; #endif /* EQs are created in ARMED state */ eq_update_ci(eq, 1); kvfree(in); return 0; err_irq: free_irq(priv->msix_arr[vecidx].vector, eq); err_eq: mlx5_cmd_destroy_eq(dev, eq->eqn); err_in: kvfree(in); err_buf: mlx5_buf_free(dev, &eq->buf); return err; } EXPORT_SYMBOL_GPL(mlx5_create_map_eq); int mlx5_destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq) { int err; free_irq(dev->priv.msix_arr[eq->irqn].vector, eq); err = mlx5_cmd_destroy_eq(dev, eq->eqn); if (err) mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n", eq->eqn); mlx5_buf_free(dev, &eq->buf); return err; } EXPORT_SYMBOL_GPL(mlx5_destroy_unmap_eq); int mlx5_eq_init(struct mlx5_core_dev *dev) { int err; spin_lock_init(&dev->priv.eq_table.lock); err = 0; return err; } void mlx5_eq_cleanup(struct mlx5_core_dev *dev) { } int mlx5_start_eqs(struct mlx5_core_dev *dev) { struct mlx5_eq_table *table = &dev->priv.eq_table; u64 async_event_mask = MLX5_ASYNC_EVENT_MASK; int err; if (MLX5_CAP_GEN(dev, port_module_event)) async_event_mask |= (1ull << MLX5_EVENT_TYPE_CODING_PORT_MODULE_EVENT); if (MLX5_CAP_GEN(dev, nic_vport_change_event)) async_event_mask |= (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE); if (MLX5_CAP_GEN(dev, dcbx)) async_event_mask |= (1ull << MLX5_EVENT_TYPE_CODING_DCBX_CHANGE_EVENT); if (MLX5_CAP_GEN(dev, fpga)) async_event_mask |= (1ull << MLX5_EVENT_TYPE_FPGA_ERROR) | (1ull << MLX5_EVENT_TYPE_FPGA_QP_ERROR); if (MLX5_CAP_GEN(dev, temp_warn_event)) async_event_mask |= (1ull << MLX5_EVENT_TYPE_TEMP_WARN_EVENT); if (MLX5_CAP_GEN(dev, general_notification_event)) { async_event_mask |= (1ull << MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT); } err = mlx5_create_map_eq(dev, &table->cmd_eq, MLX5_EQ_VEC_CMD, MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD, &dev->priv.uuari.uars[0]); if (err) { mlx5_core_warn(dev, "failed to create cmd EQ %d\n", err); return err; } mlx5_cmd_use_events(dev); err = mlx5_create_map_eq(dev, &table->async_eq, MLX5_EQ_VEC_ASYNC, MLX5_NUM_ASYNC_EQE, async_event_mask, &dev->priv.uuari.uars[0]); if (err) { mlx5_core_warn(dev, "failed to create async EQ %d\n", err); goto err1; } err = mlx5_create_map_eq(dev, &table->pages_eq, MLX5_EQ_VEC_PAGES, /* TODO: sriov max_vf + */ 1, 1 << MLX5_EVENT_TYPE_PAGE_REQUEST, &dev->priv.uuari.uars[0]); if (err) { mlx5_core_warn(dev, "failed to create pages EQ %d\n", err); goto err2; } return err; err2: mlx5_destroy_unmap_eq(dev, &table->async_eq); err1: mlx5_cmd_use_polling(dev); mlx5_destroy_unmap_eq(dev, &table->cmd_eq); return err; } int mlx5_stop_eqs(struct mlx5_core_dev *dev) { struct mlx5_eq_table *table = &dev->priv.eq_table; int err; err = mlx5_destroy_unmap_eq(dev, &table->pages_eq); if (err) return err; mlx5_destroy_unmap_eq(dev, &table->async_eq); mlx5_cmd_use_polling(dev); err = mlx5_destroy_unmap_eq(dev, &table->cmd_eq); if (err) mlx5_cmd_use_events(dev); return err; } int mlx5_core_eq_query(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u32 *out, int outlen) { u32 in[MLX5_ST_SZ_DW(query_eq_in)] = {0}; memset(out, 0, outlen); MLX5_SET(query_eq_in, in, opcode, MLX5_CMD_OP_QUERY_EQ); MLX5_SET(query_eq_in, in, eq_number, eq->eqn); return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen); } EXPORT_SYMBOL_GPL(mlx5_core_eq_query); static const char *mlx5_port_module_event_error_type_to_string(u8 error_type) { switch (error_type) { case MLX5_MODULE_EVENT_ERROR_POWER_BUDGET_EXCEEDED: return "Power budget exceeded"; case MLX5_MODULE_EVENT_ERROR_LONG_RANGE_FOR_NON_MLNX_CABLE_MODULE: return "Long Range for non MLNX cable"; case MLX5_MODULE_EVENT_ERROR_BUS_STUCK: return "Bus stuck(I2C or data shorted)"; case MLX5_MODULE_EVENT_ERROR_NO_EEPROM_RETRY_TIMEOUT: return "No EEPROM/retry timeout"; case MLX5_MODULE_EVENT_ERROR_ENFORCE_PART_NUMBER_LIST: return "Enforce part number list"; case MLX5_MODULE_EVENT_ERROR_UNSUPPORTED_CABLE: return "Unknown identifier"; case MLX5_MODULE_EVENT_ERROR_HIGH_TEMPERATURE: return "High Temperature"; case MLX5_MODULE_EVENT_ERROR_CABLE_IS_SHORTED: return "Bad or shorted cable/module"; default: return "Unknown error type"; } } unsigned int mlx5_query_module_status(struct mlx5_core_dev *dev, int module_num) { if (module_num < 0 || module_num >= MLX5_MAX_PORTS) return 0; /* undefined */ return dev->module_status[module_num]; } static void mlx5_port_module_event(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe) { unsigned int module_num; unsigned int module_status; unsigned int error_type; struct mlx5_eqe_port_module_event *module_event_eqe; - struct pci_dev *pdev = dev->pdev; module_event_eqe = &eqe->data.port_module_event; module_num = (unsigned int)module_event_eqe->module; module_status = (unsigned int)module_event_eqe->module_status & PORT_MODULE_EVENT_MODULE_STATUS_MASK; error_type = (unsigned int)module_event_eqe->error_type & PORT_MODULE_EVENT_ERROR_TYPE_MASK; if (module_status < MLX5_MODULE_STATUS_NUM) dev->priv.pme_stats.status_counters[module_status]++; switch (module_status) { case MLX5_MODULE_STATUS_PLUGGED_ENABLED: - device_printf((&pdev->dev)->bsddev, - "INFO: Module %u, status: plugged and enabled\n", + mlx5_core_info(dev, + "Module %u, status: plugged and enabled\n", module_num); break; case MLX5_MODULE_STATUS_UNPLUGGED: - device_printf((&pdev->dev)->bsddev, - "INFO: Module %u, status: unplugged\n", module_num); + mlx5_core_info(dev, + "Module %u, status: unplugged\n", module_num); break; case MLX5_MODULE_STATUS_ERROR: - device_printf((&pdev->dev)->bsddev, - "ERROR: Module %u, status: error, %s\n", + mlx5_core_err(dev, + "Module %u, status: error, %s\n", module_num, mlx5_port_module_event_error_type_to_string(error_type)); if (error_type < MLX5_MODULE_EVENT_ERROR_NUM) dev->priv.pme_stats.error_counters[error_type]++; break; default: - device_printf((&pdev->dev)->bsddev, - "INFO: Module %u, unknown status\n", module_num); + mlx5_core_info(dev, + "Module %u, unknown status\n", module_num); } /* store module status */ if (module_num < MLX5_MAX_PORTS) dev->module_status[module_num] = module_status; } static void mlx5_port_general_notification_event(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe) { u8 port = (eqe->data.port.port >> 4) & 0xf; u32 rqn; struct mlx5_eqe_general_notification_event *general_event; switch (eqe->sub_type) { case MLX5_GEN_EVENT_SUBTYPE_DELAY_DROP_TIMEOUT: general_event = &eqe->data.general_notifications; rqn = be32_to_cpu(general_event->rq_user_index_delay_drop) & 0xffffff; break; case MLX5_GEN_EVENT_SUBTYPE_PCI_POWER_CHANGE_EVENT: mlx5_trigger_health_watchdog(dev); break; default: mlx5_core_warn(dev, "general event with unrecognized subtype: port %d, sub_type %d\n", port, eqe->sub_type); break; } } Index: stable/11/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c =================================================================== --- stable/11/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c (revision 353223) +++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c (revision 353224) @@ -1,2718 +1,2725 @@ /*- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include "mlx5_core.h" #include "fs_core.h" #include #include #define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\ sizeof(struct init_tree_node)) #define ADD_PRIO(name_val, flags_val, min_level_val, max_ft_val, caps_val, \ ...) {.type = FS_TYPE_PRIO,\ .name = name_val,\ .min_ft_level = min_level_val,\ .flags = flags_val,\ .max_ft = max_ft_val,\ .caps = caps_val,\ .children = (struct init_tree_node[]) {__VA_ARGS__},\ .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ } #define ADD_FT_PRIO(name_val, flags_val, max_ft_val, ...)\ ADD_PRIO(name_val, flags_val, 0, max_ft_val, {},\ __VA_ARGS__)\ #define ADD_NS(name_val, ...) {.type = FS_TYPE_NAMESPACE,\ .name = name_val,\ .children = (struct init_tree_node[]) {__VA_ARGS__},\ .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ } #define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\ sizeof(long)) #define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap)) #define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \ .caps = (long[]) {__VA_ARGS__}} #define BYPASS_MAX_FT 5 #define BYPASS_PRIO_MAX_FT 1 #define KERNEL_MAX_FT 3 #define LEFTOVER_MAX_FT 1 #define KENREL_MIN_LEVEL 3 #define LEFTOVER_MIN_LEVEL KENREL_MIN_LEVEL + 1 #define BYPASS_MIN_LEVEL MLX5_NUM_BYPASS_FTS + LEFTOVER_MIN_LEVEL struct node_caps { size_t arr_sz; long *caps; }; struct init_tree_node { enum fs_type type; const char *name; struct init_tree_node *children; int ar_size; struct node_caps caps; u8 flags; int min_ft_level; int prio; int max_ft; } root_fs = { .type = FS_TYPE_NAMESPACE, .name = "root", .ar_size = 3, .children = (struct init_tree_node[]) { ADD_PRIO("by_pass_prio", 0, BYPASS_MIN_LEVEL, 0, FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), FS_CAP(flow_table_properties_nic_receive.modify_root)), ADD_NS("by_pass_ns", ADD_FT_PRIO("prio0", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio1", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio2", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio3", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio4", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio5", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio6", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio7", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio-mcast", 0, BYPASS_PRIO_MAX_FT))), ADD_PRIO("kernel_prio", 0, KENREL_MIN_LEVEL, 0, {}, ADD_NS("kernel_ns", ADD_FT_PRIO("prio_kernel-0", 0, KERNEL_MAX_FT))), ADD_PRIO("leftovers_prio", MLX5_CORE_FS_PRIO_SHARED, LEFTOVER_MIN_LEVEL, 0, FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), FS_CAP(flow_table_properties_nic_receive.modify_root)), ADD_NS("leftover_ns", ADD_FT_PRIO("leftovers_prio-0", MLX5_CORE_FS_PRIO_SHARED, LEFTOVER_MAX_FT))) } }; /* Tree creation functions */ static struct mlx5_flow_root_namespace *find_root(struct fs_base *node) { struct fs_base *parent; /* Make sure we only read it once while we go up the tree */ while ((parent = node->parent)) node = parent; if (node->type != FS_TYPE_NAMESPACE) { - printf("mlx5_core: WARN: ""mlx5: flow steering node %s is not in tree or garbaged\n", node->name); return NULL; } return container_of(container_of(node, struct mlx5_flow_namespace, base), struct mlx5_flow_root_namespace, ns); } static inline struct mlx5_core_dev *fs_get_dev(struct fs_base *node) { struct mlx5_flow_root_namespace *root = find_root(node); if (root) return root->dev; return NULL; } static void fs_init_node(struct fs_base *node, unsigned int refcount) { kref_init(&node->refcount); atomic_set(&node->users_refcount, refcount); init_completion(&node->complete); INIT_LIST_HEAD(&node->list); mutex_init(&node->lock); } static void _fs_add_node(struct fs_base *node, const char *name, struct fs_base *parent) { if (parent) atomic_inc(&parent->users_refcount); node->name = kstrdup_const(name, GFP_KERNEL); node->parent = parent; } static void fs_add_node(struct fs_base *node, struct fs_base *parent, const char *name, unsigned int refcount) { fs_init_node(node, refcount); _fs_add_node(node, name, parent); } static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref), bool parent_locked); static void fs_del_dst(struct mlx5_flow_rule *dst); static void _fs_del_ft(struct mlx5_flow_table *ft); static void fs_del_fg(struct mlx5_flow_group *fg); static void fs_del_fte(struct fs_fte *fte); static void cmd_remove_node(struct fs_base *base) { switch (base->type) { case FS_TYPE_FLOW_DEST: fs_del_dst(container_of(base, struct mlx5_flow_rule, base)); break; case FS_TYPE_FLOW_TABLE: _fs_del_ft(container_of(base, struct mlx5_flow_table, base)); break; case FS_TYPE_FLOW_GROUP: fs_del_fg(container_of(base, struct mlx5_flow_group, base)); break; case FS_TYPE_FLOW_ENTRY: fs_del_fte(container_of(base, struct fs_fte, base)); break; default: break; } } static void __fs_remove_node(struct kref *kref) { struct fs_base *node = container_of(kref, struct fs_base, refcount); if (node->parent) mutex_lock(&node->parent->lock); mutex_lock(&node->lock); cmd_remove_node(node); mutex_unlock(&node->lock); complete(&node->complete); if (node->parent) { mutex_unlock(&node->parent->lock); _fs_put(node->parent, _fs_remove_node, false); } } void _fs_remove_node(struct kref *kref) { struct fs_base *node = container_of(kref, struct fs_base, refcount); __fs_remove_node(kref); kfree_const(node->name); kfree(node); } static void fs_get(struct fs_base *node) { atomic_inc(&node->users_refcount); } static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref), bool parent_locked) { struct fs_base *parent_node = node->parent; if (parent_node && !parent_locked) mutex_lock(&parent_node->lock); if (atomic_dec_and_test(&node->users_refcount)) { if (parent_node) { /*remove from parent's list*/ list_del_init(&node->list); mutex_unlock(&parent_node->lock); } kref_put(&node->refcount, kref_cb); if (parent_node && parent_locked) mutex_lock(&parent_node->lock); } else if (parent_node && !parent_locked) { mutex_unlock(&parent_node->lock); } } static void fs_put(struct fs_base *node) { _fs_put(node, __fs_remove_node, false); } static void fs_put_parent_locked(struct fs_base *node) { _fs_put(node, __fs_remove_node, true); } static void fs_remove_node(struct fs_base *node) { fs_put(node); wait_for_completion(&node->complete); kfree_const(node->name); kfree(node); } static void fs_remove_node_parent_locked(struct fs_base *node) { fs_put_parent_locked(node); wait_for_completion(&node->complete); kfree_const(node->name); kfree(node); } static struct fs_fte *fs_alloc_fte(u8 action, u32 flow_tag, u32 *match_value, unsigned int index) { struct fs_fte *fte; fte = kzalloc(sizeof(*fte), GFP_KERNEL); if (!fte) return ERR_PTR(-ENOMEM); memcpy(fte->val, match_value, sizeof(fte->val)); fte->base.type = FS_TYPE_FLOW_ENTRY; fte->dests_size = 0; fte->flow_tag = flow_tag; fte->index = index; INIT_LIST_HEAD(&fte->dests); fte->action = action; return fte; } static struct fs_fte *alloc_star_ft_entry(struct mlx5_flow_table *ft, struct mlx5_flow_group *fg, u32 *match_value, unsigned int index) { int err; struct fs_fte *fte; struct mlx5_flow_rule *dst; if (fg->num_ftes == fg->max_ftes) return ERR_PTR(-ENOSPC); fte = fs_alloc_fte(MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_DEFAULT_FLOW_TAG, match_value, index); if (IS_ERR(fte)) return fte; /*create dst*/ dst = kzalloc(sizeof(*dst), GFP_KERNEL); if (!dst) { err = -ENOMEM; goto free_fte; } fte->base.parent = &fg->base; fte->dests_size = 1; dst->dest_attr.type = MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE; dst->base.parent = &fte->base; list_add(&dst->base.list, &fte->dests); /* assumed that the callee creates the star rules sorted by index */ list_add_tail(&fte->base.list, &fg->ftes); fg->num_ftes++; return fte; free_fte: kfree(fte); return ERR_PTR(err); } /* assume that fte can't be changed */ static void free_star_fte_entry(struct fs_fte *fte) { struct mlx5_flow_group *fg; struct mlx5_flow_rule *dst, *temp; fs_get_parent(fg, fte); list_for_each_entry_safe(dst, temp, &fte->dests, base.list) { fte->dests_size--; list_del(&dst->base.list); kfree(dst); } list_del(&fte->base.list); fg->num_ftes--; kfree(fte); } static struct mlx5_flow_group *fs_alloc_fg(u32 *create_fg_in) { struct mlx5_flow_group *fg; void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, create_fg_in, match_criteria); u8 match_criteria_enable = MLX5_GET(create_flow_group_in, create_fg_in, match_criteria_enable); fg = kzalloc(sizeof(*fg), GFP_KERNEL); if (!fg) return ERR_PTR(-ENOMEM); INIT_LIST_HEAD(&fg->ftes); fg->mask.match_criteria_enable = match_criteria_enable; memcpy(&fg->mask.match_criteria, match_criteria, sizeof(fg->mask.match_criteria)); fg->base.type = FS_TYPE_FLOW_GROUP; fg->start_index = MLX5_GET(create_flow_group_in, create_fg_in, start_flow_index); fg->max_ftes = MLX5_GET(create_flow_group_in, create_fg_in, end_flow_index) - fg->start_index + 1; return fg; } static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio); static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr, struct fs_prio *prio); /* assumed src_ft and dst_ft can't be freed */ static int fs_set_star_rule(struct mlx5_core_dev *dev, struct mlx5_flow_table *src_ft, struct mlx5_flow_table *dst_ft) { struct mlx5_flow_rule *src_dst; struct fs_fte *src_fte; int err = 0; u32 *match_value; int match_len = MLX5_ST_SZ_BYTES(fte_match_param); src_dst = list_first_entry(&src_ft->star_rule.fte->dests, struct mlx5_flow_rule, base.list); match_value = mlx5_vzalloc(match_len); if (!match_value) { mlx5_core_warn(dev, "failed to allocate inbox\n"); return -ENOMEM; } /*Create match context*/ fs_get_parent(src_fte, src_dst); src_dst->dest_attr.ft = dst_ft; if (dst_ft) { err = mlx5_cmd_fs_set_fte(dev, src_ft->vport, &src_fte->status, match_value, src_ft->type, src_ft->id, src_fte->index, src_ft->star_rule.fg->id, src_fte->flow_tag, src_fte->action, src_fte->dests_size, &src_fte->dests); if (err) goto free; fs_get(&dst_ft->base); } else { mlx5_cmd_fs_delete_fte(dev, src_ft->vport, &src_fte->status, src_ft->type, src_ft->id, src_fte->index); } free: kvfree(match_value); return err; } static int connect_prev_fts(struct fs_prio *locked_prio, struct fs_prio *prev_prio, struct mlx5_flow_table *next_ft) { struct mlx5_flow_table *iter; int err = 0; struct mlx5_core_dev *dev = fs_get_dev(&prev_prio->base); if (!dev) return -ENODEV; mutex_lock(&prev_prio->base.lock); fs_for_each_ft(iter, prev_prio) { struct mlx5_flow_rule *src_dst = list_first_entry(&iter->star_rule.fte->dests, struct mlx5_flow_rule, base.list); struct mlx5_flow_table *prev_ft = src_dst->dest_attr.ft; if (prev_ft == next_ft) continue; err = fs_set_star_rule(dev, iter, next_ft); if (err) { mlx5_core_warn(dev, - "mlx5: flow steering can't connect prev and next\n"); + "mlx5: flow steering can't connect prev and next\n"); goto unlock; } else { /* Assume ft's prio is locked */ if (prev_ft) { struct fs_prio *prio; fs_get_parent(prio, prev_ft); if (prio == locked_prio) fs_put_parent_locked(&prev_ft->base); else fs_put(&prev_ft->base); } } } unlock: mutex_unlock(&prev_prio->base.lock); return 0; } static int create_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio) { struct mlx5_flow_group *fg; int err; u32 *fg_in; u32 *match_value; struct mlx5_flow_table *next_ft; struct mlx5_flow_table *prev_ft; struct mlx5_flow_root_namespace *root = find_root(&prio->base); int fg_inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); int match_len = MLX5_ST_SZ_BYTES(fte_match_param); fg_in = mlx5_vzalloc(fg_inlen); if (!fg_in) { mlx5_core_warn(root->dev, "failed to allocate inbox\n"); return -ENOMEM; } match_value = mlx5_vzalloc(match_len); if (!match_value) { mlx5_core_warn(root->dev, "failed to allocate inbox\n"); kvfree(fg_in); return -ENOMEM; } MLX5_SET(create_flow_group_in, fg_in, start_flow_index, ft->max_fte); MLX5_SET(create_flow_group_in, fg_in, end_flow_index, ft->max_fte); fg = fs_alloc_fg(fg_in); if (IS_ERR(fg)) { err = PTR_ERR(fg); goto out; } ft->star_rule.fg = fg; err = mlx5_cmd_fs_create_fg(fs_get_dev(&prio->base), fg_in, ft->vport, ft->type, ft->id, &fg->id); if (err) goto free_fg; ft->star_rule.fte = alloc_star_ft_entry(ft, fg, match_value, ft->max_fte); if (IS_ERR(ft->star_rule.fte)) goto free_star_rule; mutex_lock(&root->fs_chain_lock); next_ft = find_next_ft(prio); err = fs_set_star_rule(root->dev, ft, next_ft); if (err) { mutex_unlock(&root->fs_chain_lock); goto free_star_rule; } if (next_ft) { struct fs_prio *parent; fs_get_parent(parent, next_ft); fs_put(&next_ft->base); } prev_ft = find_prev_ft(ft, prio); if (prev_ft) { struct fs_prio *prev_parent; fs_get_parent(prev_parent, prev_ft); err = connect_prev_fts(NULL, prev_parent, ft); if (err) { mutex_unlock(&root->fs_chain_lock); goto destroy_chained_star_rule; } fs_put(&prev_ft->base); } mutex_unlock(&root->fs_chain_lock); kvfree(fg_in); kvfree(match_value); return 0; destroy_chained_star_rule: fs_set_star_rule(fs_get_dev(&prio->base), ft, NULL); if (next_ft) fs_put(&next_ft->base); free_star_rule: free_star_fte_entry(ft->star_rule.fte); mlx5_cmd_fs_destroy_fg(fs_get_dev(&ft->base), ft->vport, ft->type, ft->id, fg->id); free_fg: kfree(fg); out: kvfree(fg_in); kvfree(match_value); return err; } static void destroy_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio) { int err; struct mlx5_flow_root_namespace *root; struct mlx5_core_dev *dev = fs_get_dev(&prio->base); struct mlx5_flow_table *prev_ft, *next_ft; struct fs_prio *prev_prio; WARN_ON(!dev); root = find_root(&prio->base); if (!root) - printf("mlx5_core: ERR: ""mlx5: flow steering failed to find root of priority %s", prio->base.name); + mlx5_core_err(dev, + "flow steering failed to find root of priority %s", + prio->base.name); /* In order to ensure atomic deletion, first update * prev ft to point on the next ft. */ mutex_lock(&root->fs_chain_lock); prev_ft = find_prev_ft(ft, prio); next_ft = find_next_ft(prio); if (prev_ft) { fs_get_parent(prev_prio, prev_ft); /*Prev is connected to ft, only if ft is the first(last) in the prio*/ err = connect_prev_fts(prio, prev_prio, next_ft); if (err) mlx5_core_warn(root->dev, "flow steering can't connect prev and next of flow table\n"); fs_put(&prev_ft->base); } err = fs_set_star_rule(root->dev, ft, NULL); /*One put is for fs_get in find next ft*/ if (next_ft) { fs_put(&next_ft->base); if (!err) fs_put(&next_ft->base); } mutex_unlock(&root->fs_chain_lock); err = mlx5_cmd_fs_destroy_fg(dev, ft->vport, ft->type, ft->id, ft->star_rule.fg->id); if (err) mlx5_core_warn(dev, "flow steering can't destroy star entry group(index:%d) of ft:%s\n", ft->star_rule.fg->start_index, ft->base.name); free_star_fte_entry(ft->star_rule.fte); kfree(ft->star_rule.fg); ft->star_rule.fg = NULL; } static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns, unsigned int prio) { struct fs_prio *iter_prio; fs_for_each_prio(iter_prio, ns) { if (iter_prio->prio == prio) return iter_prio; } return NULL; } static unsigned int _alloc_new_level(struct fs_prio *prio, struct mlx5_flow_namespace *match); static unsigned int __alloc_new_level(struct mlx5_flow_namespace *ns, struct fs_prio *prio) { unsigned int level = 0; struct fs_prio *p; if (!ns) return 0; mutex_lock(&ns->base.lock); fs_for_each_prio(p, ns) { if (p != prio) level += p->max_ft; else break; } mutex_unlock(&ns->base.lock); fs_get_parent(prio, ns); if (prio) WARN_ON(prio->base.type != FS_TYPE_PRIO); return level + _alloc_new_level(prio, ns); } /* Called under lock of priority, hence locking all upper objects */ static unsigned int _alloc_new_level(struct fs_prio *prio, struct mlx5_flow_namespace *match) { struct mlx5_flow_namespace *ns; struct fs_base *it; unsigned int level = 0; if (!prio) return 0; mutex_lock(&prio->base.lock); fs_for_each_ns_or_ft_reverse(it, prio) { if (it->type == FS_TYPE_NAMESPACE) { struct fs_prio *p; fs_get_obj(ns, it); if (match != ns) { mutex_lock(&ns->base.lock); fs_for_each_prio(p, ns) level += p->max_ft; mutex_unlock(&ns->base.lock); } else { break; } } else { struct mlx5_flow_table *ft; fs_get_obj(ft, it); mutex_unlock(&prio->base.lock); return level + ft->level + 1; } } fs_get_parent(ns, prio); mutex_unlock(&prio->base.lock); return __alloc_new_level(ns, prio) + level; } static unsigned int alloc_new_level(struct fs_prio *prio) { return _alloc_new_level(prio, NULL); } static int update_root_ft_create(struct mlx5_flow_root_namespace *root, struct mlx5_flow_table *ft) { int err = 0; int min_level = INT_MAX; if (root->root_ft) min_level = root->root_ft->level; if (ft->level < min_level) err = mlx5_cmd_update_root_ft(root->dev, ft->type, ft->id); else return err; if (err) mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n", ft->id); else root->root_ft = ft; return err; } static struct mlx5_flow_table *_create_ft_common(struct mlx5_flow_namespace *ns, u16 vport, struct fs_prio *fs_prio, int max_fte, const char *name) { struct mlx5_flow_table *ft; int err; int log_table_sz; int ft_size; char gen_name[20]; - struct mlx5_flow_root_namespace *root = - find_root(&ns->base); + struct mlx5_flow_root_namespace *root = find_root(&ns->base); + struct mlx5_core_dev *dev = fs_get_dev(&ns->base); if (!root) { - printf("mlx5_core: ERR: ""mlx5: flow steering failed to find root of namespace %s", ns->base.name); + mlx5_core_err(dev, + "flow steering failed to find root of namespace %s", + ns->base.name); return ERR_PTR(-ENODEV); } if (fs_prio->num_ft == fs_prio->max_ft) return ERR_PTR(-ENOSPC); ft = kzalloc(sizeof(*ft), GFP_KERNEL); if (!ft) return ERR_PTR(-ENOMEM); fs_init_node(&ft->base, 1); INIT_LIST_HEAD(&ft->fgs); /* Temporarily WA until we expose the level set in the API */ if (root->table_type == FS_FT_ESW_EGRESS_ACL || root->table_type == FS_FT_ESW_INGRESS_ACL) ft->level = 0; else ft->level = alloc_new_level(fs_prio); ft->base.type = FS_TYPE_FLOW_TABLE; ft->vport = vport; ft->type = root->table_type; /*Two entries are reserved for star rules*/ ft_size = roundup_pow_of_two(max_fte + 2); /*User isn't aware to those rules*/ ft->max_fte = ft_size - 2; log_table_sz = ilog2(ft_size); err = mlx5_cmd_fs_create_ft(root->dev, ft->vport, ft->type, ft->level, log_table_sz, &ft->id); if (err) goto free_ft; err = create_star_rule(ft, fs_prio); if (err) goto del_ft; if ((root->table_type == FS_FT_NIC_RX) && MLX5_CAP_FLOWTABLE(root->dev, flow_table_properties_nic_receive.modify_root)) { err = update_root_ft_create(root, ft); if (err) goto destroy_star_rule; } if (!name || !strlen(name)) { snprintf(gen_name, 20, "flow_table_%u", ft->id); _fs_add_node(&ft->base, gen_name, &fs_prio->base); } else { _fs_add_node(&ft->base, name, &fs_prio->base); } list_add_tail(&ft->base.list, &fs_prio->objs); fs_prio->num_ft++; return ft; destroy_star_rule: destroy_star_rule(ft, fs_prio); del_ft: mlx5_cmd_fs_destroy_ft(root->dev, ft->vport, ft->type, ft->id); free_ft: kfree(ft); return ERR_PTR(err); } static struct mlx5_flow_table *create_ft_common(struct mlx5_flow_namespace *ns, u16 vport, unsigned int prio, int max_fte, const char *name) { struct fs_prio *fs_prio = NULL; fs_prio = find_prio(ns, prio); if (!fs_prio) return ERR_PTR(-EINVAL); return _create_ft_common(ns, vport, fs_prio, max_fte, name); } static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns, struct list_head *start); static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio, struct list_head *start); static struct mlx5_flow_table *mlx5_create_autogrouped_shared_flow_table(struct fs_prio *fs_prio) { struct mlx5_flow_table *ft; ft = find_first_ft_in_prio(fs_prio, &fs_prio->objs); if (ft) { ft->shared_refcount++; return ft; } return NULL; } struct mlx5_flow_table *mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns, int prio, const char *name, int num_flow_table_entries, int max_num_groups) { struct mlx5_flow_table *ft = NULL; struct fs_prio *fs_prio; bool is_shared_prio; fs_prio = find_prio(ns, prio); if (!fs_prio) return ERR_PTR(-EINVAL); is_shared_prio = fs_prio->flags & MLX5_CORE_FS_PRIO_SHARED; if (is_shared_prio) { mutex_lock(&fs_prio->shared_lock); ft = mlx5_create_autogrouped_shared_flow_table(fs_prio); } if (ft) goto return_ft; ft = create_ft_common(ns, 0, prio, num_flow_table_entries, name); if (IS_ERR(ft)) goto return_ft; ft->autogroup.active = true; ft->autogroup.max_types = max_num_groups; if (is_shared_prio) ft->shared_refcount = 1; return_ft: if (is_shared_prio) mutex_unlock(&fs_prio->shared_lock); return ft; } EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table); struct mlx5_flow_table *mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns, u16 vport, int prio, const char *name, int num_flow_table_entries) { return create_ft_common(ns, vport, prio, num_flow_table_entries, name); } EXPORT_SYMBOL(mlx5_create_vport_flow_table); struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns, int prio, const char *name, int num_flow_table_entries) { return create_ft_common(ns, 0, prio, num_flow_table_entries, name); } EXPORT_SYMBOL(mlx5_create_flow_table); static void _fs_del_ft(struct mlx5_flow_table *ft) { int err; struct mlx5_core_dev *dev = fs_get_dev(&ft->base); struct fs_prio *prio; err = mlx5_cmd_fs_destroy_ft(dev, ft->vport, ft->type, ft->id); if (err) mlx5_core_warn(dev, "flow steering can't destroy ft %s\n", ft->base.name); fs_get_parent(prio, ft); prio->num_ft--; } static int update_root_ft_destroy(struct mlx5_flow_root_namespace *root, struct mlx5_flow_table *ft) { int err = 0; struct fs_prio *prio; struct mlx5_flow_table *next_ft = NULL; struct mlx5_flow_table *put_ft = NULL; if (root->root_ft != ft) return 0; fs_get_parent(prio, ft); /*Assuming objs containis only flow tables and * flow tables are sorted by level. */ if (!list_is_last(&ft->base.list, &prio->objs)) { next_ft = list_next_entry(ft, base.list); } else { next_ft = find_next_ft(prio); put_ft = next_ft; } if (next_ft) { err = mlx5_cmd_update_root_ft(root->dev, next_ft->type, next_ft->id); if (err) mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n", ft->id); } if (!err) root->root_ft = next_ft; if (put_ft) fs_put(&put_ft->base); return err; } /*Objects in the same prio are destroyed in the reverse order they were createrd*/ int mlx5_destroy_flow_table(struct mlx5_flow_table *ft) { int err = 0; struct fs_prio *prio; struct mlx5_flow_root_namespace *root; bool is_shared_prio; + struct mlx5_core_dev *dev; fs_get_parent(prio, ft); root = find_root(&prio->base); + dev = fs_get_dev(&prio->base); if (!root) { - printf("mlx5_core: ERR: ""mlx5: flow steering failed to find root of priority %s", prio->base.name); + mlx5_core_err(dev, + "flow steering failed to find root of priority %s", + prio->base.name); return -ENODEV; } is_shared_prio = prio->flags & MLX5_CORE_FS_PRIO_SHARED; if (is_shared_prio) { mutex_lock(&prio->shared_lock); if (ft->shared_refcount > 1) { --ft->shared_refcount; fs_put(&ft->base); mutex_unlock(&prio->shared_lock); return 0; } } mutex_lock(&prio->base.lock); mutex_lock(&ft->base.lock); err = update_root_ft_destroy(root, ft); if (err) goto unlock_ft; /* delete two last entries */ destroy_star_rule(ft, prio); mutex_unlock(&ft->base.lock); fs_remove_node_parent_locked(&ft->base); mutex_unlock(&prio->base.lock); if (is_shared_prio) mutex_unlock(&prio->shared_lock); return err; unlock_ft: mutex_unlock(&ft->base.lock); mutex_unlock(&prio->base.lock); if (is_shared_prio) mutex_unlock(&prio->shared_lock); return err; } EXPORT_SYMBOL(mlx5_destroy_flow_table); static struct mlx5_flow_group *fs_create_fg(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft, struct list_head *prev, u32 *fg_in, int refcount) { struct mlx5_flow_group *fg; int err; unsigned int end_index; char name[20]; fg = fs_alloc_fg(fg_in); if (IS_ERR(fg)) return fg; end_index = fg->start_index + fg->max_ftes - 1; err = mlx5_cmd_fs_create_fg(dev, fg_in, ft->vport, ft->type, ft->id, &fg->id); if (err) goto free_fg; mutex_lock(&ft->base.lock); if (ft->autogroup.active) ft->autogroup.num_types++; snprintf(name, sizeof(name), "group_%u", fg->id); /*Add node to tree*/ fs_add_node(&fg->base, &ft->base, name, refcount); /*Add node to group list*/ list_add(&fg->base.list, prev); mutex_unlock(&ft->base.lock); return fg; free_fg: kfree(fg); return ERR_PTR(err); } struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft, u32 *in) { struct mlx5_flow_group *fg; struct mlx5_core_dev *dev = fs_get_dev(&ft->base); if (!dev) return ERR_PTR(-ENODEV); if (ft->autogroup.active) return ERR_PTR(-EPERM); fg = fs_create_fg(dev, ft, ft->fgs.prev, in, 1); return fg; } EXPORT_SYMBOL(mlx5_create_flow_group); /*Group is destoyed when all the rules in the group were removed*/ static void fs_del_fg(struct mlx5_flow_group *fg) { struct mlx5_flow_table *parent_ft; struct mlx5_core_dev *dev; fs_get_parent(parent_ft, fg); dev = fs_get_dev(&parent_ft->base); WARN_ON(!dev); if (parent_ft->autogroup.active) parent_ft->autogroup.num_types--; if (mlx5_cmd_fs_destroy_fg(dev, parent_ft->vport, parent_ft->type, parent_ft->id, fg->id)) mlx5_core_warn(dev, "flow steering can't destroy fg\n"); } void mlx5_destroy_flow_group(struct mlx5_flow_group *fg) { fs_remove_node(&fg->base); } EXPORT_SYMBOL(mlx5_destroy_flow_group); static bool _fs_match_exact_val(void *mask, void *val1, void *val2, size_t size) { unsigned int i; /* TODO: optimize by comparing 64bits when possible */ for (i = 0; i < size; i++, mask++, val1++, val2++) if ((*((u8 *)val1) & (*(u8 *)mask)) != ((*(u8 *)val2) & (*(u8 *)mask))) return false; return true; } bool fs_match_exact_val(struct mlx5_core_fs_mask *mask, void *val1, void *val2) { if (mask->match_criteria_enable & 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS) { void *fte_match1 = MLX5_ADDR_OF(fte_match_param, val1, outer_headers); void *fte_match2 = MLX5_ADDR_OF(fte_match_param, val2, outer_headers); void *fte_mask = MLX5_ADDR_OF(fte_match_param, mask->match_criteria, outer_headers); if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4))) return false; } if (mask->match_criteria_enable & 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) { void *fte_match1 = MLX5_ADDR_OF(fte_match_param, val1, misc_parameters); void *fte_match2 = MLX5_ADDR_OF(fte_match_param, val2, misc_parameters); void *fte_mask = MLX5_ADDR_OF(fte_match_param, mask->match_criteria, misc_parameters); if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, MLX5_ST_SZ_BYTES(fte_match_set_misc))) return false; } if (mask->match_criteria_enable & 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS) { void *fte_match1 = MLX5_ADDR_OF(fte_match_param, val1, inner_headers); void *fte_match2 = MLX5_ADDR_OF(fte_match_param, val2, inner_headers); void *fte_mask = MLX5_ADDR_OF(fte_match_param, mask->match_criteria, inner_headers); if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4))) return false; } return true; } bool fs_match_exact_mask(u8 match_criteria_enable1, u8 match_criteria_enable2, void *mask1, void *mask2) { return match_criteria_enable1 == match_criteria_enable2 && !memcmp(mask1, mask2, MLX5_ST_SZ_BYTES(fte_match_param)); } static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns, struct list_head *start); static struct mlx5_flow_table *_find_first_ft_in_prio_reverse(struct fs_prio *prio, struct list_head *start) { struct fs_base *it = container_of(start, struct fs_base, list); if (!prio) return NULL; fs_for_each_ns_or_ft_continue_reverse(it, prio) { struct mlx5_flow_namespace *ns; struct mlx5_flow_table *ft; if (it->type == FS_TYPE_FLOW_TABLE) { fs_get_obj(ft, it); fs_get(&ft->base); return ft; } fs_get_obj(ns, it); WARN_ON(ns->base.type != FS_TYPE_NAMESPACE); ft = find_first_ft_in_ns_reverse(ns, &ns->prios); if (ft) return ft; } return NULL; } static struct mlx5_flow_table *find_first_ft_in_prio_reverse(struct fs_prio *prio, struct list_head *start) { struct mlx5_flow_table *ft; if (!prio) return NULL; mutex_lock(&prio->base.lock); ft = _find_first_ft_in_prio_reverse(prio, start); mutex_unlock(&prio->base.lock); return ft; } static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns, struct list_head *start) { struct fs_prio *prio; if (!ns) return NULL; fs_get_obj(prio, container_of(start, struct fs_base, list)); mutex_lock(&ns->base.lock); fs_for_each_prio_continue_reverse(prio, ns) { struct mlx5_flow_table *ft; ft = find_first_ft_in_prio_reverse(prio, &prio->objs); if (ft) { mutex_unlock(&ns->base.lock); return ft; } } mutex_unlock(&ns->base.lock); return NULL; } /* Returned a held ft, assumed curr is protected, assumed curr's parent is * locked */ static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr, struct fs_prio *prio) { struct mlx5_flow_table *ft = NULL; struct fs_base *curr_base; if (!curr) return NULL; /* prio has either namespace or flow-tables, but not both */ if (!list_empty(&prio->objs) && list_first_entry(&prio->objs, struct mlx5_flow_table, base.list) != curr) return NULL; while (!ft && prio) { struct mlx5_flow_namespace *ns; fs_get_parent(ns, prio); ft = find_first_ft_in_ns_reverse(ns, &prio->base.list); curr_base = &ns->base; fs_get_parent(prio, ns); if (prio && !ft) ft = find_first_ft_in_prio_reverse(prio, &curr_base->list); } return ft; } static struct mlx5_flow_table *_find_first_ft_in_prio(struct fs_prio *prio, struct list_head *start) { struct fs_base *it = container_of(start, struct fs_base, list); if (!prio) return NULL; fs_for_each_ns_or_ft_continue(it, prio) { struct mlx5_flow_namespace *ns; struct mlx5_flow_table *ft; if (it->type == FS_TYPE_FLOW_TABLE) { fs_get_obj(ft, it); fs_get(&ft->base); return ft; } fs_get_obj(ns, it); WARN_ON(ns->base.type != FS_TYPE_NAMESPACE); ft = find_first_ft_in_ns(ns, &ns->prios); if (ft) return ft; } return NULL; } static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio, struct list_head *start) { struct mlx5_flow_table *ft; if (!prio) return NULL; mutex_lock(&prio->base.lock); ft = _find_first_ft_in_prio(prio, start); mutex_unlock(&prio->base.lock); return ft; } static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns, struct list_head *start) { struct fs_prio *prio; if (!ns) return NULL; fs_get_obj(prio, container_of(start, struct fs_base, list)); mutex_lock(&ns->base.lock); fs_for_each_prio_continue(prio, ns) { struct mlx5_flow_table *ft; ft = find_first_ft_in_prio(prio, &prio->objs); if (ft) { mutex_unlock(&ns->base.lock); return ft; } } mutex_unlock(&ns->base.lock); return NULL; } /* returned a held ft, assumed curr is protected, assumed curr's parent is * locked */ static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio) { struct mlx5_flow_table *ft = NULL; struct fs_base *curr_base; while (!ft && prio) { struct mlx5_flow_namespace *ns; fs_get_parent(ns, prio); ft = find_first_ft_in_ns(ns, &prio->base.list); curr_base = &ns->base; fs_get_parent(prio, ns); if (!ft && prio) ft = _find_first_ft_in_prio(prio, &curr_base->list); } return ft; } /* called under ft mutex lock */ static struct mlx5_flow_group *create_autogroup(struct mlx5_flow_table *ft, u8 match_criteria_enable, u32 *match_criteria) { unsigned int group_size; unsigned int candidate_index = 0; unsigned int candidate_group_num = 0; struct mlx5_flow_group *g; struct mlx5_flow_group *ret; struct list_head *prev = &ft->fgs; struct mlx5_core_dev *dev; u32 *in; int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); void *match_criteria_addr; if (!ft->autogroup.active) return ERR_PTR(-ENOENT); dev = fs_get_dev(&ft->base); if (!dev) return ERR_PTR(-ENODEV); in = mlx5_vzalloc(inlen); if (!in) { mlx5_core_warn(dev, "failed to allocate inbox\n"); return ERR_PTR(-ENOMEM); } if (ft->autogroup.num_types < ft->autogroup.max_types) group_size = ft->max_fte / (ft->autogroup.max_types + 1); else group_size = 1; if (group_size == 0) { mlx5_core_warn(dev, "flow steering can't create group size of 0\n"); ret = ERR_PTR(-EINVAL); goto out; } /* sorted by start_index */ fs_for_each_fg(g, ft) { candidate_group_num++; if (candidate_index + group_size > g->start_index) candidate_index = g->start_index + g->max_ftes; else break; prev = &g->base.list; } if (candidate_index + group_size > ft->max_fte) { ret = ERR_PTR(-ENOSPC); goto out; } MLX5_SET(create_flow_group_in, in, match_criteria_enable, match_criteria_enable); MLX5_SET(create_flow_group_in, in, start_flow_index, candidate_index); MLX5_SET(create_flow_group_in, in, end_flow_index, candidate_index + group_size - 1); match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); memcpy(match_criteria_addr, match_criteria, MLX5_ST_SZ_BYTES(fte_match_param)); ret = fs_create_fg(dev, ft, prev, in, 0); out: kvfree(in); return ret; } static struct mlx5_flow_namespace *get_ns_with_notifiers(struct fs_base *node) { struct mlx5_flow_namespace *ns = NULL; while (node && (node->type != FS_TYPE_NAMESPACE || list_empty(&container_of(node, struct mlx5_flow_namespace, base)->list_notifiers))) node = node->parent; if (node) fs_get_obj(ns, node); return ns; } /*Assumption- fte is locked*/ static void call_to_add_rule_notifiers(struct mlx5_flow_rule *dst, struct fs_fte *fte) { struct mlx5_flow_namespace *ns; struct mlx5_flow_handler *iter_handler; struct fs_client_priv_data *iter_client; void *data; bool is_new_rule = list_first_entry(&fte->dests, struct mlx5_flow_rule, base.list) == dst; int err; ns = get_ns_with_notifiers(&fte->base); if (!ns) return; down_read(&ns->notifiers_rw_sem); list_for_each_entry(iter_handler, &ns->list_notifiers, list) { if (iter_handler->add_dst_cb) { data = NULL; mutex_lock(&dst->clients_lock); list_for_each_entry( iter_client, &dst->clients_data, list) { if (iter_client->fs_handler == iter_handler) { data = iter_client->client_dst_data; break; } } mutex_unlock(&dst->clients_lock); err = iter_handler->add_dst_cb(dst, is_new_rule, NULL, iter_handler->client_context); if (err) break; } } up_read(&ns->notifiers_rw_sem); } static void call_to_del_rule_notifiers(struct mlx5_flow_rule *dst, struct fs_fte *fte) { struct mlx5_flow_namespace *ns; struct mlx5_flow_handler *iter_handler; struct fs_client_priv_data *iter_client; void *data; bool ctx_changed = (fte->dests_size == 0); ns = get_ns_with_notifiers(&fte->base); if (!ns) return; down_read(&ns->notifiers_rw_sem); list_for_each_entry(iter_handler, &ns->list_notifiers, list) { data = NULL; mutex_lock(&dst->clients_lock); list_for_each_entry(iter_client, &dst->clients_data, list) { if (iter_client->fs_handler == iter_handler) { data = iter_client->client_dst_data; break; } } mutex_unlock(&dst->clients_lock); if (iter_handler->del_dst_cb) { iter_handler->del_dst_cb(dst, ctx_changed, data, iter_handler->client_context); } } up_read(&ns->notifiers_rw_sem); } /* fte should not be deleted while calling this function */ static struct mlx5_flow_rule *_fs_add_dst_fte(struct fs_fte *fte, struct mlx5_flow_group *fg, struct mlx5_flow_destination *dest) { struct mlx5_flow_table *ft; struct mlx5_flow_rule *dst; int err; dst = kzalloc(sizeof(*dst), GFP_KERNEL); if (!dst) return ERR_PTR(-ENOMEM); memcpy(&dst->dest_attr, dest, sizeof(*dest)); dst->base.type = FS_TYPE_FLOW_DEST; INIT_LIST_HEAD(&dst->clients_data); mutex_init(&dst->clients_lock); fs_get_parent(ft, fg); /*Add dest to dests list- added as first element after the head*/ list_add_tail(&dst->base.list, &fte->dests); fte->dests_size++; err = mlx5_cmd_fs_set_fte(fs_get_dev(&ft->base), ft->vport, &fte->status, fte->val, ft->type, ft->id, fte->index, fg->id, fte->flow_tag, fte->action, fte->dests_size, &fte->dests); if (err) goto free_dst; list_del(&dst->base.list); return dst; free_dst: list_del(&dst->base.list); kfree(dst); fte->dests_size--; return ERR_PTR(err); } static char *get_dest_name(struct mlx5_flow_destination *dest) { char *name = kzalloc(sizeof(char) * 20, GFP_KERNEL); switch (dest->type) { case MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE: snprintf(name, 20, "dest_%s_%u", "flow_table", dest->ft->id); return name; case MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT: snprintf(name, 20, "dest_%s_%u", "vport", dest->vport_num); return name; case MLX5_FLOW_CONTEXT_DEST_TYPE_TIR: snprintf(name, 20, "dest_%s_%u", "tir", dest->tir_num); return name; default: kfree(name); return NULL; } } /* assumed fg is locked */ static unsigned int fs_get_free_fg_index(struct mlx5_flow_group *fg, struct list_head **prev) { struct fs_fte *fte; unsigned int start = fg->start_index; if (prev) *prev = &fg->ftes; /* assumed list is sorted by index */ fs_for_each_fte(fte, fg) { if (fte->index != start) return start; start++; if (prev) *prev = &fte->base.list; } return start; } static struct fs_fte *fs_create_fte(struct mlx5_flow_group *fg, u32 *match_value, u8 action, u32 flow_tag, struct list_head **prev) { struct fs_fte *fte; int index = 0; index = fs_get_free_fg_index(fg, prev); fte = fs_alloc_fte(action, flow_tag, match_value, index); if (IS_ERR(fte)) return fte; return fte; } static void add_rule_to_tree(struct mlx5_flow_rule *rule, struct fs_fte *fte) { char *dest_name; dest_name = get_dest_name(&rule->dest_attr); fs_add_node(&rule->base, &fte->base, dest_name, 1); /* re-add to list, since fs_add_node reset our list */ list_add_tail(&rule->base.list, &fte->dests); kfree(dest_name); call_to_add_rule_notifiers(rule, fte); } static void fs_del_dst(struct mlx5_flow_rule *dst) { struct mlx5_flow_table *ft; struct mlx5_flow_group *fg; struct fs_fte *fte; u32 *match_value; struct mlx5_core_dev *dev = fs_get_dev(&dst->base); int match_len = MLX5_ST_SZ_BYTES(fte_match_param); int err; WARN_ON(!dev); match_value = mlx5_vzalloc(match_len); if (!match_value) { mlx5_core_warn(dev, "failed to allocate inbox\n"); return; } fs_get_parent(fte, dst); fs_get_parent(fg, fte); mutex_lock(&fg->base.lock); memcpy(match_value, fte->val, sizeof(fte->val)); /* ft can't be changed as fg is locked */ fs_get_parent(ft, fg); list_del(&dst->base.list); fte->dests_size--; if (fte->dests_size) { err = mlx5_cmd_fs_set_fte(dev, ft->vport, &fte->status, match_value, ft->type, ft->id, fte->index, fg->id, fte->flow_tag, fte->action, fte->dests_size, &fte->dests); if (err) { mlx5_core_warn(dev, "%s can't delete dst %s\n", __func__, dst->base.name); goto err; } } call_to_del_rule_notifiers(dst, fte); err: mutex_unlock(&fg->base.lock); kvfree(match_value); } static void fs_del_fte(struct fs_fte *fte) { struct mlx5_flow_table *ft; struct mlx5_flow_group *fg; int err; struct mlx5_core_dev *dev; fs_get_parent(fg, fte); fs_get_parent(ft, fg); dev = fs_get_dev(&ft->base); WARN_ON(!dev); err = mlx5_cmd_fs_delete_fte(dev, ft->vport, &fte->status, ft->type, ft->id, fte->index); if (err) mlx5_core_warn(dev, "flow steering can't delete fte %s\n", fte->base.name); fg->num_ftes--; } /* assuming parent fg is locked */ /* Add dst algorithm */ static struct mlx5_flow_rule *fs_add_dst_fg(struct mlx5_flow_group *fg, u32 *match_value, u8 action, u32 flow_tag, struct mlx5_flow_destination *dest) { struct fs_fte *fte; struct mlx5_flow_rule *dst; struct mlx5_flow_table *ft; struct list_head *prev; char fte_name[20]; mutex_lock(&fg->base.lock); fs_for_each_fte(fte, fg) { /* TODO: Check of size against PRM max size */ mutex_lock(&fte->base.lock); if (fs_match_exact_val(&fg->mask, match_value, &fte->val) && action == fte->action && flow_tag == fte->flow_tag) { dst = _fs_add_dst_fte(fte, fg, dest); mutex_unlock(&fte->base.lock); if (IS_ERR(dst)) goto unlock_fg; goto add_rule; } mutex_unlock(&fte->base.lock); } fs_get_parent(ft, fg); if (fg->num_ftes == fg->max_ftes) { dst = ERR_PTR(-ENOSPC); goto unlock_fg; } fte = fs_create_fte(fg, match_value, action, flow_tag, &prev); if (IS_ERR(fte)) { dst = (void *)fte; goto unlock_fg; } dst = _fs_add_dst_fte(fte, fg, dest); if (IS_ERR(dst)) { kfree(fte); goto unlock_fg; } fg->num_ftes++; snprintf(fte_name, sizeof(fte_name), "fte%u", fte->index); /* Add node to tree */ fs_add_node(&fte->base, &fg->base, fte_name, 0); list_add(&fte->base.list, prev); add_rule: add_rule_to_tree(dst, fte); unlock_fg: mutex_unlock(&fg->base.lock); return dst; } static struct mlx5_flow_rule *fs_add_dst_ft(struct mlx5_flow_table *ft, u8 match_criteria_enable, u32 *match_criteria, u32 *match_value, u8 action, u32 flow_tag, struct mlx5_flow_destination *dest) { /*? where dst_entry is allocated*/ struct mlx5_flow_group *g; struct mlx5_flow_rule *dst; fs_get(&ft->base); mutex_lock(&ft->base.lock); fs_for_each_fg(g, ft) if (fs_match_exact_mask(g->mask.match_criteria_enable, match_criteria_enable, g->mask.match_criteria, match_criteria)) { mutex_unlock(&ft->base.lock); dst = fs_add_dst_fg(g, match_value, action, flow_tag, dest); if (PTR_ERR(dst) && PTR_ERR(dst) != -ENOSPC) goto unlock; } mutex_unlock(&ft->base.lock); g = create_autogroup(ft, match_criteria_enable, match_criteria); if (IS_ERR(g)) { dst = (void *)g; goto unlock; } dst = fs_add_dst_fg(g, match_value, action, flow_tag, dest); if (IS_ERR(dst)) { /* Remove assumes refcount > 0 and autogroup creates a group * with a refcount = 0. */ fs_get(&g->base); fs_remove_node(&g->base); goto unlock; } unlock: fs_put(&ft->base); return dst; } struct mlx5_flow_rule * mlx5_add_flow_rule(struct mlx5_flow_table *ft, u8 match_criteria_enable, u32 *match_criteria, u32 *match_value, u32 action, u32 flow_tag, struct mlx5_flow_destination *dest) { struct mlx5_flow_rule *dst; struct mlx5_flow_namespace *ns; ns = get_ns_with_notifiers(&ft->base); if (ns) down_read(&ns->dests_rw_sem); dst = fs_add_dst_ft(ft, match_criteria_enable, match_criteria, match_value, action, flow_tag, dest); if (ns) up_read(&ns->dests_rw_sem); return dst; } EXPORT_SYMBOL(mlx5_add_flow_rule); void mlx5_del_flow_rule(struct mlx5_flow_rule *dst) { struct mlx5_flow_namespace *ns; ns = get_ns_with_notifiers(&dst->base); if (ns) down_read(&ns->dests_rw_sem); fs_remove_node(&dst->base); if (ns) up_read(&ns->dests_rw_sem); } EXPORT_SYMBOL(mlx5_del_flow_rule); #define MLX5_CORE_FS_ROOT_NS_NAME "root" #define MLX5_CORE_FS_ESW_EGRESS_ACL "esw_egress_root" #define MLX5_CORE_FS_ESW_INGRESS_ACL "esw_ingress_root" #define MLX5_CORE_FS_FDB_ROOT_NS_NAME "fdb_root" #define MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME "sniffer_rx_root" #define MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME "sniffer_tx_root" #define MLX5_CORE_FS_PRIO_MAX_FT 4 #define MLX5_CORE_FS_PRIO_MAX_NS 1 static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns, unsigned prio, int max_ft, const char *name, u8 flags) { struct fs_prio *fs_prio; fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL); if (!fs_prio) return ERR_PTR(-ENOMEM); fs_prio->base.type = FS_TYPE_PRIO; fs_add_node(&fs_prio->base, &ns->base, name, 1); fs_prio->max_ft = max_ft; fs_prio->max_ns = MLX5_CORE_FS_PRIO_MAX_NS; fs_prio->prio = prio; fs_prio->flags = flags; list_add_tail(&fs_prio->base.list, &ns->prios); INIT_LIST_HEAD(&fs_prio->objs); mutex_init(&fs_prio->shared_lock); return fs_prio; } static void cleanup_root_ns(struct mlx5_core_dev *dev) { struct mlx5_flow_root_namespace *root_ns = dev->root_ns; struct fs_prio *iter_prio; if (!root_ns) return; /* stage 1 */ fs_for_each_prio(iter_prio, &root_ns->ns) { struct mlx5_flow_namespace *iter_ns; fs_for_each_ns(iter_ns, iter_prio) { while (!list_empty(&iter_ns->prios)) { struct fs_base *iter_prio2 = list_first_entry(&iter_ns->prios, struct fs_base, list); fs_remove_node(iter_prio2); } } } /* stage 2 */ fs_for_each_prio(iter_prio, &root_ns->ns) { while (!list_empty(&iter_prio->objs)) { struct fs_base *iter_ns = list_first_entry(&iter_prio->objs, struct fs_base, list); fs_remove_node(iter_ns); } } /* stage 3 */ while (!list_empty(&root_ns->ns.prios)) { struct fs_base *iter_prio = list_first_entry(&root_ns->ns.prios, struct fs_base, list); fs_remove_node(iter_prio); } fs_remove_node(&root_ns->ns.base); dev->root_ns = NULL; } static void cleanup_single_prio_root_ns(struct mlx5_core_dev *dev, struct mlx5_flow_root_namespace *root_ns) { struct fs_base *prio; if (!root_ns) return; if (!list_empty(&root_ns->ns.prios)) { prio = list_first_entry(&root_ns->ns.prios, struct fs_base, list); fs_remove_node(prio); } fs_remove_node(&root_ns->ns.base); root_ns = NULL; } void mlx5_cleanup_fs(struct mlx5_core_dev *dev) { cleanup_root_ns(dev); cleanup_single_prio_root_ns(dev, dev->sniffer_rx_root_ns); cleanup_single_prio_root_ns(dev, dev->sniffer_tx_root_ns); cleanup_single_prio_root_ns(dev, dev->fdb_root_ns); cleanup_single_prio_root_ns(dev, dev->esw_egress_root_ns); cleanup_single_prio_root_ns(dev, dev->esw_ingress_root_ns); } static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace *ns) { ns->base.type = FS_TYPE_NAMESPACE; init_rwsem(&ns->dests_rw_sem); init_rwsem(&ns->notifiers_rw_sem); INIT_LIST_HEAD(&ns->prios); INIT_LIST_HEAD(&ns->list_notifiers); return ns; } static struct mlx5_flow_root_namespace *create_root_ns(struct mlx5_core_dev *dev, enum fs_ft_type table_type, char *name) { struct mlx5_flow_root_namespace *root_ns; struct mlx5_flow_namespace *ns; /* create the root namespace */ root_ns = mlx5_vzalloc(sizeof(*root_ns)); if (!root_ns) goto err; root_ns->dev = dev; root_ns->table_type = table_type; mutex_init(&root_ns->fs_chain_lock); ns = &root_ns->ns; fs_init_namespace(ns); fs_add_node(&ns->base, NULL, name, 1); return root_ns; err: return NULL; } static int init_fdb_root_ns(struct mlx5_core_dev *dev) { struct fs_prio *prio; dev->fdb_root_ns = create_root_ns(dev, FS_FT_FDB, MLX5_CORE_FS_FDB_ROOT_NS_NAME); if (!dev->fdb_root_ns) return -ENOMEM; /* create 1 prio*/ prio = fs_create_prio(&dev->fdb_root_ns->ns, 0, 1, "fdb_prio", 0); if (IS_ERR(prio)) return PTR_ERR(prio); else return 0; } #define MAX_VPORTS 128 static int init_egress_acl_root_ns(struct mlx5_core_dev *dev) { struct fs_prio *prio; dev->esw_egress_root_ns = create_root_ns(dev, FS_FT_ESW_EGRESS_ACL, MLX5_CORE_FS_ESW_EGRESS_ACL); if (!dev->esw_egress_root_ns) return -ENOMEM; /* create 1 prio*/ prio = fs_create_prio(&dev->esw_egress_root_ns->ns, 0, MAX_VPORTS, "esw_egress_prio", 0); if (IS_ERR(prio)) return PTR_ERR(prio); else return 0; } static int init_ingress_acl_root_ns(struct mlx5_core_dev *dev) { struct fs_prio *prio; dev->esw_ingress_root_ns = create_root_ns(dev, FS_FT_ESW_INGRESS_ACL, MLX5_CORE_FS_ESW_INGRESS_ACL); if (!dev->esw_ingress_root_ns) return -ENOMEM; /* create 1 prio*/ prio = fs_create_prio(&dev->esw_ingress_root_ns->ns, 0, MAX_VPORTS, "esw_ingress_prio", 0); if (IS_ERR(prio)) return PTR_ERR(prio); else return 0; } static int init_sniffer_rx_root_ns(struct mlx5_core_dev *dev) { struct fs_prio *prio; dev->sniffer_rx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_RX, MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME); if (!dev->sniffer_rx_root_ns) return -ENOMEM; /* create 1 prio*/ prio = fs_create_prio(&dev->sniffer_rx_root_ns->ns, 0, 1, "sniffer_prio", 0); if (IS_ERR(prio)) return PTR_ERR(prio); else return 0; } static int init_sniffer_tx_root_ns(struct mlx5_core_dev *dev) { struct fs_prio *prio; dev->sniffer_tx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_TX, MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME); if (!dev->sniffer_tx_root_ns) return -ENOMEM; /* create 1 prio*/ prio = fs_create_prio(&dev->sniffer_tx_root_ns->ns, 0, 1, "sniffer_prio", 0); if (IS_ERR(prio)) return PTR_ERR(prio); else return 0; } static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio, const char *name) { struct mlx5_flow_namespace *ns; ns = kzalloc(sizeof(*ns), GFP_KERNEL); if (!ns) return ERR_PTR(-ENOMEM); fs_init_namespace(ns); fs_add_node(&ns->base, &prio->base, name, 1); list_add_tail(&ns->base.list, &prio->objs); return ns; } #define FLOW_TABLE_BIT_SZ 1 #define GET_FLOW_TABLE_CAP(dev, offset) \ ((be32_to_cpu(*((__be32 *)(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) + \ offset / 32)) >> \ (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ) static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps) { int i; for (i = 0; i < caps->arr_sz; i++) { if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i])) return false; } return true; } static int _init_root_tree(struct mlx5_core_dev *dev, int max_ft_level, struct init_tree_node *node, struct fs_base *base_parent, struct init_tree_node *tree_parent) { struct mlx5_flow_namespace *fs_ns; struct fs_prio *fs_prio; int priority; struct fs_base *base; int i; int err = 0; if (node->type == FS_TYPE_PRIO) { if ((node->min_ft_level > max_ft_level) || !has_required_caps(dev, &node->caps)) goto out; fs_get_obj(fs_ns, base_parent); priority = node - tree_parent->children; fs_prio = fs_create_prio(fs_ns, priority, node->max_ft, node->name, node->flags); if (IS_ERR(fs_prio)) { err = PTR_ERR(fs_prio); goto out; } base = &fs_prio->base; } else if (node->type == FS_TYPE_NAMESPACE) { fs_get_obj(fs_prio, base_parent); fs_ns = fs_create_namespace(fs_prio, node->name); if (IS_ERR(fs_ns)) { err = PTR_ERR(fs_ns); goto out; } base = &fs_ns->base; } else { return -EINVAL; } for (i = 0; i < node->ar_size; i++) { err = _init_root_tree(dev, max_ft_level, &node->children[i], base, node); if (err) break; } out: return err; } static int init_root_tree(struct mlx5_core_dev *dev, int max_ft_level, struct init_tree_node *node, struct fs_base *parent) { int i; struct mlx5_flow_namespace *fs_ns; int err = 0; fs_get_obj(fs_ns, parent); for (i = 0; i < node->ar_size; i++) { err = _init_root_tree(dev, max_ft_level, &node->children[i], &fs_ns->base, node); if (err) break; } return err; } static int sum_max_ft_in_prio(struct fs_prio *prio); static int sum_max_ft_in_ns(struct mlx5_flow_namespace *ns) { struct fs_prio *prio; int sum = 0; fs_for_each_prio(prio, ns) { sum += sum_max_ft_in_prio(prio); } return sum; } static int sum_max_ft_in_prio(struct fs_prio *prio) { int sum = 0; struct fs_base *it; struct mlx5_flow_namespace *ns; if (prio->max_ft) return prio->max_ft; fs_for_each_ns_or_ft(it, prio) { if (it->type == FS_TYPE_FLOW_TABLE) continue; fs_get_obj(ns, it); sum += sum_max_ft_in_ns(ns); } prio->max_ft = sum; return sum; } static void set_max_ft(struct mlx5_flow_namespace *ns) { struct fs_prio *prio; if (!ns) return; fs_for_each_prio(prio, ns) sum_max_ft_in_prio(prio); } static int init_root_ns(struct mlx5_core_dev *dev) { int max_ft_level = MLX5_CAP_FLOWTABLE(dev, flow_table_properties_nic_receive. max_ft_level); dev->root_ns = create_root_ns(dev, FS_FT_NIC_RX, MLX5_CORE_FS_ROOT_NS_NAME); if (IS_ERR_OR_NULL(dev->root_ns)) goto err; if (init_root_tree(dev, max_ft_level, &root_fs, &dev->root_ns->ns.base)) goto err; set_max_ft(&dev->root_ns->ns); return 0; err: return -ENOMEM; } u8 mlx5_get_match_criteria_enable(struct mlx5_flow_rule *rule) { struct fs_base *pbase; struct mlx5_flow_group *fg; pbase = rule->base.parent; WARN_ON(!pbase); pbase = pbase->parent; WARN_ON(!pbase); fs_get_obj(fg, pbase); return fg->mask.match_criteria_enable; } void mlx5_get_match_value(u32 *match_value, struct mlx5_flow_rule *rule) { struct fs_base *pbase; struct fs_fte *fte; pbase = rule->base.parent; WARN_ON(!pbase); fs_get_obj(fte, pbase); memcpy(match_value, fte->val, sizeof(fte->val)); } void mlx5_get_match_criteria(u32 *match_criteria, struct mlx5_flow_rule *rule) { struct fs_base *pbase; struct mlx5_flow_group *fg; pbase = rule->base.parent; WARN_ON(!pbase); pbase = pbase->parent; WARN_ON(!pbase); fs_get_obj(fg, pbase); memcpy(match_criteria, &fg->mask.match_criteria, sizeof(fg->mask.match_criteria)); } int mlx5_init_fs(struct mlx5_core_dev *dev) { int err; if (MLX5_CAP_GEN(dev, nic_flow_table)) { err = init_root_ns(dev); if (err) goto err; } err = init_fdb_root_ns(dev); if (err) goto err; err = init_egress_acl_root_ns(dev); if (err) goto err; err = init_ingress_acl_root_ns(dev); if (err) goto err; err = init_sniffer_tx_root_ns(dev); if (err) goto err; err = init_sniffer_rx_root_ns(dev); if (err) goto err; return 0; err: mlx5_cleanup_fs(dev); return err; } struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type) { struct mlx5_flow_root_namespace *root_ns = dev->root_ns; int prio; static struct fs_prio *fs_prio; struct mlx5_flow_namespace *ns; switch (type) { case MLX5_FLOW_NAMESPACE_BYPASS: prio = 0; break; case MLX5_FLOW_NAMESPACE_KERNEL: prio = 1; break; case MLX5_FLOW_NAMESPACE_LEFTOVERS: prio = 2; break; case MLX5_FLOW_NAMESPACE_FDB: if (dev->fdb_root_ns) return &dev->fdb_root_ns->ns; else return NULL; case MLX5_FLOW_NAMESPACE_ESW_EGRESS: if (dev->esw_egress_root_ns) return &dev->esw_egress_root_ns->ns; else return NULL; case MLX5_FLOW_NAMESPACE_ESW_INGRESS: if (dev->esw_ingress_root_ns) return &dev->esw_ingress_root_ns->ns; else return NULL; case MLX5_FLOW_NAMESPACE_SNIFFER_RX: if (dev->sniffer_rx_root_ns) return &dev->sniffer_rx_root_ns->ns; else return NULL; case MLX5_FLOW_NAMESPACE_SNIFFER_TX: if (dev->sniffer_tx_root_ns) return &dev->sniffer_tx_root_ns->ns; else return NULL; default: return NULL; } if (!root_ns) return NULL; fs_prio = find_prio(&root_ns->ns, prio); if (!fs_prio) return NULL; ns = list_first_entry(&fs_prio->objs, typeof(*ns), base.list); return ns; } EXPORT_SYMBOL(mlx5_get_flow_namespace); int mlx5_set_rule_private_data(struct mlx5_flow_rule *rule, struct mlx5_flow_handler *fs_handler, void *client_data) { struct fs_client_priv_data *priv_data; mutex_lock(&rule->clients_lock); /*Check that hanlder isn't exists in the list already*/ list_for_each_entry(priv_data, &rule->clients_data, list) { if (priv_data->fs_handler == fs_handler) { priv_data->client_dst_data = client_data; goto unlock; } } priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL); if (!priv_data) { mutex_unlock(&rule->clients_lock); return -ENOMEM; } priv_data->client_dst_data = client_data; priv_data->fs_handler = fs_handler; list_add(&priv_data->list, &rule->clients_data); unlock: mutex_unlock(&rule->clients_lock); return 0; } static int remove_from_clients(struct mlx5_flow_rule *rule, bool ctx_changed, void *client_data, void *context) { struct fs_client_priv_data *iter_client; struct fs_client_priv_data *temp_client; struct mlx5_flow_handler *handler = (struct mlx5_flow_handler*)context; mutex_lock(&rule->clients_lock); list_for_each_entry_safe(iter_client, temp_client, &rule->clients_data, list) { if (iter_client->fs_handler == handler) { list_del(&iter_client->list); kfree(iter_client); break; } } mutex_unlock(&rule->clients_lock); return 0; } struct mlx5_flow_handler *mlx5_register_rule_notifier(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type ns_type, rule_event_fn add_cb, rule_event_fn del_cb, void *context) { struct mlx5_flow_namespace *ns; struct mlx5_flow_handler *handler; ns = mlx5_get_flow_namespace(dev, ns_type); if (!ns) return ERR_PTR(-EINVAL); handler = kzalloc(sizeof(*handler), GFP_KERNEL); if (!handler) return ERR_PTR(-ENOMEM); handler->add_dst_cb = add_cb; handler->del_dst_cb = del_cb; handler->client_context = context; handler->ns = ns; down_write(&ns->notifiers_rw_sem); list_add_tail(&handler->list, &ns->list_notifiers); up_write(&ns->notifiers_rw_sem); return handler; } static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns, rule_event_fn add_rule_cb, void *context); void mlx5_unregister_rule_notifier(struct mlx5_flow_handler *handler) { struct mlx5_flow_namespace *ns = handler->ns; /*Remove from dst's clients*/ down_write(&ns->dests_rw_sem); down_write(&ns->notifiers_rw_sem); iterate_rules_in_ns(ns, remove_from_clients, handler); list_del(&handler->list); up_write(&ns->notifiers_rw_sem); up_write(&ns->dests_rw_sem); kfree(handler); } static void iterate_rules_in_ft(struct mlx5_flow_table *ft, rule_event_fn add_rule_cb, void *context) { struct mlx5_flow_group *iter_fg; struct fs_fte *iter_fte; struct mlx5_flow_rule *iter_rule; int err = 0; bool is_new_rule; mutex_lock(&ft->base.lock); fs_for_each_fg(iter_fg, ft) { mutex_lock(&iter_fg->base.lock); fs_for_each_fte(iter_fte, iter_fg) { mutex_lock(&iter_fte->base.lock); is_new_rule = true; fs_for_each_dst(iter_rule, iter_fte) { fs_get(&iter_rule->base); err = add_rule_cb(iter_rule, is_new_rule, NULL, context); fs_put_parent_locked(&iter_rule->base); if (err) break; is_new_rule = false; } mutex_unlock(&iter_fte->base.lock); if (err) break; } mutex_unlock(&iter_fg->base.lock); if (err) break; } mutex_unlock(&ft->base.lock); } static void iterate_rules_in_prio(struct fs_prio *prio, rule_event_fn add_rule_cb, void *context) { struct fs_base *it; mutex_lock(&prio->base.lock); fs_for_each_ns_or_ft(it, prio) { if (it->type == FS_TYPE_FLOW_TABLE) { struct mlx5_flow_table *ft; fs_get_obj(ft, it); iterate_rules_in_ft(ft, add_rule_cb, context); } else { struct mlx5_flow_namespace *ns; fs_get_obj(ns, it); iterate_rules_in_ns(ns, add_rule_cb, context); } } mutex_unlock(&prio->base.lock); } static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns, rule_event_fn add_rule_cb, void *context) { struct fs_prio *iter_prio; mutex_lock(&ns->base.lock); fs_for_each_prio(iter_prio, ns) { iterate_rules_in_prio(iter_prio, add_rule_cb, context); } mutex_unlock(&ns->base.lock); } void mlx5_flow_iterate_existing_rules(struct mlx5_flow_namespace *ns, rule_event_fn add_rule_cb, void *context) { down_write(&ns->dests_rw_sem); down_read(&ns->notifiers_rw_sem); iterate_rules_in_ns(ns, add_rule_cb, context); up_read(&ns->notifiers_rw_sem); up_write(&ns->dests_rw_sem); } void mlx5_del_flow_rules_list(struct mlx5_flow_rules_list *rules_list) { struct mlx5_flow_rule_node *iter_node; struct mlx5_flow_rule_node *temp_node; list_for_each_entry_safe(iter_node, temp_node, &rules_list->head, list) { list_del(&iter_node->list); kfree(iter_node); } kfree(rules_list); } #define ROCEV1_ETHERTYPE 0x8915 static int set_rocev1_rules(struct list_head *rules_list) { struct mlx5_flow_rule_node *rocev1_rule; rocev1_rule = kzalloc(sizeof(*rocev1_rule), GFP_KERNEL); if (!rocev1_rule) return -ENOMEM; rocev1_rule->match_criteria_enable = 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_criteria, ethertype, 0xffff); MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_value, ethertype, ROCEV1_ETHERTYPE); list_add_tail(&rocev1_rule->list, rules_list); return 0; } #define ROCEV2_UDP_PORT 4791 static int set_rocev2_rules(struct list_head *rules_list) { struct mlx5_flow_rule_node *ipv4_rule; struct mlx5_flow_rule_node *ipv6_rule; ipv4_rule = kzalloc(sizeof(*ipv4_rule), GFP_KERNEL); if (!ipv4_rule) return -ENOMEM; ipv6_rule = kzalloc(sizeof(*ipv6_rule), GFP_KERNEL); if (!ipv6_rule) { kfree(ipv4_rule); return -ENOMEM; } ipv4_rule->match_criteria_enable = 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ethertype, 0xffff); MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ethertype, 0x0800); MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ip_protocol, 0xff); MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ip_protocol, IPPROTO_UDP); MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, udp_dport, 0xffff); MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, udp_dport, ROCEV2_UDP_PORT); ipv6_rule->match_criteria_enable = 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ethertype, 0xffff); MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ethertype, 0x86dd); MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ip_protocol, 0xff); MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ip_protocol, IPPROTO_UDP); MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, udp_dport, 0xffff); MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, udp_dport, ROCEV2_UDP_PORT); list_add_tail(&ipv4_rule->list, rules_list); list_add_tail(&ipv6_rule->list, rules_list); return 0; } struct mlx5_flow_rules_list *get_roce_flow_rules(u8 roce_mode) { int err = 0; struct mlx5_flow_rules_list *rules_list = kzalloc(sizeof(*rules_list), GFP_KERNEL); if (!rules_list) return NULL; INIT_LIST_HEAD(&rules_list->head); if (roce_mode & MLX5_ROCE_VERSION_1_CAP) { err = set_rocev1_rules(&rules_list->head); if (err) goto free_list; } if (roce_mode & MLX5_ROCE_VERSION_2_CAP) err = set_rocev2_rules(&rules_list->head); if (err) goto free_list; return rules_list; free_list: mlx5_del_flow_rules_list(rules_list); return NULL; } Index: stable/11/sys/dev/mlx5/mlx5_core/mlx5_fw.c =================================================================== --- stable/11/sys/dev/mlx5/mlx5_core/mlx5_fw.c (revision 353223) +++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_fw.c (revision 353224) @@ -1,616 +1,616 @@ /*- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include "mlx5_core.h" static int mlx5_cmd_query_adapter(struct mlx5_core_dev *dev, u32 *out, int outlen) { u32 in[MLX5_ST_SZ_DW(query_adapter_in)]; int err; memset(in, 0, sizeof(in)); MLX5_SET(query_adapter_in, in, opcode, MLX5_CMD_OP_QUERY_ADAPTER); err = mlx5_cmd_exec(dev, in, sizeof(in), out, outlen); return err; } int mlx5_query_board_id(struct mlx5_core_dev *dev) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_adapter_out); int err; out = kzalloc(outlen, GFP_KERNEL); err = mlx5_cmd_query_adapter(dev, out, outlen); if (err) goto out_out; memcpy(dev->board_id, MLX5_ADDR_OF(query_adapter_out, out, query_adapter_struct.vsd_contd_psid), MLX5_FLD_SZ_BYTES(query_adapter_out, query_adapter_struct.vsd_contd_psid)); out_out: kfree(out); return err; } int mlx5_core_query_vendor_id(struct mlx5_core_dev *mdev, u32 *vendor_id) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_adapter_out); int err; out = kzalloc(outlen, GFP_KERNEL); err = mlx5_cmd_query_adapter(mdev, out, outlen); if (err) goto out_out; *vendor_id = MLX5_GET(query_adapter_out, out, query_adapter_struct.ieee_vendor_id); out_out: kfree(out); return err; } EXPORT_SYMBOL(mlx5_core_query_vendor_id); static int mlx5_core_query_special_contexts(struct mlx5_core_dev *dev) { u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)]; u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)]; int err; memset(in, 0, sizeof(in)); memset(out, 0, sizeof(out)); MLX5_SET(query_special_contexts_in, in, opcode, MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS); err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); if (err) return err; dev->special_contexts.resd_lkey = MLX5_GET(query_special_contexts_out, out, resd_lkey); return err; } static int mlx5_get_qcam_reg(struct mlx5_core_dev *dev) { return mlx5_query_qcam_reg(dev, dev->caps.qcam, MLX5_QCAM_FEATURE_ENHANCED_FEATURES, MLX5_QCAM_REGS_FIRST_128); } static int mlx5_get_pcam_reg(struct mlx5_core_dev *dev) { return mlx5_query_pcam_reg(dev, dev->caps.pcam, MLX5_PCAM_FEATURE_ENHANCED_FEATURES, MLX5_PCAM_REGS_5000_TO_507F); } static int mlx5_get_mcam_reg(struct mlx5_core_dev *dev) { return mlx5_query_mcam_reg(dev, dev->caps.mcam, MLX5_MCAM_FEATURE_ENHANCED_FEATURES, MLX5_MCAM_REGS_FIRST_128); } int mlx5_query_hca_caps(struct mlx5_core_dev *dev) { int err; err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL); if (err) return err; if (MLX5_CAP_GEN(dev, eth_net_offloads)) { err = mlx5_core_get_caps(dev, MLX5_CAP_ETHERNET_OFFLOADS); if (err) return err; } if (MLX5_CAP_GEN(dev, pg)) { err = mlx5_core_get_caps(dev, MLX5_CAP_ODP); if (err) return err; } if (MLX5_CAP_GEN(dev, atomic)) { err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC); if (err) return err; } if (MLX5_CAP_GEN(dev, roce)) { err = mlx5_core_get_caps(dev, MLX5_CAP_ROCE); if (err) return err; } if ((MLX5_CAP_GEN(dev, port_type) == MLX5_CMD_HCA_CAP_PORT_TYPE_ETHERNET && MLX5_CAP_GEN(dev, nic_flow_table)) || (MLX5_CAP_GEN(dev, port_type) == MLX5_CMD_HCA_CAP_PORT_TYPE_IB && MLX5_CAP_GEN(dev, ipoib_enhanced_offloads))) { err = mlx5_core_get_caps(dev, MLX5_CAP_FLOW_TABLE); if (err) return err; } if (MLX5_CAP_GEN(dev, eswitch_flow_table)) { err = mlx5_core_get_caps(dev, MLX5_CAP_ESWITCH_FLOW_TABLE); if (err) return err; } if (MLX5_CAP_GEN(dev, vport_group_manager)) { err = mlx5_core_get_caps(dev, MLX5_CAP_ESWITCH); if (err) return err; } if (MLX5_CAP_GEN(dev, snapshot)) { err = mlx5_core_get_caps(dev, MLX5_CAP_SNAPSHOT); if (err) return err; } if (MLX5_CAP_GEN(dev, ipoib_enhanced_offloads)) { err = mlx5_core_get_caps(dev, MLX5_CAP_EOIB_OFFLOADS); if (err) return err; } if (MLX5_CAP_GEN(dev, debug)) { err = mlx5_core_get_caps(dev, MLX5_CAP_DEBUG); if (err) return err; } if (MLX5_CAP_GEN(dev, qos)) { err = mlx5_core_get_caps(dev, MLX5_CAP_QOS); if (err) return err; } if (MLX5_CAP_GEN(dev, qcam_reg)) { err = mlx5_get_qcam_reg(dev); if (err) return err; } if (MLX5_CAP_GEN(dev, mcam_reg)) { err = mlx5_get_mcam_reg(dev); if (err) return err; } if (MLX5_CAP_GEN(dev, pcam_reg)) { err = mlx5_get_pcam_reg(dev); if (err) return err; } err = mlx5_core_query_special_contexts(dev); if (err) return err; return 0; } int mlx5_cmd_init_hca(struct mlx5_core_dev *dev) { u32 in[MLX5_ST_SZ_DW(init_hca_in)]; u32 out[MLX5_ST_SZ_DW(init_hca_out)]; memset(in, 0, sizeof(in)); MLX5_SET(init_hca_in, in, opcode, MLX5_CMD_OP_INIT_HCA); memset(out, 0, sizeof(out)); return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev) { u32 in[MLX5_ST_SZ_DW(teardown_hca_in)] = {0}; u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {0}; MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA); return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *dev) { u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {0}; u32 in[MLX5_ST_SZ_DW(teardown_hca_in)] = {0}; int force_state; int ret; if (!MLX5_CAP_GEN(dev, force_teardown)) { mlx5_core_dbg(dev, "force teardown is not supported in the firmware\n"); return -EOPNOTSUPP; } MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA); MLX5_SET(teardown_hca_in, in, profile, MLX5_TEARDOWN_HCA_IN_PROFILE_FORCE_CLOSE); ret = mlx5_cmd_exec_polling(dev, in, sizeof(in), out, sizeof(out)); if (ret) return ret; force_state = MLX5_GET(teardown_hca_out, out, state); if (force_state == MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_FAIL) { mlx5_core_err(dev, "teardown with force mode failed\n"); return -EIO; } return 0; } #define MLX5_FAST_TEARDOWN_WAIT_MS 3000 int mlx5_cmd_fast_teardown_hca(struct mlx5_core_dev *dev) { int end, delay_ms = MLX5_FAST_TEARDOWN_WAIT_MS; u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {}; u32 in[MLX5_ST_SZ_DW(teardown_hca_in)] = {}; int state; int ret; if (!MLX5_CAP_GEN(dev, fast_teardown)) { mlx5_core_dbg(dev, "fast teardown is not supported in the firmware\n"); return -EOPNOTSUPP; } MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA); MLX5_SET(teardown_hca_in, in, profile, MLX5_TEARDOWN_HCA_IN_PROFILE_PREPARE_FAST_TEARDOWN); ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); if (ret) return ret; state = MLX5_GET(teardown_hca_out, out, state); if (state == MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_FAIL) { mlx5_core_warn(dev, "teardown with fast mode failed\n"); return -EIO; } mlx5_set_nic_state(dev, MLX5_NIC_IFC_DISABLED); /* Loop until device state turns to disable */ end = jiffies + msecs_to_jiffies(delay_ms); do { if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) break; pause("W", 1); } while (!time_after(jiffies, end)); if (mlx5_get_nic_state(dev) != MLX5_NIC_IFC_DISABLED) { - dev_err(&dev->pdev->dev, "NIC IFC still %d after %ums.\n", + mlx5_core_err(dev, "NIC IFC still %d after %ums.\n", mlx5_get_nic_state(dev), delay_ms); return -EIO; } return 0; } int mlx5_core_set_dc_cnak_trace(struct mlx5_core_dev *dev, int enable, u64 addr) { u32 in[MLX5_ST_SZ_DW(set_dc_cnak_trace_in)] = {0}; u32 out[MLX5_ST_SZ_DW(set_dc_cnak_trace_out)] = {0}; __be64 be_addr; void *pas; MLX5_SET(set_dc_cnak_trace_in, in, opcode, MLX5_CMD_OP_SET_DC_CNAK_TRACE); MLX5_SET(set_dc_cnak_trace_in, in, enable, enable); pas = MLX5_ADDR_OF(set_dc_cnak_trace_in, in, pas); be_addr = cpu_to_be64(addr); memcpy(MLX5_ADDR_OF(cmd_pas, pas, pa_h), &be_addr, sizeof(be_addr)); return mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); } enum mlxsw_reg_mcc_instruction { MLX5_REG_MCC_INSTRUCTION_LOCK_UPDATE_HANDLE = 0x01, MLX5_REG_MCC_INSTRUCTION_RELEASE_UPDATE_HANDLE = 0x02, MLX5_REG_MCC_INSTRUCTION_UPDATE_COMPONENT = 0x03, MLX5_REG_MCC_INSTRUCTION_VERIFY_COMPONENT = 0x04, MLX5_REG_MCC_INSTRUCTION_ACTIVATE = 0x06, MLX5_REG_MCC_INSTRUCTION_CANCEL = 0x08, }; static int mlx5_reg_mcc_set(struct mlx5_core_dev *dev, enum mlxsw_reg_mcc_instruction instr, u16 component_index, u32 update_handle, u32 component_size) { u32 out[MLX5_ST_SZ_DW(mcc_reg)]; u32 in[MLX5_ST_SZ_DW(mcc_reg)]; memset(in, 0, sizeof(in)); MLX5_SET(mcc_reg, in, instruction, instr); MLX5_SET(mcc_reg, in, component_index, component_index); MLX5_SET(mcc_reg, in, update_handle, update_handle); MLX5_SET(mcc_reg, in, component_size, component_size); return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MCC, 0, 1); } static int mlx5_reg_mcc_query(struct mlx5_core_dev *dev, u32 *update_handle, u8 *error_code, u8 *control_state) { u32 out[MLX5_ST_SZ_DW(mcc_reg)]; u32 in[MLX5_ST_SZ_DW(mcc_reg)]; int err; memset(in, 0, sizeof(in)); memset(out, 0, sizeof(out)); MLX5_SET(mcc_reg, in, update_handle, *update_handle); err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MCC, 0, 0); if (err) goto out; *update_handle = MLX5_GET(mcc_reg, out, update_handle); *error_code = MLX5_GET(mcc_reg, out, error_code); *control_state = MLX5_GET(mcc_reg, out, control_state); out: return err; } static int mlx5_reg_mcda_set(struct mlx5_core_dev *dev, u32 update_handle, u32 offset, u16 size, u8 *data) { int err, in_size = MLX5_ST_SZ_BYTES(mcda_reg) + size; u32 out[MLX5_ST_SZ_DW(mcda_reg)]; int i, j, dw_size = size >> 2; __be32 data_element; u32 *in; in = kzalloc(in_size, GFP_KERNEL); if (!in) return -ENOMEM; MLX5_SET(mcda_reg, in, update_handle, update_handle); MLX5_SET(mcda_reg, in, offset, offset); MLX5_SET(mcda_reg, in, size, size); for (i = 0; i < dw_size; i++) { j = i * 4; data_element = htonl(*(u32 *)&data[j]); memcpy(MLX5_ADDR_OF(mcda_reg, in, data) + j, &data_element, 4); } err = mlx5_core_access_reg(dev, in, in_size, out, sizeof(out), MLX5_REG_MCDA, 0, 1); kfree(in); return err; } static int mlx5_reg_mcqi_query(struct mlx5_core_dev *dev, u16 component_index, u32 *max_component_size, u8 *log_mcda_word_size, u16 *mcda_max_write_size) { u32 out[MLX5_ST_SZ_DW(mcqi_reg) + MLX5_ST_SZ_DW(mcqi_cap)]; int offset = MLX5_ST_SZ_DW(mcqi_reg); u32 in[MLX5_ST_SZ_DW(mcqi_reg)]; int err; memset(in, 0, sizeof(in)); memset(out, 0, sizeof(out)); MLX5_SET(mcqi_reg, in, component_index, component_index); MLX5_SET(mcqi_reg, in, data_size, MLX5_ST_SZ_BYTES(mcqi_cap)); err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MCQI, 0, 0); if (err) goto out; *max_component_size = MLX5_GET(mcqi_cap, out + offset, max_component_size); *log_mcda_word_size = MLX5_GET(mcqi_cap, out + offset, log_mcda_word_size); *mcda_max_write_size = MLX5_GET(mcqi_cap, out + offset, mcda_max_write_size); out: return err; } struct mlx5_mlxfw_dev { struct mlxfw_dev mlxfw_dev; struct mlx5_core_dev *mlx5_core_dev; }; static int mlx5_component_query(struct mlxfw_dev *mlxfw_dev, u16 component_index, u32 *p_max_size, u8 *p_align_bits, u16 *p_max_write_size) { struct mlx5_mlxfw_dev *mlx5_mlxfw_dev = container_of(mlxfw_dev, struct mlx5_mlxfw_dev, mlxfw_dev); struct mlx5_core_dev *dev = mlx5_mlxfw_dev->mlx5_core_dev; return mlx5_reg_mcqi_query(dev, component_index, p_max_size, p_align_bits, p_max_write_size); } static int mlx5_fsm_lock(struct mlxfw_dev *mlxfw_dev, u32 *fwhandle) { struct mlx5_mlxfw_dev *mlx5_mlxfw_dev = container_of(mlxfw_dev, struct mlx5_mlxfw_dev, mlxfw_dev); struct mlx5_core_dev *dev = mlx5_mlxfw_dev->mlx5_core_dev; u8 control_state, error_code; int err; *fwhandle = 0; err = mlx5_reg_mcc_query(dev, fwhandle, &error_code, &control_state); if (err) return err; if (control_state != MLXFW_FSM_STATE_IDLE) return -EBUSY; return mlx5_reg_mcc_set(dev, MLX5_REG_MCC_INSTRUCTION_LOCK_UPDATE_HANDLE, 0, *fwhandle, 0); } static int mlx5_fsm_component_update(struct mlxfw_dev *mlxfw_dev, u32 fwhandle, u16 component_index, u32 component_size) { struct mlx5_mlxfw_dev *mlx5_mlxfw_dev = container_of(mlxfw_dev, struct mlx5_mlxfw_dev, mlxfw_dev); struct mlx5_core_dev *dev = mlx5_mlxfw_dev->mlx5_core_dev; return mlx5_reg_mcc_set(dev, MLX5_REG_MCC_INSTRUCTION_UPDATE_COMPONENT, component_index, fwhandle, component_size); } static int mlx5_fsm_block_download(struct mlxfw_dev *mlxfw_dev, u32 fwhandle, u8 *data, u16 size, u32 offset) { struct mlx5_mlxfw_dev *mlx5_mlxfw_dev = container_of(mlxfw_dev, struct mlx5_mlxfw_dev, mlxfw_dev); struct mlx5_core_dev *dev = mlx5_mlxfw_dev->mlx5_core_dev; return mlx5_reg_mcda_set(dev, fwhandle, offset, size, data); } static int mlx5_fsm_component_verify(struct mlxfw_dev *mlxfw_dev, u32 fwhandle, u16 component_index) { struct mlx5_mlxfw_dev *mlx5_mlxfw_dev = container_of(mlxfw_dev, struct mlx5_mlxfw_dev, mlxfw_dev); struct mlx5_core_dev *dev = mlx5_mlxfw_dev->mlx5_core_dev; return mlx5_reg_mcc_set(dev, MLX5_REG_MCC_INSTRUCTION_VERIFY_COMPONENT, component_index, fwhandle, 0); } static int mlx5_fsm_activate(struct mlxfw_dev *mlxfw_dev, u32 fwhandle) { struct mlx5_mlxfw_dev *mlx5_mlxfw_dev = container_of(mlxfw_dev, struct mlx5_mlxfw_dev, mlxfw_dev); struct mlx5_core_dev *dev = mlx5_mlxfw_dev->mlx5_core_dev; return mlx5_reg_mcc_set(dev, MLX5_REG_MCC_INSTRUCTION_ACTIVATE, 0, fwhandle, 0); } static int mlx5_fsm_query_state(struct mlxfw_dev *mlxfw_dev, u32 fwhandle, enum mlxfw_fsm_state *fsm_state, enum mlxfw_fsm_state_err *fsm_state_err) { struct mlx5_mlxfw_dev *mlx5_mlxfw_dev = container_of(mlxfw_dev, struct mlx5_mlxfw_dev, mlxfw_dev); struct mlx5_core_dev *dev = mlx5_mlxfw_dev->mlx5_core_dev; u8 control_state, error_code; int err; err = mlx5_reg_mcc_query(dev, &fwhandle, &error_code, &control_state); if (err) return err; *fsm_state = control_state; *fsm_state_err = min_t(enum mlxfw_fsm_state_err, error_code, MLXFW_FSM_STATE_ERR_MAX); return 0; } static void mlx5_fsm_cancel(struct mlxfw_dev *mlxfw_dev, u32 fwhandle) { struct mlx5_mlxfw_dev *mlx5_mlxfw_dev = container_of(mlxfw_dev, struct mlx5_mlxfw_dev, mlxfw_dev); struct mlx5_core_dev *dev = mlx5_mlxfw_dev->mlx5_core_dev; mlx5_reg_mcc_set(dev, MLX5_REG_MCC_INSTRUCTION_CANCEL, 0, fwhandle, 0); } static void mlx5_fsm_release(struct mlxfw_dev *mlxfw_dev, u32 fwhandle) { struct mlx5_mlxfw_dev *mlx5_mlxfw_dev = container_of(mlxfw_dev, struct mlx5_mlxfw_dev, mlxfw_dev); struct mlx5_core_dev *dev = mlx5_mlxfw_dev->mlx5_core_dev; mlx5_reg_mcc_set(dev, MLX5_REG_MCC_INSTRUCTION_RELEASE_UPDATE_HANDLE, 0, fwhandle, 0); } static const struct mlxfw_dev_ops mlx5_mlxfw_dev_ops = { .component_query = mlx5_component_query, .fsm_lock = mlx5_fsm_lock, .fsm_component_update = mlx5_fsm_component_update, .fsm_block_download = mlx5_fsm_block_download, .fsm_component_verify = mlx5_fsm_component_verify, .fsm_activate = mlx5_fsm_activate, .fsm_query_state = mlx5_fsm_query_state, .fsm_cancel = mlx5_fsm_cancel, .fsm_release = mlx5_fsm_release }; int mlx5_firmware_flash(struct mlx5_core_dev *dev, const struct firmware *firmware) { struct mlx5_mlxfw_dev mlx5_mlxfw_dev = { .mlxfw_dev = { .ops = &mlx5_mlxfw_dev_ops, .psid = dev->board_id, .psid_size = strlen(dev->board_id), }, .mlx5_core_dev = dev }; if (!MLX5_CAP_GEN(dev, mcam_reg) || !MLX5_CAP_MCAM_REG(dev, mcqi) || !MLX5_CAP_MCAM_REG(dev, mcc) || !MLX5_CAP_MCAM_REG(dev, mcda)) { pr_info("%s flashing isn't supported by the running FW\n", __func__); return -EOPNOTSUPP; } return mlxfw_firmware_flash(&mlx5_mlxfw_dev.mlxfw_dev, firmware); } Index: stable/11/sys/dev/mlx5/mlx5_core/mlx5_fwdump.c =================================================================== --- stable/11/sys/dev/mlx5/mlx5_core/mlx5_fwdump.c (revision 353223) +++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_fwdump.c (revision 353224) @@ -1,432 +1,432 @@ /*- * Copyright (c) 2018, 2019 Mellanox Technologies, Ltd. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include static MALLOC_DEFINE(M_MLX5_DUMP, "MLX5DUMP", "MLX5 Firmware dump"); static unsigned mlx5_fwdump_getsize(const struct mlx5_crspace_regmap *rege) { const struct mlx5_crspace_regmap *r; unsigned sz; for (sz = 0, r = rege; r->cnt != 0; r++) sz += r->cnt; return (sz); } static void mlx5_fwdump_destroy_dd(struct mlx5_core_dev *mdev) { mtx_assert(&mdev->dump_lock, MA_OWNED); free(mdev->dump_data, M_MLX5_DUMP); mdev->dump_data = NULL; } void mlx5_fwdump_prep(struct mlx5_core_dev *mdev) { device_t dev; int error, vsc_addr; unsigned i, sz; u32 addr, in, out, next_addr; mdev->dump_data = NULL; error = mlx5_vsc_find_cap(mdev); if (error != 0) { /* Inability to create a firmware dump is not fatal. */ - device_printf((&mdev->pdev->dev)->bsddev, "WARN: " + mlx5_core_warn(mdev, "mlx5_fwdump_prep failed %d\n", error); return; } error = mlx5_vsc_lock(mdev); if (error != 0) return; error = mlx5_vsc_set_space(mdev, MLX5_VSC_DOMAIN_SCAN_CRSPACE); if (error != 0) { mlx5_core_warn(mdev, "VSC scan space is not supported\n"); goto unlock_vsc; } dev = mdev->pdev->dev.bsddev; vsc_addr = mdev->vsc_addr; if (vsc_addr == 0) { mlx5_core_warn(mdev, "Cannot read vsc, no address\n"); goto unlock_vsc; } in = 0; for (sz = 1, addr = 0;;) { MLX5_VSC_SET(vsc_addr, &in, address, addr); pci_write_config(dev, vsc_addr + MLX5_VSC_ADDR_OFFSET, in, 4); error = mlx5_vsc_wait_on_flag(mdev, 1); if (error != 0) { mlx5_core_warn(mdev, "Failed waiting for read complete flag, error %d\n", error); goto unlock_vsc; } pci_read_config(dev, vsc_addr + MLX5_VSC_DATA_OFFSET, 4); out = pci_read_config(dev, vsc_addr + MLX5_VSC_ADDR_OFFSET, 4); next_addr = MLX5_VSC_GET(vsc_addr, &out, address); if (next_addr == 0 || next_addr == addr) break; if (next_addr != addr + 4) sz++; addr = next_addr; } mdev->dump_rege = malloc(sz * sizeof(struct mlx5_crspace_regmap), M_MLX5_DUMP, M_WAITOK | M_ZERO); for (i = 0, addr = 0;;) { MPASS(i < sz); mdev->dump_rege[i].cnt++; MLX5_VSC_SET(vsc_addr, &in, address, addr); pci_write_config(dev, vsc_addr + MLX5_VSC_ADDR_OFFSET, in, 4); error = mlx5_vsc_wait_on_flag(mdev, 1); if (error != 0) { mlx5_core_warn(mdev, "Failed waiting for read complete flag, error %d\n", error); free(mdev->dump_rege, M_MLX5_DUMP); mdev->dump_rege = NULL; goto unlock_vsc; } pci_read_config(dev, vsc_addr + MLX5_VSC_DATA_OFFSET, 4); out = pci_read_config(dev, vsc_addr + MLX5_VSC_ADDR_OFFSET, 4); next_addr = MLX5_VSC_GET(vsc_addr, &out, address); if (next_addr == 0 || next_addr == addr) break; if (next_addr != addr + 4) mdev->dump_rege[++i].addr = next_addr; addr = next_addr; } KASSERT(i + 1 == sz, ("inconsistent hw crspace reads: sz %u i %u addr %#lx", sz, i, (unsigned long)addr)); mdev->dump_size = mlx5_fwdump_getsize(mdev->dump_rege); mdev->dump_data = malloc(mdev->dump_size * sizeof(uint32_t), M_MLX5_DUMP, M_WAITOK | M_ZERO); mdev->dump_valid = false; mdev->dump_copyout = false; unlock_vsc: mlx5_vsc_unlock(mdev); } void mlx5_fwdump(struct mlx5_core_dev *mdev) { const struct mlx5_crspace_regmap *r; uint32_t i, ri; int error; - dev_info(&mdev->pdev->dev, "Issuing FW dump\n"); + mlx5_core_info(mdev, "Issuing FW dump\n"); mtx_lock(&mdev->dump_lock); if (mdev->dump_data == NULL) goto failed; if (mdev->dump_valid) { /* only one dump */ - dev_warn(&mdev->pdev->dev, + mlx5_core_warn(mdev, "Only one FW dump can be captured aborting FW dump\n"); goto failed; } /* mlx5_vsc already warns, be silent. */ error = mlx5_vsc_lock(mdev); if (error != 0) goto failed; error = mlx5_vsc_set_space(mdev, MLX5_VSC_DOMAIN_PROTECTED_CRSPACE); if (error != 0) goto unlock_vsc; for (i = 0, r = mdev->dump_rege; r->cnt != 0; r++) { for (ri = 0; ri < r->cnt; ri++) { error = mlx5_vsc_read(mdev, r->addr + ri * 4, &mdev->dump_data[i]); if (error != 0) goto unlock_vsc; i++; } } mdev->dump_valid = true; unlock_vsc: mlx5_vsc_unlock(mdev); failed: mtx_unlock(&mdev->dump_lock); } void mlx5_fwdump_clean(struct mlx5_core_dev *mdev) { mtx_lock(&mdev->dump_lock); while (mdev->dump_copyout) msleep(&mdev->dump_copyout, &mdev->dump_lock, 0, "mlx5fwc", 0); mlx5_fwdump_destroy_dd(mdev); mtx_unlock(&mdev->dump_lock); free(mdev->dump_rege, M_MLX5_DUMP); } static int mlx5_fwdump_reset(struct mlx5_core_dev *mdev) { int error; error = 0; mtx_lock(&mdev->dump_lock); if (mdev->dump_data != NULL) { while (mdev->dump_copyout) { msleep(&mdev->dump_copyout, &mdev->dump_lock, 0, "mlx5fwr", 0); } mdev->dump_valid = false; } else { error = ENOENT; } mtx_unlock(&mdev->dump_lock); return (error); } static int mlx5_dbsf_to_core(const struct mlx5_tool_addr *devaddr, struct mlx5_core_dev **mdev) { device_t dev; struct pci_dev *pdev; dev = pci_find_dbsf(devaddr->domain, devaddr->bus, devaddr->slot, devaddr->func); if (dev == NULL) return (ENOENT); if (device_get_devclass(dev) != mlx5_core_driver.bsdclass) return (EINVAL); pdev = device_get_softc(dev); *mdev = pci_get_drvdata(pdev); if (*mdev == NULL) return (ENOENT); return (0); } static int mlx5_fwdump_copyout(struct mlx5_core_dev *mdev, struct mlx5_fwdump_get *fwg) { const struct mlx5_crspace_regmap *r; struct mlx5_fwdump_reg rv, *urv; uint32_t i, ri; int error; mtx_lock(&mdev->dump_lock); if (mdev->dump_data == NULL) { mtx_unlock(&mdev->dump_lock); return (ENOENT); } if (fwg->buf == NULL) { fwg->reg_filled = mdev->dump_size; mtx_unlock(&mdev->dump_lock); return (0); } if (!mdev->dump_valid) { mtx_unlock(&mdev->dump_lock); return (ENOENT); } mdev->dump_copyout = true; mtx_unlock(&mdev->dump_lock); urv = fwg->buf; for (i = 0, r = mdev->dump_rege; r->cnt != 0; r++) { for (ri = 0; ri < r->cnt; ri++) { if (i >= fwg->reg_cnt) goto out; rv.addr = r->addr + ri * 4; rv.val = mdev->dump_data[i]; error = copyout(&rv, urv, sizeof(rv)); if (error != 0) return (error); urv++; i++; } } out: fwg->reg_filled = i; mtx_lock(&mdev->dump_lock); mdev->dump_copyout = false; wakeup(&mdev->dump_copyout); mtx_unlock(&mdev->dump_lock); return (0); } static int mlx5_fw_reset(struct mlx5_core_dev *mdev) { device_t dev, bus; int error; error = -mlx5_set_mfrl_reg(mdev, MLX5_FRL_LEVEL3); if (error == 0) { dev = mdev->pdev->dev.bsddev; mtx_lock(&Giant); bus = device_get_parent(dev); error = BUS_RESET_CHILD(device_get_parent(bus), bus, DEVF_RESET_DETACH); mtx_unlock(&Giant); } return (error); } static int mlx5_ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td) { struct mlx5_core_dev *mdev; struct mlx5_fwdump_get *fwg; struct mlx5_tool_addr *devaddr; struct mlx5_fw_update *fu; struct firmware fake_fw; int error; error = 0; switch (cmd) { case MLX5_FWDUMP_GET: if ((fflag & FREAD) == 0) { error = EBADF; break; } fwg = (struct mlx5_fwdump_get *)data; devaddr = &fwg->devaddr; error = mlx5_dbsf_to_core(devaddr, &mdev); if (error != 0) break; error = mlx5_fwdump_copyout(mdev, fwg); break; case MLX5_FWDUMP_RESET: if ((fflag & FWRITE) == 0) { error = EBADF; break; } devaddr = (struct mlx5_tool_addr *)data; error = mlx5_dbsf_to_core(devaddr, &mdev); if (error == 0) error = mlx5_fwdump_reset(mdev); break; case MLX5_FWDUMP_FORCE: if ((fflag & FWRITE) == 0) { error = EBADF; break; } devaddr = (struct mlx5_tool_addr *)data; error = mlx5_dbsf_to_core(devaddr, &mdev); if (error != 0) break; mlx5_fwdump(mdev); break; case MLX5_FW_UPDATE: if ((fflag & FWRITE) == 0) { error = EBADF; break; } fu = (struct mlx5_fw_update *)data; if (fu->img_fw_data_len > 10 * 1024 * 1024) { error = EINVAL; break; } devaddr = &fu->devaddr; error = mlx5_dbsf_to_core(devaddr, &mdev); if (error != 0) break; bzero(&fake_fw, sizeof(fake_fw)); fake_fw.name = "umlx_fw_up"; fake_fw.datasize = fu->img_fw_data_len; fake_fw.version = 1; fake_fw.data = (void *)kmem_malloc(kmem_arena, fu->img_fw_data_len, M_WAITOK); if (fake_fw.data == NULL) { error = ENOMEM; break; } error = copyin(fu->img_fw_data, __DECONST(void *, fake_fw.data), fu->img_fw_data_len); if (error == 0) error = -mlx5_firmware_flash(mdev, &fake_fw); kmem_free(kmem_arena, (vm_offset_t)fake_fw.data, fu->img_fw_data_len); break; case MLX5_FW_RESET: if ((fflag & FWRITE) == 0) { error = EBADF; break; } devaddr = (struct mlx5_tool_addr *)data; error = mlx5_dbsf_to_core(devaddr, &mdev); if (error != 0) break; error = mlx5_fw_reset(mdev); break; default: error = ENOTTY; break; } return (error); } static struct cdevsw mlx5_ctl_devsw = { .d_version = D_VERSION, .d_ioctl = mlx5_ctl_ioctl, }; static struct cdev *mlx5_ctl_dev; int mlx5_ctl_init(void) { struct make_dev_args mda; int error; make_dev_args_init(&mda); mda.mda_flags = MAKEDEV_WAITOK | MAKEDEV_CHECKNAME; mda.mda_devsw = &mlx5_ctl_devsw; mda.mda_uid = UID_ROOT; mda.mda_gid = GID_OPERATOR; mda.mda_mode = 0640; error = make_dev_s(&mda, &mlx5_ctl_dev, "mlx5ctl"); return (-error); } void mlx5_ctl_fini(void) { if (mlx5_ctl_dev != NULL) destroy_dev(mlx5_ctl_dev); } Index: stable/11/sys/dev/mlx5/mlx5_core/mlx5_health.c =================================================================== --- stable/11/sys/dev/mlx5/mlx5_core/mlx5_health.c (revision 353223) +++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_health.c (revision 353224) @@ -1,730 +1,751 @@ /*- * Copyright (c) 2013-2019, Mellanox Technologies, Ltd. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include #include #include #include #include #include "mlx5_core.h" #define MLX5_HEALTH_POLL_INTERVAL (2 * HZ) #define MAX_MISSES 3 enum { MLX5_DROP_NEW_HEALTH_WORK, MLX5_DROP_NEW_RECOVERY_WORK, MLX5_DROP_NEW_WATCHDOG_WORK, }; enum { MLX5_SENSOR_NO_ERR = 0, MLX5_SENSOR_PCI_COMM_ERR = 1, MLX5_SENSOR_PCI_ERR = 2, MLX5_SENSOR_NIC_DISABLED = 3, MLX5_SENSOR_NIC_SW_RESET = 4, MLX5_SENSOR_FW_SYND_RFR = 5, }; static int mlx5_fw_reset_enable = 1; SYSCTL_INT(_hw_mlx5, OID_AUTO, fw_reset_enable, CTLFLAG_RWTUN, &mlx5_fw_reset_enable, 0, "Enable firmware reset"); static unsigned int sw_reset_to = 1200; SYSCTL_UINT(_hw_mlx5, OID_AUTO, sw_reset_timeout, CTLFLAG_RWTUN, &sw_reset_to, 0, "Minimum timeout in seconds between two firmware resets"); static int lock_sem_sw_reset(struct mlx5_core_dev *dev) { int ret; /* Lock GW access */ ret = -mlx5_vsc_lock(dev); if (ret) { mlx5_core_warn(dev, "Timed out locking gateway %d\n", ret); return ret; } ret = -mlx5_vsc_lock_addr_space(dev, MLX5_SEMAPHORE_SW_RESET); if (ret) { if (ret == -EBUSY) - mlx5_core_dbg(dev, "SW reset FW semaphore already locked, another function will handle the reset\n"); + mlx5_core_dbg(dev, + "SW reset FW semaphore already locked, another function will handle the reset\n"); else - mlx5_core_warn(dev, "SW reset semaphore lock return %d\n", ret); + mlx5_core_warn(dev, + "SW reset semaphore lock return %d\n", ret); } /* Unlock GW access */ mlx5_vsc_unlock(dev); return ret; } static int unlock_sem_sw_reset(struct mlx5_core_dev *dev) { int ret; /* Lock GW access */ ret = -mlx5_vsc_lock(dev); if (ret) { mlx5_core_warn(dev, "Timed out locking gateway %d\n", ret); return ret; } ret = -mlx5_vsc_unlock_addr_space(dev, MLX5_SEMAPHORE_SW_RESET); /* Unlock GW access */ mlx5_vsc_unlock(dev); return ret; } u8 mlx5_get_nic_state(struct mlx5_core_dev *dev) { return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 7; } void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state) { u32 cur_cmdq_addr_l_sz; cur_cmdq_addr_l_sz = ioread32be(&dev->iseg->cmdq_addr_l_sz); iowrite32be((cur_cmdq_addr_l_sz & 0xFFFFF000) | state << MLX5_NIC_IFC_OFFSET, &dev->iseg->cmdq_addr_l_sz); } static bool sensor_fw_synd_rfr(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; struct mlx5_health_buffer __iomem *h = health->health; u32 rfr = ioread32be(&h->rfr) >> MLX5_RFR_OFFSET; u8 synd = ioread8(&h->synd); if (rfr && synd) mlx5_core_dbg(dev, "FW requests reset, synd: %d\n", synd); return rfr && synd; } static void mlx5_trigger_cmd_completions(struct work_struct *work) { struct mlx5_core_dev *dev = container_of(work, struct mlx5_core_dev, priv.health.work_cmd_completion); unsigned long flags; u64 vector; /* wait for pending handlers to complete */ synchronize_irq(dev->priv.msix_arr[MLX5_EQ_VEC_CMD].vector); spin_lock_irqsave(&dev->cmd.alloc_lock, flags); vector = ~dev->cmd.bitmask & ((1ul << (1 << dev->cmd.log_sz)) - 1); if (!vector) goto no_trig; vector |= MLX5_TRIGGERED_CMD_COMP; spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); mlx5_core_dbg(dev, "vector 0x%jx\n", (uintmax_t)vector); mlx5_cmd_comp_handler(dev, vector, MLX5_CMD_MODE_EVENTS); return; no_trig: spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); } static bool sensor_pci_no_comm(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; struct mlx5_health_buffer __iomem *h = health->health; bool err = ioread32be(&h->fw_ver) == 0xffffffff; return err; } static bool sensor_nic_disabled(struct mlx5_core_dev *dev) { return mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED; } static bool sensor_nic_sw_reset(struct mlx5_core_dev *dev) { return mlx5_get_nic_state(dev) == MLX5_NIC_IFC_SW_RESET; } static u32 check_fatal_sensors(struct mlx5_core_dev *dev) { if (sensor_pci_no_comm(dev)) return MLX5_SENSOR_PCI_COMM_ERR; if (pci_channel_offline(dev->pdev)) return MLX5_SENSOR_PCI_ERR; if (sensor_nic_disabled(dev)) return MLX5_SENSOR_NIC_DISABLED; if (sensor_nic_sw_reset(dev)) return MLX5_SENSOR_NIC_SW_RESET; if (sensor_fw_synd_rfr(dev)) return MLX5_SENSOR_FW_SYND_RFR; return MLX5_SENSOR_NO_ERR; } static void reset_fw_if_needed(struct mlx5_core_dev *dev) { bool supported; u32 cmdq_addr, fatal_error; if (!mlx5_fw_reset_enable) return; supported = (ioread32be(&dev->iseg->initializing) >> MLX5_FW_RESET_SUPPORTED_OFFSET) & 1; if (!supported) return; /* The reset only needs to be issued by one PF. The health buffer is * shared between all functions, and will be cleared during a reset. * Check again to avoid a redundant 2nd reset. If the fatal erros was * PCI related a reset won't help. */ fatal_error = check_fatal_sensors(dev); if (fatal_error == MLX5_SENSOR_PCI_COMM_ERR || fatal_error == MLX5_SENSOR_NIC_DISABLED || fatal_error == MLX5_SENSOR_NIC_SW_RESET) { - mlx5_core_warn(dev, "Not issuing FW reset. Either it's already done or won't help.\n"); + mlx5_core_warn(dev, + "Not issuing FW reset. Either it's already done or won't help.\n"); return; } - mlx5_core_warn(dev, "Issuing FW Reset\n"); + mlx5_core_info(dev, "Issuing FW Reset\n"); /* Write the NIC interface field to initiate the reset, the command * interface address also resides here, don't overwrite it. */ cmdq_addr = ioread32be(&dev->iseg->cmdq_addr_l_sz); iowrite32be((cmdq_addr & 0xFFFFF000) | MLX5_NIC_IFC_SW_RESET << MLX5_NIC_IFC_OFFSET, &dev->iseg->cmdq_addr_l_sz); } static bool mlx5_health_allow_reset(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; unsigned int delta; bool ret; if (health->last_reset_req != 0) { delta = ticks - health->last_reset_req; delta /= hz; ret = delta >= sw_reset_to; } else { ret = true; } /* * In principle, ticks may be 0. Setting it to off by one (-1) * to prevent certain reset in next request. */ health->last_reset_req = ticks ? : -1; if (!ret) - mlx5_core_warn(dev, "Firmware reset elided due to " - "auto-reset frequency threshold.\n"); + mlx5_core_warn(dev, + "Firmware reset elided due to auto-reset frequency threshold.\n"); return (ret); } #define MLX5_CRDUMP_WAIT_MS 60000 #define MLX5_FW_RESET_WAIT_MS 1000 #define MLX5_NIC_STATE_POLL_MS 5 void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force) { int end, delay_ms = MLX5_CRDUMP_WAIT_MS; u32 fatal_error; int lock = -EBUSY; fatal_error = check_fatal_sensors(dev); if (fatal_error || force) { if (xchg(&dev->state, MLX5_DEVICE_STATE_INTERNAL_ERROR) == MLX5_DEVICE_STATE_INTERNAL_ERROR) return; if (!force) mlx5_core_err(dev, "internal state error detected\n"); /* * Queue the command completion handler on the command * work queue to avoid racing with the real command * completion handler and then wait for it to * complete: */ queue_work(dev->priv.health.wq_cmd, &dev->priv.health.work_cmd_completion); flush_workqueue(dev->priv.health.wq_cmd); } mutex_lock(&dev->intf_state_mutex); if (force) goto err_state_done; if (fatal_error == MLX5_SENSOR_FW_SYND_RFR && mlx5_health_allow_reset(dev)) { /* Get cr-dump and reset FW semaphore */ if (mlx5_core_is_pf(dev)) lock = lock_sem_sw_reset(dev); /* Execute cr-dump and SW reset */ if (lock != -EBUSY) { mlx5_fwdump(dev); reset_fw_if_needed(dev); delay_ms = MLX5_FW_RESET_WAIT_MS; } } /* Recover from SW reset */ end = jiffies + msecs_to_jiffies(delay_ms); do { if (sensor_nic_disabled(dev)) break; msleep(MLX5_NIC_STATE_POLL_MS); } while (!time_after(jiffies, end)); if (!sensor_nic_disabled(dev)) { - dev_err(&dev->pdev->dev, "NIC IFC still %d after %ums.\n", + mlx5_core_err(dev, "NIC IFC still %d after %ums.\n", mlx5_get_nic_state(dev), delay_ms); } /* Release FW semaphore if you are the lock owner */ if (!lock) unlock_sem_sw_reset(dev); - mlx5_core_err(dev, "system error event triggered\n"); + mlx5_core_info(dev, "System error event triggered\n"); err_state_done: mlx5_core_event(dev, MLX5_DEV_EVENT_SYS_ERROR, 1); mutex_unlock(&dev->intf_state_mutex); } static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) { u8 nic_mode = mlx5_get_nic_state(dev); if (nic_mode == MLX5_NIC_IFC_SW_RESET) { /* The IFC mode field is 3 bits, so it will read 0x7 in two cases: * 1. PCI has been disabled (ie. PCI-AER, PF driver unloaded * and this is a VF), this is not recoverable by SW reset. * Logging of this is handled elsewhere. * 2. FW reset has been issued by another function, driver can * be reloaded to recover after the mode switches to * MLX5_NIC_IFC_DISABLED. */ if (dev->priv.health.fatal_error != MLX5_SENSOR_PCI_COMM_ERR) - mlx5_core_warn(dev, "NIC SW reset is already progress\n"); + mlx5_core_warn(dev, + "NIC SW reset is already progress\n"); else - mlx5_core_warn(dev, "Communication with FW over the PCI link is down\n"); + mlx5_core_warn(dev, + "Communication with FW over the PCI link is down\n"); } else { mlx5_core_warn(dev, "NIC mode %d\n", nic_mode); } mlx5_disable_device(dev); } #define MLX5_FW_RESET_WAIT_MS 1000 #define MLX5_NIC_STATE_POLL_MS 5 static void health_recover(struct work_struct *work) { unsigned long end = jiffies + msecs_to_jiffies(MLX5_FW_RESET_WAIT_MS); struct mlx5_core_health *health; struct delayed_work *dwork; struct mlx5_core_dev *dev; struct mlx5_priv *priv; bool recover = true; u8 nic_mode; dwork = container_of(work, struct delayed_work, work); health = container_of(dwork, struct mlx5_core_health, recover_work); priv = container_of(health, struct mlx5_priv, health); dev = container_of(priv, struct mlx5_core_dev, priv); mtx_lock(&Giant); /* XXX newbus needs this */ if (sensor_pci_no_comm(dev)) { - dev_err(&dev->pdev->dev, "health recovery flow aborted, PCI reads still not working\n"); + mlx5_core_err(dev, + "health recovery flow aborted, PCI reads still not working\n"); recover = false; } nic_mode = mlx5_get_nic_state(dev); while (nic_mode != MLX5_NIC_IFC_DISABLED && !time_after(jiffies, end)) { msleep(MLX5_NIC_STATE_POLL_MS); nic_mode = mlx5_get_nic_state(dev); } if (nic_mode != MLX5_NIC_IFC_DISABLED) { - dev_err(&dev->pdev->dev, "health recovery flow aborted, unexpected NIC IFC mode %d.\n", - nic_mode); + mlx5_core_err(dev, + "health recovery flow aborted, unexpected NIC IFC mode %d.\n", + nic_mode); recover = false; } if (recover) { - dev_err(&dev->pdev->dev, "starting health recovery flow\n"); + mlx5_core_info(dev, "Starting health recovery flow\n"); mlx5_recover_device(dev); } mtx_unlock(&Giant); } /* How much time to wait until health resetting the driver (in msecs) */ #define MLX5_RECOVERY_DELAY_MSECS 60000 #define MLX5_RECOVERY_NO_DELAY 0 static unsigned long get_recovery_delay(struct mlx5_core_dev *dev) { return dev->priv.health.fatal_error == MLX5_SENSOR_PCI_ERR || dev->priv.health.fatal_error == MLX5_SENSOR_PCI_COMM_ERR ? MLX5_RECOVERY_DELAY_MSECS : MLX5_RECOVERY_NO_DELAY; } static void health_care(struct work_struct *work) { struct mlx5_core_health *health; unsigned long recover_delay; struct mlx5_core_dev *dev; struct mlx5_priv *priv; unsigned long flags; health = container_of(work, struct mlx5_core_health, work); priv = container_of(health, struct mlx5_priv, health); dev = container_of(priv, struct mlx5_core_dev, priv); mlx5_core_warn(dev, "handling bad device here\n"); mlx5_handle_bad_state(dev); recover_delay = msecs_to_jiffies(get_recovery_delay(dev)); spin_lock_irqsave(&health->wq_lock, flags); if (!test_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags)) { - mlx5_core_warn(dev, "Scheduling recovery work with %lums delay\n", - recover_delay); + mlx5_core_warn(dev, + "Scheduling recovery work with %lums delay\n", + recover_delay); schedule_delayed_work(&health->recover_work, recover_delay); } else { - dev_err(&dev->pdev->dev, - "new health works are not permitted at this stage\n"); + mlx5_core_err(dev, + "new health works are not permitted at this stage\n"); } spin_unlock_irqrestore(&health->wq_lock, flags); } static int get_next_poll_jiffies(void) { unsigned long next; get_random_bytes(&next, sizeof(next)); next %= HZ; next += jiffies + MLX5_HEALTH_POLL_INTERVAL; return next; } void mlx5_trigger_health_work(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; unsigned long flags; spin_lock_irqsave(&health->wq_lock, flags); if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) queue_work(health->wq, &health->work); else - dev_err(&dev->pdev->dev, + mlx5_core_err(dev, "new health works are not permitted at this stage\n"); spin_unlock_irqrestore(&health->wq_lock, flags); } static const char *hsynd_str(u8 synd) { switch (synd) { case MLX5_HEALTH_SYNDR_FW_ERR: return "firmware internal error"; case MLX5_HEALTH_SYNDR_IRISC_ERR: return "irisc not responding"; case MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR: return "unrecoverable hardware error"; case MLX5_HEALTH_SYNDR_CRC_ERR: return "firmware CRC error"; case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR: return "ICM fetch PCI error"; case MLX5_HEALTH_SYNDR_HW_FTL_ERR: return "HW fatal error\n"; case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR: return "async EQ buffer overrun"; case MLX5_HEALTH_SYNDR_EQ_ERR: return "EQ error"; case MLX5_HEALTH_SYNDR_EQ_INV: return "Invalid EQ referenced"; case MLX5_HEALTH_SYNDR_FFSER_ERR: return "FFSER error"; case MLX5_HEALTH_SYNDR_HIGH_TEMP: return "High temprature"; default: return "unrecognized error"; } } static u8 print_health_info(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; struct mlx5_health_buffer __iomem *h = health->health; u8 synd = ioread8(&h->synd); char fw_str[18]; u32 fw; int i; /* * If synd is 0x0 - this indicates that FW is unable to * respond to initialization segment reads and health buffer * should not be read. */ if (synd == 0) return (0); for (i = 0; i < ARRAY_SIZE(h->assert_var); i++) - printf("mlx5_core: INFO: ""assert_var[%d] 0x%08x\n", i, ioread32be(h->assert_var + i)); + mlx5_core_info(dev, "assert_var[%d] 0x%08x\n", i, + ioread32be(h->assert_var + i)); - printf("mlx5_core: INFO: ""assert_exit_ptr 0x%08x\n", ioread32be(&h->assert_exit_ptr)); - printf("mlx5_core: INFO: ""assert_callra 0x%08x\n", ioread32be(&h->assert_callra)); - snprintf(fw_str, sizeof(fw_str), "%d.%d.%d", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev)); - printf("mlx5_core: INFO: ""fw_ver %s\n", fw_str); - printf("mlx5_core: INFO: ""hw_id 0x%08x\n", ioread32be(&h->hw_id)); - printf("mlx5_core: INFO: ""irisc_index %d\n", ioread8(&h->irisc_index)); - printf("mlx5_core: INFO: ""synd 0x%x: %s\n", synd, hsynd_str(synd)); - printf("mlx5_core: INFO: ""ext_synd 0x%04x\n", ioread16be(&h->ext_synd)); + mlx5_core_info(dev, "assert_exit_ptr 0x%08x\n", + ioread32be(&h->assert_exit_ptr)); + mlx5_core_info(dev, "assert_callra 0x%08x\n", + ioread32be(&h->assert_callra)); + snprintf(fw_str, sizeof(fw_str), "%d.%d.%d", + fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev)); + mlx5_core_info(dev, "fw_ver %s\n", fw_str); + mlx5_core_info(dev, "hw_id 0x%08x\n", ioread32be(&h->hw_id)); + mlx5_core_info(dev, "irisc_index %d\n", ioread8(&h->irisc_index)); + mlx5_core_info(dev, "synd 0x%x: %s\n", + ioread8(&h->synd), hsynd_str(ioread8(&h->synd))); + mlx5_core_info(dev, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd)); fw = ioread32be(&h->fw_ver); - printf("mlx5_core: INFO: ""raw fw_ver 0x%08x\n", fw); + mlx5_core_info(dev, "raw fw_ver 0x%08x\n", fw); return synd; } static void health_watchdog(struct work_struct *work) { struct mlx5_core_dev *dev; u16 power; u8 status; int err; dev = container_of(work, struct mlx5_core_dev, priv.health.work_watchdog); if (!MLX5_CAP_GEN(dev, mcam_reg) || !MLX5_CAP_MCAM_FEATURE(dev, pcie_status_and_power)) return; err = mlx5_pci_read_power_status(dev, &power, &status); if (err < 0) { - mlx5_core_warn(dev, "Failed reading power status: %d\n", err); + mlx5_core_warn(dev, "Failed reading power status: %d\n", + err); return; } dev->pwr_value = power; if (dev->pwr_status != status) { - device_t bsddev = dev->pdev->dev.bsddev; switch (status) { case 0: dev->pwr_status = status; - device_printf(bsddev, "PCI power is not published by the PCIe slot.\n"); + mlx5_core_info(dev, + "PCI power is not published by the PCIe slot.\n"); break; case 1: dev->pwr_status = status; - device_printf(bsddev, "PCIe slot advertised sufficient power (%uW).\n", power); + mlx5_core_info(dev, + "PCIe slot advertised sufficient power (%uW).\n", + power); break; case 2: dev->pwr_status = status; - device_printf(bsddev, "WARN: Detected insufficient power on the PCIe slot (%uW).\n", power); + mlx5_core_warn(dev, + "Detected insufficient power on the PCIe slot (%uW).\n", + power); break; default: dev->pwr_status = 0; - device_printf(bsddev, "WARN: Unknown power state detected(%d).\n", status); + mlx5_core_warn(dev, + "Unknown power state detected(%d).\n", + status); break; } } } void mlx5_trigger_health_watchdog(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; unsigned long flags; spin_lock_irqsave(&health->wq_lock, flags); if (!test_bit(MLX5_DROP_NEW_WATCHDOG_WORK, &health->flags)) queue_work(health->wq_watchdog, &health->work_watchdog); else - dev_err(&dev->pdev->dev, - "scheduling watchdog is not permitted at this stage\n"); + mlx5_core_err(dev, + "scheduling watchdog is not permitted at this stage\n"); spin_unlock_irqrestore(&health->wq_lock, flags); } static void poll_health(unsigned long data) { struct mlx5_core_dev *dev = (struct mlx5_core_dev *)data; struct mlx5_core_health *health = &dev->priv.health; u32 fatal_error; u32 count; if (dev->state != MLX5_DEVICE_STATE_UP) return; count = ioread32be(health->health_counter); if (count == health->prev) ++health->miss_counter; else health->miss_counter = 0; health->prev = count; if (health->miss_counter == MAX_MISSES) { mlx5_core_err(dev, "device's health compromised - reached miss count\n"); if (print_health_info(dev) == 0) mlx5_core_err(dev, "FW is unable to respond to initialization segment reads\n"); } fatal_error = check_fatal_sensors(dev); if (fatal_error && !health->fatal_error) { - mlx5_core_err(dev, "Fatal error %u detected\n", fatal_error); + mlx5_core_err(dev, + "Fatal error %u detected\n", fatal_error); dev->priv.health.fatal_error = fatal_error; print_health_info(dev); mlx5_trigger_health_work(dev); } mod_timer(&health->timer, get_next_poll_jiffies()); } void mlx5_start_health_poll(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; init_timer(&health->timer); health->fatal_error = MLX5_SENSOR_NO_ERR; clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); clear_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags); clear_bit(MLX5_DROP_NEW_WATCHDOG_WORK, &health->flags); health->health = &dev->iseg->health; health->health_counter = &dev->iseg->health_counter; setup_timer(&health->timer, poll_health, (unsigned long)dev); mod_timer(&health->timer, round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL)); /* do initial PCI power state readout */ mlx5_trigger_health_watchdog(dev); } void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health) { struct mlx5_core_health *health = &dev->priv.health; unsigned long flags; if (disable_health) { spin_lock_irqsave(&health->wq_lock, flags); set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags); set_bit(MLX5_DROP_NEW_WATCHDOG_WORK, &health->flags); spin_unlock_irqrestore(&health->wq_lock, flags); } del_timer_sync(&health->timer); } void mlx5_drain_health_wq(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; unsigned long flags; spin_lock_irqsave(&health->wq_lock, flags); set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags); set_bit(MLX5_DROP_NEW_WATCHDOG_WORK, &health->flags); spin_unlock_irqrestore(&health->wq_lock, flags); cancel_delayed_work_sync(&health->recover_work); cancel_work_sync(&health->work); cancel_work_sync(&health->work_watchdog); } void mlx5_drain_health_recovery(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; unsigned long flags; spin_lock_irqsave(&health->wq_lock, flags); set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags); spin_unlock_irqrestore(&health->wq_lock, flags); cancel_delayed_work_sync(&dev->priv.health.recover_work); } void mlx5_health_cleanup(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; destroy_workqueue(health->wq); destroy_workqueue(health->wq_watchdog); destroy_workqueue(health->wq_cmd); } int mlx5_health_init(struct mlx5_core_dev *dev) { struct mlx5_core_health *health; char name[64]; health = &dev->priv.health; snprintf(name, sizeof(name), "%s-rec", dev_name(&dev->pdev->dev)); health->wq = create_singlethread_workqueue(name); if (!health->wq) goto err_recovery; snprintf(name, sizeof(name), "%s-wdg", dev_name(&dev->pdev->dev)); health->wq_watchdog = create_singlethread_workqueue(name); if (!health->wq_watchdog) goto err_watchdog; snprintf(name, sizeof(name), "%s-cmd", dev_name(&dev->pdev->dev)); health->wq_cmd = create_singlethread_workqueue(name); if (!health->wq_cmd) goto err_cmd; spin_lock_init(&health->wq_lock); INIT_WORK(&health->work, health_care); INIT_WORK(&health->work_watchdog, health_watchdog); INIT_WORK(&health->work_cmd_completion, mlx5_trigger_cmd_completions); INIT_DELAYED_WORK(&health->recover_work, health_recover); return 0; err_cmd: destroy_workqueue(health->wq_watchdog); err_watchdog: destroy_workqueue(health->wq); err_recovery: return -ENOMEM; } Index: stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c =================================================================== --- stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c (revision 353223) +++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c (revision 353224) @@ -1,1655 +1,1659 @@ /*- * Copyright (c) 2013-2019, Mellanox Technologies, Ltd. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "mlx5_core.h" #include "fs_core.h" static const char mlx5_version[] = "Mellanox Core driver " DRIVER_VERSION " (" DRIVER_RELDATE ")"; MODULE_AUTHOR("Eli Cohen "); MODULE_DESCRIPTION("Mellanox Connect-IB, ConnectX-4 core driver"); MODULE_LICENSE("Dual BSD/GPL"); MODULE_DEPEND(mlx5, linuxkpi, 1, 1, 1); MODULE_DEPEND(mlx5, mlxfw, 1, 1, 1); MODULE_DEPEND(mlx5, firmware, 1, 1, 1); MODULE_VERSION(mlx5, 1); SYSCTL_NODE(_hw, OID_AUTO, mlx5, CTLFLAG_RW, 0, "mlx5 hardware controls"); int mlx5_core_debug_mask; SYSCTL_INT(_hw_mlx5, OID_AUTO, debug_mask, CTLFLAG_RWTUN, &mlx5_core_debug_mask, 0, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0"); #define MLX5_DEFAULT_PROF 2 static int mlx5_prof_sel = MLX5_DEFAULT_PROF; SYSCTL_INT(_hw_mlx5, OID_AUTO, prof_sel, CTLFLAG_RWTUN, &mlx5_prof_sel, 0, "profile selector. Valid range 0 - 2"); static int mlx5_fast_unload_enabled = 1; SYSCTL_INT(_hw_mlx5, OID_AUTO, fast_unload_enabled, CTLFLAG_RWTUN, &mlx5_fast_unload_enabled, 0, "Set to enable fast unload. Clear to disable."); #define NUMA_NO_NODE -1 static LIST_HEAD(intf_list); static LIST_HEAD(dev_list); static DEFINE_MUTEX(intf_mutex); struct mlx5_device_context { struct list_head list; struct mlx5_interface *intf; void *context; }; enum { MLX5_ATOMIC_REQ_MODE_BE = 0x0, MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1, }; static struct mlx5_profile profiles[] = { [0] = { .mask = 0, }, [1] = { .mask = MLX5_PROF_MASK_QP_SIZE, .log_max_qp = 12, }, [2] = { .mask = MLX5_PROF_MASK_QP_SIZE | MLX5_PROF_MASK_MR_CACHE, .log_max_qp = 17, .mr_cache[0] = { .size = 500, .limit = 250 }, .mr_cache[1] = { .size = 500, .limit = 250 }, .mr_cache[2] = { .size = 500, .limit = 250 }, .mr_cache[3] = { .size = 500, .limit = 250 }, .mr_cache[4] = { .size = 500, .limit = 250 }, .mr_cache[5] = { .size = 500, .limit = 250 }, .mr_cache[6] = { .size = 500, .limit = 250 }, .mr_cache[7] = { .size = 500, .limit = 250 }, .mr_cache[8] = { .size = 500, .limit = 250 }, .mr_cache[9] = { .size = 500, .limit = 250 }, .mr_cache[10] = { .size = 500, .limit = 250 }, .mr_cache[11] = { .size = 500, .limit = 250 }, .mr_cache[12] = { .size = 64, .limit = 32 }, .mr_cache[13] = { .size = 32, .limit = 16 }, .mr_cache[14] = { .size = 16, .limit = 8 }, }, [3] = { .mask = MLX5_PROF_MASK_QP_SIZE, .log_max_qp = 17, }, }; static int set_dma_caps(struct pci_dev *pdev) { + struct mlx5_core_dev *dev = pci_get_drvdata(pdev); int err; err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); if (err) { - device_printf((&pdev->dev)->bsddev, "WARN: ""Warning: couldn't set 64-bit PCI DMA mask\n"); + mlx5_core_warn(dev, "couldn't set 64-bit PCI DMA mask\n"); err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""Can't set PCI DMA mask, aborting\n"); + mlx5_core_err(dev, "Can't set PCI DMA mask, aborting\n"); return err; } } err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); if (err) { - device_printf((&pdev->dev)->bsddev, "WARN: ""Warning: couldn't set 64-bit consistent PCI DMA mask\n"); + mlx5_core_warn(dev, "couldn't set 64-bit consistent PCI DMA mask\n"); err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""Can't set consistent PCI DMA mask, aborting\n"); + mlx5_core_err(dev, "Can't set consistent PCI DMA mask, aborting\n"); return err; } } dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); return err; } int mlx5_pci_read_power_status(struct mlx5_core_dev *dev, u16 *p_power, u8 *p_status) { u32 in[MLX5_ST_SZ_DW(mpein_reg)] = {}; u32 out[MLX5_ST_SZ_DW(mpein_reg)] = {}; int err; err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_ACCESS_REG_SUMMARY_CTRL_ID_MPEIN, 0, 0); *p_status = MLX5_GET(mpein_reg, out, pwr_status); *p_power = MLX5_GET(mpein_reg, out, pci_power); return err; } static int mlx5_pci_enable_device(struct mlx5_core_dev *dev) { struct pci_dev *pdev = dev->pdev; int err = 0; mutex_lock(&dev->pci_status_mutex); if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) { err = pci_enable_device(pdev); if (!err) dev->pci_status = MLX5_PCI_STATUS_ENABLED; } mutex_unlock(&dev->pci_status_mutex); return err; } static void mlx5_pci_disable_device(struct mlx5_core_dev *dev) { struct pci_dev *pdev = dev->pdev; mutex_lock(&dev->pci_status_mutex); if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) { pci_disable_device(pdev); dev->pci_status = MLX5_PCI_STATUS_DISABLED; } mutex_unlock(&dev->pci_status_mutex); } static int request_bar(struct pci_dev *pdev) { + struct mlx5_core_dev *dev = pci_get_drvdata(pdev); int err = 0; if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { - device_printf((&pdev->dev)->bsddev, "ERR: ""Missing registers BAR, aborting\n"); + mlx5_core_err(dev, "Missing registers BAR, aborting\n"); return -ENODEV; } err = pci_request_regions(pdev, DRIVER_NAME); if (err) - device_printf((&pdev->dev)->bsddev, "ERR: ""Couldn't get PCI resources, aborting\n"); + mlx5_core_err(dev, "Couldn't get PCI resources, aborting\n"); return err; } static void release_bar(struct pci_dev *pdev) { pci_release_regions(pdev); } static int mlx5_enable_msix(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; struct mlx5_eq_table *table = &priv->eq_table; int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq); int limit = dev->msix_eqvec; int nvec = MLX5_EQ_VEC_COMP_BASE; int i; if (limit > 0) nvec += limit; else nvec += MLX5_CAP_GEN(dev, num_ports) * num_online_cpus(); if (nvec > num_eqs) nvec = num_eqs; if (nvec > 256) nvec = 256; /* limit of firmware API */ if (nvec <= MLX5_EQ_VEC_COMP_BASE) return -ENOMEM; priv->msix_arr = kzalloc(nvec * sizeof(*priv->msix_arr), GFP_KERNEL); for (i = 0; i < nvec; i++) priv->msix_arr[i].entry = i; nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr, MLX5_EQ_VEC_COMP_BASE + 1, nvec); if (nvec < 0) return nvec; table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE; return 0; } static void mlx5_disable_msix(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; pci_disable_msix(dev->pdev); kfree(priv->msix_arr); } struct mlx5_reg_host_endianess { u8 he; u8 rsvd[15]; }; #define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) enum { MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | MLX5_DEV_CAP_FLAG_DCT | MLX5_DEV_CAP_FLAG_DRAIN_SIGERR, }; -static u16 to_fw_pkey_sz(u32 size) +static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size) { switch (size) { case 128: return 0; case 256: return 1; case 512: return 2; case 1024: return 3; case 2048: return 4; case 4096: return 5; default: - printf("mlx5_core: WARN: ""invalid pkey table size %d\n", size); + mlx5_core_warn(dev, "invalid pkey table size %d\n", size); return 0; } } static int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type, enum mlx5_cap_mode cap_mode) { u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)]; int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); void *out, *hca_caps; u16 opmod = (cap_type << 1) | (cap_mode & 0x01); int err; memset(in, 0, sizeof(in)); out = kzalloc(out_sz, GFP_KERNEL); MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); MLX5_SET(query_hca_cap_in, in, op_mod, opmod); err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); if (err) { mlx5_core_warn(dev, "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n", cap_type, cap_mode, err); goto query_ex; } hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability); switch (cap_mode) { case HCA_CAP_OPMOD_GET_MAX: memcpy(dev->hca_caps_max[cap_type], hca_caps, MLX5_UN_SZ_BYTES(hca_cap_union)); break; case HCA_CAP_OPMOD_GET_CUR: memcpy(dev->hca_caps_cur[cap_type], hca_caps, MLX5_UN_SZ_BYTES(hca_cap_union)); break; default: mlx5_core_warn(dev, "Tried to query dev cap type(%x) with wrong opmode(%x)\n", cap_type, cap_mode); err = -EINVAL; break; } query_ex: kfree(out); return err; } int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type) { int ret; ret = mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_CUR); if (ret) return ret; return mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_MAX); } static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz) { u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)] = {0}; MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP); return mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out)); } static int handle_hca_cap(struct mlx5_core_dev *dev) { void *set_ctx = NULL; struct mlx5_profile *prof = dev->profile; int err = -ENOMEM; int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); void *set_hca_cap; set_ctx = kzalloc(set_sz, GFP_KERNEL); err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL); if (err) goto query_ex; set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); memcpy(set_hca_cap, dev->hca_caps_cur[MLX5_CAP_GENERAL], MLX5_ST_SZ_BYTES(cmd_hca_cap)); mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n", mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)), 128); /* we limit the size of the pkey table to 128 entries for now */ MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size, - to_fw_pkey_sz(128)); + to_fw_pkey_sz(dev, 128)); if (prof->mask & MLX5_PROF_MASK_QP_SIZE) MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, prof->log_max_qp); /* disable cmdif checksum */ MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0); /* enable drain sigerr */ MLX5_SET(cmd_hca_cap, set_hca_cap, drain_sigerr, 1); MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12); err = set_caps(dev, set_ctx, set_sz); query_ex: kfree(set_ctx); return err; } static int handle_hca_cap_atomic(struct mlx5_core_dev *dev) { void *set_ctx; void *set_hca_cap; int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); int req_endianness; int err; if (MLX5_CAP_GEN(dev, atomic)) { err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC); if (err) return err; } else { return 0; } req_endianness = MLX5_CAP_ATOMIC(dev, supported_atomic_req_8B_endianess_mode_1); if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS) return 0; set_ctx = kzalloc(set_sz, GFP_KERNEL); if (!set_ctx) return -ENOMEM; MLX5_SET(set_hca_cap_in, set_ctx, op_mod, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC << 1); set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); /* Set requestor to host endianness */ MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianess_mode, MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS); err = set_caps(dev, set_ctx, set_sz); kfree(set_ctx); return err; } static int set_hca_ctrl(struct mlx5_core_dev *dev) { struct mlx5_reg_host_endianess he_in; struct mlx5_reg_host_endianess he_out; int err; if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH && !MLX5_CAP_GEN(dev, roce)) return 0; memset(&he_in, 0, sizeof(he_in)); he_in.he = MLX5_SET_HOST_ENDIANNESS; err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in), &he_out, sizeof(he_out), MLX5_REG_HOST_ENDIANNESS, 0, 1); return err; } static int mlx5_core_enable_hca(struct mlx5_core_dev *dev) { u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {0}; u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {0}; MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA); return mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); } static int mlx5_core_disable_hca(struct mlx5_core_dev *dev) { u32 out[MLX5_ST_SZ_DW(disable_hca_out)] = {0}; u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {0}; MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA); return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } static int mlx5_core_set_issi(struct mlx5_core_dev *dev) { u32 query_in[MLX5_ST_SZ_DW(query_issi_in)] = {0}; u32 query_out[MLX5_ST_SZ_DW(query_issi_out)] = {0}; u32 sup_issi; int err; MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI); err = mlx5_cmd_exec(dev, query_in, sizeof(query_in), query_out, sizeof(query_out)); if (err) { u32 syndrome; u8 status; mlx5_cmd_mbox_status(query_out, &status, &syndrome); if (status == MLX5_CMD_STAT_BAD_OP_ERR) { - pr_debug("Only ISSI 0 is supported\n"); + mlx5_core_dbg(dev, "Only ISSI 0 is supported\n"); return 0; } - printf("mlx5_core: ERR: ""failed to query ISSI\n"); + mlx5_core_err(dev, "failed to query ISSI\n"); return err; } sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0); if (sup_issi & (1 << 1)) { u32 set_in[MLX5_ST_SZ_DW(set_issi_in)] = {0}; u32 set_out[MLX5_ST_SZ_DW(set_issi_out)] = {0}; MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI); MLX5_SET(set_issi_in, set_in, current_issi, 1); err = mlx5_cmd_exec(dev, set_in, sizeof(set_in), set_out, sizeof(set_out)); if (err) { - printf("mlx5_core: ERR: ""failed to set ISSI=1 err(%d)\n", err); + mlx5_core_err(dev, "failed to set ISSI=1 err(%d)\n", err); return err; } dev->issi = 1; return 0; } else if (sup_issi & (1 << 0)) { return 0; } return -ENOTSUPP; } int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, int *irqn) { struct mlx5_eq_table *table = &dev->priv.eq_table; struct mlx5_eq *eq; int err = -ENOENT; spin_lock(&table->lock); list_for_each_entry(eq, &table->comp_eqs_list, list) { if (eq->index == vector) { *eqn = eq->eqn; *irqn = eq->irqn; err = 0; break; } } spin_unlock(&table->lock); return err; } EXPORT_SYMBOL(mlx5_vector2eqn); static void free_comp_eqs(struct mlx5_core_dev *dev) { struct mlx5_eq_table *table = &dev->priv.eq_table; struct mlx5_eq *eq, *n; spin_lock(&table->lock); list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { list_del(&eq->list); spin_unlock(&table->lock); if (mlx5_destroy_unmap_eq(dev, eq)) mlx5_core_warn(dev, "failed to destroy EQ 0x%x\n", eq->eqn); kfree(eq); spin_lock(&table->lock); } spin_unlock(&table->lock); } static int alloc_comp_eqs(struct mlx5_core_dev *dev) { struct mlx5_eq_table *table = &dev->priv.eq_table; struct mlx5_eq *eq; int ncomp_vec; int nent; int err; int i; INIT_LIST_HEAD(&table->comp_eqs_list); ncomp_vec = table->num_comp_vectors; nent = MLX5_COMP_EQ_SIZE; for (i = 0; i < ncomp_vec; i++) { eq = kzalloc(sizeof(*eq), GFP_KERNEL); err = mlx5_create_map_eq(dev, eq, i + MLX5_EQ_VEC_COMP_BASE, nent, 0, &dev->priv.uuari.uars[0]); if (err) { kfree(eq); goto clean; } mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->eqn); eq->index = i; spin_lock(&table->lock); list_add_tail(&eq->list, &table->comp_eqs_list); spin_unlock(&table->lock); } return 0; clean: free_comp_eqs(dev); return err; } static int map_bf_area(struct mlx5_core_dev *dev) { resource_size_t bf_start = pci_resource_start(dev->pdev, 0); resource_size_t bf_len = pci_resource_len(dev->pdev, 0); dev->priv.bf_mapping = io_mapping_create_wc(bf_start, bf_len); return dev->priv.bf_mapping ? 0 : -ENOMEM; } static void unmap_bf_area(struct mlx5_core_dev *dev) { if (dev->priv.bf_mapping) io_mapping_free(dev->priv.bf_mapping); } static inline int fw_initializing(struct mlx5_core_dev *dev) { return ioread32be(&dev->iseg->initializing) >> 31; } static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili) { u64 end = jiffies + msecs_to_jiffies(max_wait_mili); int err = 0; while (fw_initializing(dev)) { if (time_after(jiffies, end)) { err = -EBUSY; break; } msleep(FW_INIT_WAIT_MS); } return err; } static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv) { struct mlx5_device_context *dev_ctx; struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); dev_ctx = kzalloc(sizeof(*dev_ctx), GFP_KERNEL); if (!dev_ctx) return; dev_ctx->intf = intf; CURVNET_SET_QUIET(vnet0); dev_ctx->context = intf->add(dev); CURVNET_RESTORE(); if (dev_ctx->context) { spin_lock_irq(&priv->ctx_lock); list_add_tail(&dev_ctx->list, &priv->ctx_list); spin_unlock_irq(&priv->ctx_lock); } else { kfree(dev_ctx); } } static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv) { struct mlx5_device_context *dev_ctx; struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); list_for_each_entry(dev_ctx, &priv->ctx_list, list) if (dev_ctx->intf == intf) { spin_lock_irq(&priv->ctx_lock); list_del(&dev_ctx->list); spin_unlock_irq(&priv->ctx_lock); intf->remove(dev, dev_ctx->context); kfree(dev_ctx); return; } } int mlx5_register_device(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; struct mlx5_interface *intf; mutex_lock(&intf_mutex); list_add_tail(&priv->dev_list, &dev_list); list_for_each_entry(intf, &intf_list, list) mlx5_add_device(intf, priv); mutex_unlock(&intf_mutex); return 0; } void mlx5_unregister_device(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; struct mlx5_interface *intf; mutex_lock(&intf_mutex); list_for_each_entry(intf, &intf_list, list) mlx5_remove_device(intf, priv); list_del(&priv->dev_list); mutex_unlock(&intf_mutex); } int mlx5_register_interface(struct mlx5_interface *intf) { struct mlx5_priv *priv; if (!intf->add || !intf->remove) return -EINVAL; mutex_lock(&intf_mutex); list_add_tail(&intf->list, &intf_list); list_for_each_entry(priv, &dev_list, dev_list) mlx5_add_device(intf, priv); mutex_unlock(&intf_mutex); return 0; } EXPORT_SYMBOL(mlx5_register_interface); void mlx5_unregister_interface(struct mlx5_interface *intf) { struct mlx5_priv *priv; mutex_lock(&intf_mutex); list_for_each_entry(priv, &dev_list, dev_list) mlx5_remove_device(intf, priv); list_del(&intf->list); mutex_unlock(&intf_mutex); } EXPORT_SYMBOL(mlx5_unregister_interface); void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol) { struct mlx5_priv *priv = &mdev->priv; struct mlx5_device_context *dev_ctx; unsigned long flags; void *result = NULL; spin_lock_irqsave(&priv->ctx_lock, flags); list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list) if ((dev_ctx->intf->protocol == protocol) && dev_ctx->intf->get_dev) { result = dev_ctx->intf->get_dev(dev_ctx->context); break; } spin_unlock_irqrestore(&priv->ctx_lock, flags); return result; } EXPORT_SYMBOL(mlx5_get_protocol_dev); static int mlx5_auto_fw_update; SYSCTL_INT(_hw_mlx5, OID_AUTO, auto_fw_update, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &mlx5_auto_fw_update, 0, "Allow automatic firmware update on driver start"); static int mlx5_firmware_update(struct mlx5_core_dev *dev) { const struct firmware *fw; int err; TUNABLE_INT_FETCH("hw.mlx5.auto_fw_update", &mlx5_auto_fw_update); if (!mlx5_auto_fw_update) return (0); fw = firmware_get("mlx5fw_mfa"); if (fw) { err = mlx5_firmware_flash(dev, fw); firmware_put(fw, FIRMWARE_UNLOAD); } else return (-ENOENT); return err; } static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv) { struct pci_dev *pdev = dev->pdev; int err = 0; pci_set_drvdata(dev->pdev, dev); strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN); priv->name[MLX5_MAX_NAME_LEN - 1] = 0; mutex_init(&priv->pgdir_mutex); INIT_LIST_HEAD(&priv->pgdir_list); spin_lock_init(&priv->mkey_lock); priv->numa_node = NUMA_NO_NODE; err = mlx5_pci_enable_device(dev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""Cannot enable PCI device, aborting\n"); + mlx5_core_err(dev, "Cannot enable PCI device, aborting\n"); goto err_dbg; } err = request_bar(pdev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""error requesting BARs, aborting\n"); + mlx5_core_err(dev, "error requesting BARs, aborting\n"); goto err_disable; } pci_set_master(pdev); err = set_dma_caps(pdev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""Failed setting DMA capabilities mask, aborting\n"); + mlx5_core_err(dev, "Failed setting DMA capabilities mask, aborting\n"); goto err_clr_master; } dev->iseg_base = pci_resource_start(dev->pdev, 0); dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); if (!dev->iseg) { err = -ENOMEM; - device_printf((&pdev->dev)->bsddev, "ERR: ""Failed mapping initialization segment, aborting\n"); + mlx5_core_err(dev, "Failed mapping initialization segment, aborting\n"); goto err_clr_master; } return 0; err_clr_master: release_bar(dev->pdev); err_disable: mlx5_pci_disable_device(dev); err_dbg: return err; } static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv) { iounmap(dev->iseg); release_bar(dev->pdev); mlx5_pci_disable_device(dev); } static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv) { - struct pci_dev *pdev = dev->pdev; int err; err = mlx5_vsc_find_cap(dev); if (err) - dev_err(&pdev->dev, "Unable to find vendor specific capabilities\n"); + mlx5_core_err(dev, "Unable to find vendor specific capabilities\n"); err = mlx5_query_hca_caps(dev); if (err) { - dev_err(&pdev->dev, "query hca failed\n"); + mlx5_core_err(dev, "query hca failed\n"); goto out; } err = mlx5_query_board_id(dev); if (err) { - dev_err(&pdev->dev, "query board id failed\n"); + mlx5_core_err(dev, "query board id failed\n"); goto out; } err = mlx5_eq_init(dev); if (err) { - dev_err(&pdev->dev, "failed to initialize eq\n"); + mlx5_core_err(dev, "failed to initialize eq\n"); goto out; } MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock); err = mlx5_init_cq_table(dev); if (err) { - dev_err(&pdev->dev, "failed to initialize cq table\n"); + mlx5_core_err(dev, "failed to initialize cq table\n"); goto err_eq_cleanup; } mlx5_init_qp_table(dev); mlx5_init_srq_table(dev); mlx5_init_mr_table(dev); mlx5_init_reserved_gids(dev); mlx5_fpga_init(dev); return 0; err_eq_cleanup: mlx5_eq_cleanup(dev); out: return err; } static void mlx5_cleanup_once(struct mlx5_core_dev *dev) { mlx5_fpga_cleanup(dev); mlx5_cleanup_reserved_gids(dev); mlx5_cleanup_mr_table(dev); mlx5_cleanup_srq_table(dev); mlx5_cleanup_qp_table(dev); mlx5_cleanup_cq_table(dev); mlx5_eq_cleanup(dev); } static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, bool boot) { - struct pci_dev *pdev = dev->pdev; int err; mutex_lock(&dev->intf_state_mutex); if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { - dev_warn(&dev->pdev->dev, "%s: interface is up, NOP\n", - __func__); + mlx5_core_warn(dev, "interface is up, NOP\n"); goto out; } - device_printf((&pdev->dev)->bsddev, "INFO: ""firmware version: %d.%d.%d\n", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev)); + mlx5_core_dbg(dev, "firmware version: %d.%d.%d\n", + fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev)); /* * On load removing any previous indication of internal error, * device is up */ dev->state = MLX5_DEVICE_STATE_UP; err = mlx5_cmd_init(dev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""Failed initializing command interface, aborting\n"); + mlx5_core_err(dev, "Failed initializing command interface, aborting\n"); goto out_err; } err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI); if (err) { - device_printf((&dev->pdev->dev)->bsddev, "ERR: ""Firmware over %d MS in initializing state, aborting\n", FW_INIT_TIMEOUT_MILI); + mlx5_core_err(dev, "Firmware over %d MS in initializing state, aborting\n", FW_INIT_TIMEOUT_MILI); goto err_cmd_cleanup; } err = mlx5_core_enable_hca(dev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""enable hca failed\n"); + mlx5_core_err(dev, "enable hca failed\n"); goto err_cmd_cleanup; } err = mlx5_core_set_issi(dev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""failed to set issi\n"); + mlx5_core_err(dev, "failed to set issi\n"); goto err_disable_hca; } err = mlx5_pagealloc_start(dev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_pagealloc_start failed\n"); + mlx5_core_err(dev, "mlx5_pagealloc_start failed\n"); goto err_disable_hca; } err = mlx5_satisfy_startup_pages(dev, 1); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""failed to allocate boot pages\n"); + mlx5_core_err(dev, "failed to allocate boot pages\n"); goto err_pagealloc_stop; } err = set_hca_ctrl(dev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""set_hca_ctrl failed\n"); + mlx5_core_err(dev, "set_hca_ctrl failed\n"); goto reclaim_boot_pages; } err = handle_hca_cap(dev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""handle_hca_cap failed\n"); + mlx5_core_err(dev, "handle_hca_cap failed\n"); goto reclaim_boot_pages; } err = handle_hca_cap_atomic(dev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""handle_hca_cap_atomic failed\n"); + mlx5_core_err(dev, "handle_hca_cap_atomic failed\n"); goto reclaim_boot_pages; } err = mlx5_satisfy_startup_pages(dev, 0); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""failed to allocate init pages\n"); + mlx5_core_err(dev, "failed to allocate init pages\n"); goto reclaim_boot_pages; } err = mlx5_cmd_init_hca(dev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""init hca failed\n"); + mlx5_core_err(dev, "init hca failed\n"); goto reclaim_boot_pages; } mlx5_start_health_poll(dev); if (boot && mlx5_init_once(dev, priv)) { - dev_err(&pdev->dev, "sw objs init failed\n"); + mlx5_core_err(dev, "sw objs init failed\n"); goto err_stop_poll; } err = mlx5_enable_msix(dev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""enable msix failed\n"); + mlx5_core_err(dev, "enable msix failed\n"); goto err_cleanup_once; } err = mlx5_alloc_uuars(dev, &priv->uuari); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""Failed allocating uar, aborting\n"); + mlx5_core_err(dev, "Failed allocating uar, aborting\n"); goto err_disable_msix; } err = mlx5_start_eqs(dev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to start pages and async EQs\n"); + mlx5_core_err(dev, "Failed to start pages and async EQs\n"); goto err_free_uar; } err = alloc_comp_eqs(dev); if (err) { - device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to alloc completion EQs\n"); + mlx5_core_err(dev, "Failed to alloc completion EQs\n"); goto err_stop_eqs; } if (map_bf_area(dev)) - device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to map blue flame area\n"); + mlx5_core_err(dev, "Failed to map blue flame area\n"); err = mlx5_init_fs(dev); if (err) { mlx5_core_err(dev, "flow steering init %d\n", err); goto err_free_comp_eqs; } err = mlx5_mpfs_init(dev); if (err) { mlx5_core_err(dev, "mpfs init failed %d\n", err); goto err_fs; } err = mlx5_fpga_device_start(dev); if (err) { - dev_err(&pdev->dev, "fpga device start failed %d\n", err); + mlx5_core_err(dev, "fpga device start failed %d\n", err); goto err_mpfs; } err = mlx5_register_device(dev); if (err) { - dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err); + mlx5_core_err(dev, "mlx5_register_device failed %d\n", err); goto err_fpga; } set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); out: mutex_unlock(&dev->intf_state_mutex); return 0; err_fpga: mlx5_fpga_device_stop(dev); err_mpfs: mlx5_mpfs_destroy(dev); err_fs: mlx5_cleanup_fs(dev); err_free_comp_eqs: free_comp_eqs(dev); unmap_bf_area(dev); err_stop_eqs: mlx5_stop_eqs(dev); err_free_uar: mlx5_free_uuars(dev, &priv->uuari); err_disable_msix: mlx5_disable_msix(dev); err_cleanup_once: if (boot) mlx5_cleanup_once(dev); err_stop_poll: mlx5_stop_health_poll(dev, boot); if (mlx5_cmd_teardown_hca(dev)) { - device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n"); + mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n"); goto out_err; } reclaim_boot_pages: mlx5_reclaim_startup_pages(dev); err_pagealloc_stop: mlx5_pagealloc_stop(dev); err_disable_hca: mlx5_core_disable_hca(dev); err_cmd_cleanup: mlx5_cmd_cleanup(dev); out_err: dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; mutex_unlock(&dev->intf_state_mutex); return err; } static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, bool cleanup) { int err = 0; if (cleanup) mlx5_drain_health_recovery(dev); mutex_lock(&dev->intf_state_mutex); if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { - dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n", __func__); + mlx5_core_warn(dev, "%s: interface is down, NOP\n", __func__); if (cleanup) mlx5_cleanup_once(dev); goto out; } mlx5_unregister_device(dev); mlx5_fpga_device_stop(dev); mlx5_mpfs_destroy(dev); mlx5_cleanup_fs(dev); unmap_bf_area(dev); mlx5_wait_for_reclaim_vfs_pages(dev); free_comp_eqs(dev); mlx5_stop_eqs(dev); mlx5_free_uuars(dev, &priv->uuari); mlx5_disable_msix(dev); if (cleanup) mlx5_cleanup_once(dev); mlx5_stop_health_poll(dev, cleanup); err = mlx5_cmd_teardown_hca(dev); if (err) { - device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n"); + mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n"); goto out; } mlx5_pagealloc_stop(dev); mlx5_reclaim_startup_pages(dev); mlx5_core_disable_hca(dev); mlx5_cmd_cleanup(dev); out: clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); mutex_unlock(&dev->intf_state_mutex); return err; } void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event, unsigned long param) { struct mlx5_priv *priv = &dev->priv; struct mlx5_device_context *dev_ctx; unsigned long flags; spin_lock_irqsave(&priv->ctx_lock, flags); list_for_each_entry(dev_ctx, &priv->ctx_list, list) if (dev_ctx->intf->event) dev_ctx->intf->event(dev, dev_ctx->context, event, param); spin_unlock_irqrestore(&priv->ctx_lock, flags); } struct mlx5_core_event_handler { void (*event)(struct mlx5_core_dev *dev, enum mlx5_dev_event event, void *data); }; #define MLX5_STATS_DESC(a, b, c, d, e, ...) d, e, #define MLX5_PORT_MODULE_ERROR_STATS(m) \ m(+1, u64, power_budget_exceeded, "power_budget", "Module Power Budget Exceeded") \ m(+1, u64, long_range, "long_range", "Module Long Range for non MLNX cable/module") \ m(+1, u64, bus_stuck, "bus_stuck", "Module Bus stuck(I2C or data shorted)") \ m(+1, u64, no_eeprom, "no_eeprom", "No EEPROM/retry timeout") \ m(+1, u64, enforce_part_number, "enforce_part_number", "Module Enforce part number list") \ m(+1, u64, unknown_id, "unknown_id", "Module Unknown identifier") \ m(+1, u64, high_temp, "high_temp", "Module High Temperature") \ m(+1, u64, cable_shorted, "cable_shorted", "Module Cable is shorted") static const char *mlx5_pme_err_desc[] = { MLX5_PORT_MODULE_ERROR_STATS(MLX5_STATS_DESC) }; static int init_one(struct pci_dev *pdev, const struct pci_device_id *id) { struct mlx5_core_dev *dev; struct mlx5_priv *priv; device_t bsddev = pdev->dev.bsddev; int i,err; struct sysctl_oid *pme_sysctl_node; struct sysctl_oid *pme_err_sysctl_node; dev = kzalloc(sizeof(*dev), GFP_KERNEL); priv = &dev->priv; if (id) priv->pci_dev_data = id->driver_data; if (mlx5_prof_sel < 0 || mlx5_prof_sel >= ARRAY_SIZE(profiles)) { - device_printf(bsddev, "WARN: selected profile out of range, selecting default (%d)\n", MLX5_DEFAULT_PROF); + device_printf(bsddev, + "WARN: selected profile out of range, selecting default (%d)\n", + MLX5_DEFAULT_PROF); mlx5_prof_sel = MLX5_DEFAULT_PROF; } dev->profile = &profiles[mlx5_prof_sel]; dev->pdev = pdev; dev->event = mlx5_core_event; /* Set desc */ device_set_desc(bsddev, mlx5_version); sysctl_ctx_init(&dev->sysctl_ctx); SYSCTL_ADD_INT(&dev->sysctl_ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(bsddev)), OID_AUTO, "msix_eqvec", CTLFLAG_RDTUN, &dev->msix_eqvec, 0, "Maximum number of MSIX event queue vectors, if set"); SYSCTL_ADD_INT(&dev->sysctl_ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(bsddev)), OID_AUTO, "power_status", CTLFLAG_RD, &dev->pwr_status, 0, "0:Invalid 1:Sufficient 2:Insufficient"); SYSCTL_ADD_INT(&dev->sysctl_ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(bsddev)), OID_AUTO, "power_value", CTLFLAG_RD, &dev->pwr_value, 0, "Current power value in Watts"); pme_sysctl_node = SYSCTL_ADD_NODE(&dev->sysctl_ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(bsddev)), OID_AUTO, "pme_stats", CTLFLAG_RD, NULL, "Port module event statistics"); if (pme_sysctl_node == NULL) { err = -ENOMEM; goto clean_sysctl_ctx; } pme_err_sysctl_node = SYSCTL_ADD_NODE(&dev->sysctl_ctx, SYSCTL_CHILDREN(pme_sysctl_node), OID_AUTO, "errors", CTLFLAG_RD, NULL, "Port module event error statistics"); if (pme_err_sysctl_node == NULL) { err = -ENOMEM; goto clean_sysctl_ctx; } SYSCTL_ADD_U64(&dev->sysctl_ctx, SYSCTL_CHILDREN(pme_sysctl_node), OID_AUTO, "module_plug", CTLFLAG_RD | CTLFLAG_MPSAFE, &dev->priv.pme_stats.status_counters[MLX5_MODULE_STATUS_PLUGGED_ENABLED], 0, "Number of time module plugged"); SYSCTL_ADD_U64(&dev->sysctl_ctx, SYSCTL_CHILDREN(pme_sysctl_node), OID_AUTO, "module_unplug", CTLFLAG_RD | CTLFLAG_MPSAFE, &dev->priv.pme_stats.status_counters[MLX5_MODULE_STATUS_UNPLUGGED], 0, "Number of time module unplugged"); for (i = 0 ; i < MLX5_MODULE_EVENT_ERROR_NUM; i++) { SYSCTL_ADD_U64(&dev->sysctl_ctx, SYSCTL_CHILDREN(pme_err_sysctl_node), OID_AUTO, mlx5_pme_err_desc[2 * i], CTLFLAG_RD | CTLFLAG_MPSAFE, &dev->priv.pme_stats.error_counters[i], 0, mlx5_pme_err_desc[2 * i + 1]); } INIT_LIST_HEAD(&priv->ctx_list); spin_lock_init(&priv->ctx_lock); mutex_init(&dev->pci_status_mutex); mutex_init(&dev->intf_state_mutex); mtx_init(&dev->dump_lock, "mlx5dmp", NULL, MTX_DEF | MTX_NEW); err = mlx5_pci_init(dev, priv); if (err) { - device_printf(bsddev, "ERR: mlx5_pci_init failed %d\n", err); + mlx5_core_err(dev, "mlx5_pci_init failed %d\n", err); goto clean_dev; } err = mlx5_health_init(dev); if (err) { - device_printf(bsddev, "ERR: mlx5_health_init failed %d\n", err); + mlx5_core_err(dev, "mlx5_health_init failed %d\n", err); goto close_pci; } mlx5_pagealloc_init(dev); err = mlx5_load_one(dev, priv, true); if (err) { - device_printf(bsddev, "ERR: mlx5_load_one failed %d\n", err); + mlx5_core_err(dev, "mlx5_load_one failed %d\n", err); goto clean_health; } mlx5_fwdump_prep(dev); mlx5_firmware_update(dev); pci_save_state(bsddev); return 0; clean_health: mlx5_pagealloc_cleanup(dev); mlx5_health_cleanup(dev); close_pci: mlx5_pci_close(dev, priv); clean_dev: mtx_destroy(&dev->dump_lock); clean_sysctl_ctx: sysctl_ctx_free(&dev->sysctl_ctx); kfree(dev); return err; } static void remove_one(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); struct mlx5_priv *priv = &dev->priv; if (mlx5_unload_one(dev, priv, true)) { - dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n"); + mlx5_core_err(dev, "mlx5_unload_one failed\n"); mlx5_health_cleanup(dev); return; } mlx5_pagealloc_cleanup(dev); mlx5_health_cleanup(dev); mlx5_fwdump_clean(dev); mlx5_pci_close(dev, priv); mtx_destroy(&dev->dump_lock); pci_set_drvdata(pdev, NULL); sysctl_ctx_free(&dev->sysctl_ctx); kfree(dev); } static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t state) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); struct mlx5_priv *priv = &dev->priv; - dev_info(&pdev->dev, "%s was called\n", __func__); + mlx5_core_info(dev, "%s was called\n", __func__); mlx5_enter_error_state(dev, false); mlx5_unload_one(dev, priv, false); if (state) { mlx5_drain_health_wq(dev); mlx5_pci_disable_device(dev); } return state == pci_channel_io_perm_failure ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; } static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); int err = 0; - dev_info(&pdev->dev, "%s was called\n", __func__); + mlx5_core_info(dev,"%s was called\n", __func__); err = mlx5_pci_enable_device(dev); if (err) { - dev_err(&pdev->dev, "%s: mlx5_pci_enable_device failed with error code: %d\n" - , __func__, err); + mlx5_core_err(dev, "mlx5_pci_enable_device failed with error code: %d\n" + ,err); return PCI_ERS_RESULT_DISCONNECT; } pci_set_master(pdev); pci_set_powerstate(pdev->dev.bsddev, PCI_POWERSTATE_D0); pci_restore_state(pdev->dev.bsddev); pci_save_state(pdev->dev.bsddev); return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; } /* wait for the device to show vital signs. For now we check * that we can read the device ID and that the health buffer * shows a non zero value which is different than 0xffffffff */ static void wait_vital(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); struct mlx5_core_health *health = &dev->priv.health; const int niter = 100; u32 count; u16 did; int i; /* Wait for firmware to be ready after reset */ msleep(1000); for (i = 0; i < niter; i++) { if (pci_read_config_word(pdev, 2, &did)) { - dev_warn(&pdev->dev, "failed reading config word\n"); + mlx5_core_warn(dev, "failed reading config word\n"); break; } if (did == pdev->device) { - dev_info(&pdev->dev, "device ID correctly read after %d iterations\n", i); + mlx5_core_info(dev, + "device ID correctly read after %d iterations\n", i); break; } msleep(50); } if (i == niter) - dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__); + mlx5_core_warn(dev, "could not read device ID\n"); for (i = 0; i < niter; i++) { count = ioread32be(health->health_counter); if (count && count != 0xffffffff) { - dev_info(&pdev->dev, "Counter value 0x%x after %d iterations\n", count, i); + mlx5_core_info(dev, + "Counter value 0x%x after %d iterations\n", count, i); break; } msleep(50); } if (i == niter) - dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__); + mlx5_core_warn(dev, "could not read device ID\n"); } static void mlx5_pci_resume(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); struct mlx5_priv *priv = &dev->priv; int err; - dev_info(&pdev->dev, "%s was called\n", __func__); + mlx5_core_info(dev,"%s was called\n", __func__); wait_vital(pdev); err = mlx5_load_one(dev, priv, false); if (err) - dev_err(&pdev->dev, "%s: mlx5_load_one failed with error code: %d\n" - , __func__, err); + mlx5_core_err(dev, + "mlx5_load_one failed with error code: %d\n" ,err); else - dev_info(&pdev->dev, "%s: device recovered\n", __func__); + mlx5_core_info(dev,"device recovered\n"); } static const struct pci_error_handlers mlx5_err_handler = { .error_detected = mlx5_pci_err_detected, .slot_reset = mlx5_pci_slot_reset, .resume = mlx5_pci_resume }; static int mlx5_try_fast_unload(struct mlx5_core_dev *dev) { bool fast_teardown, force_teardown; int err; if (!mlx5_fast_unload_enabled) { mlx5_core_dbg(dev, "fast unload is disabled by user\n"); return -EOPNOTSUPP; } fast_teardown = MLX5_CAP_GEN(dev, fast_teardown); force_teardown = MLX5_CAP_GEN(dev, force_teardown); mlx5_core_dbg(dev, "force teardown firmware support=%d\n", force_teardown); mlx5_core_dbg(dev, "fast teardown firmware support=%d\n", fast_teardown); if (!fast_teardown && !force_teardown) return -EOPNOTSUPP; if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { mlx5_core_dbg(dev, "Device in internal error state, giving up\n"); return -EAGAIN; } /* Panic tear down fw command will stop the PCI bus communication * with the HCA, so the health polll is no longer needed. */ mlx5_drain_health_wq(dev); mlx5_stop_health_poll(dev, false); err = mlx5_cmd_fast_teardown_hca(dev); if (!err) goto done; err = mlx5_cmd_force_teardown_hca(dev); if (!err) goto done; mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", err); mlx5_start_health_poll(dev); return err; done: mlx5_enter_error_state(dev, true); return 0; } static void mlx5_disable_interrupts(struct mlx5_core_dev *mdev) { int nvec = mdev->priv.eq_table.num_comp_vectors + MLX5_EQ_VEC_COMP_BASE; int x; mdev->priv.disable_irqs = 1; /* wait for all IRQ handlers to finish processing */ for (x = 0; x != nvec; x++) synchronize_irq(mdev->priv.msix_arr[x].vector); } static void shutdown_one(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); struct mlx5_priv *priv = &dev->priv; int err; /* enter polling mode */ mlx5_cmd_use_polling(dev); /* disable all interrupts */ mlx5_disable_interrupts(dev); err = mlx5_try_fast_unload(dev); if (err) mlx5_unload_one(dev, priv, false); mlx5_pci_disable_device(dev); } static const struct pci_device_id mlx5_core_pci_table[] = { { PCI_VDEVICE(MELLANOX, 4113) }, /* Connect-IB */ { PCI_VDEVICE(MELLANOX, 4114) }, /* Connect-IB VF */ { PCI_VDEVICE(MELLANOX, 4115) }, /* ConnectX-4 */ { PCI_VDEVICE(MELLANOX, 4116) }, /* ConnectX-4 VF */ { PCI_VDEVICE(MELLANOX, 4117) }, /* ConnectX-4LX */ { PCI_VDEVICE(MELLANOX, 4118) }, /* ConnectX-4LX VF */ { PCI_VDEVICE(MELLANOX, 4119) }, /* ConnectX-5 */ { PCI_VDEVICE(MELLANOX, 4120) }, /* ConnectX-5 VF */ { PCI_VDEVICE(MELLANOX, 4121) }, { PCI_VDEVICE(MELLANOX, 4122) }, { PCI_VDEVICE(MELLANOX, 4123) }, { PCI_VDEVICE(MELLANOX, 4124) }, { PCI_VDEVICE(MELLANOX, 4125) }, { PCI_VDEVICE(MELLANOX, 4126) }, { PCI_VDEVICE(MELLANOX, 4127) }, { PCI_VDEVICE(MELLANOX, 4128) }, { PCI_VDEVICE(MELLANOX, 4129) }, { PCI_VDEVICE(MELLANOX, 4130) }, { PCI_VDEVICE(MELLANOX, 4131) }, { PCI_VDEVICE(MELLANOX, 4132) }, { PCI_VDEVICE(MELLANOX, 4133) }, { PCI_VDEVICE(MELLANOX, 4134) }, { PCI_VDEVICE(MELLANOX, 4135) }, { PCI_VDEVICE(MELLANOX, 4136) }, { PCI_VDEVICE(MELLANOX, 4137) }, { PCI_VDEVICE(MELLANOX, 4138) }, { PCI_VDEVICE(MELLANOX, 4139) }, { PCI_VDEVICE(MELLANOX, 4140) }, { PCI_VDEVICE(MELLANOX, 4141) }, { PCI_VDEVICE(MELLANOX, 4142) }, { PCI_VDEVICE(MELLANOX, 4143) }, { PCI_VDEVICE(MELLANOX, 4144) }, { 0, } }; MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); void mlx5_disable_device(struct mlx5_core_dev *dev) { mlx5_pci_err_detected(dev->pdev, 0); } void mlx5_recover_device(struct mlx5_core_dev *dev) { mlx5_pci_disable_device(dev); if (mlx5_pci_slot_reset(dev->pdev) == PCI_ERS_RESULT_RECOVERED) mlx5_pci_resume(dev->pdev); } struct pci_driver mlx5_core_driver = { .name = DRIVER_NAME, .id_table = mlx5_core_pci_table, .shutdown = shutdown_one, .probe = init_one, .remove = remove_one, .err_handler = &mlx5_err_handler }; static int __init init(void) { int err; err = pci_register_driver(&mlx5_core_driver); if (err) goto err_debug; err = mlx5_ctl_init(); if (err) goto err_ctl; return 0; err_ctl: pci_unregister_driver(&mlx5_core_driver); err_debug: return err; } static void __exit cleanup(void) { mlx5_ctl_fini(); pci_unregister_driver(&mlx5_core_driver); } module_init(init); module_exit(cleanup); Index: stable/11/sys/dev/mlx5/mlx5_core/mlx5_mpfs.c =================================================================== --- stable/11/sys/dev/mlx5/mlx5_core/mlx5_mpfs.c (revision 353223) +++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_mpfs.c (revision 353224) @@ -1,125 +1,127 @@ /*- * Copyright (c) 2019, Mellanox Technologies, Ltd. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include #include #include +#include "mlx5_core.h" + #define MPFS_LOCK(dev) spin_lock(&(dev)->mpfs.spinlock) #define MPFS_UNLOCK(dev) spin_unlock(&(dev)->mpfs.spinlock) int mlx5_mpfs_add_mac(struct mlx5_core_dev *dev, u32 *p_index, const u8 *mac) { const u32 l2table_size = MIN(1U << MLX5_CAP_GEN(dev, log_max_l2_table), MLX5_MPFS_TABLE_MAX); u32 in[MLX5_ST_SZ_DW(set_l2_table_entry_in)] = {}; u32 out[MLX5_ST_SZ_DW(set_l2_table_entry_out)] = {}; u8 *in_mac_addr; u32 index; int err; if (!MLX5_CAP_GEN(dev, eswitch_flow_table)) { *p_index = 0; return (0); } MPFS_LOCK(dev); index = find_first_zero_bit(dev->mpfs.bitmap, l2table_size); if (index < l2table_size) set_bit(index, dev->mpfs.bitmap); MPFS_UNLOCK(dev); if (index >= l2table_size) return (-ENOMEM); MLX5_SET(set_l2_table_entry_in, in, opcode, MLX5_CMD_OP_SET_L2_TABLE_ENTRY); MLX5_SET(set_l2_table_entry_in, in, table_index, index); in_mac_addr = MLX5_ADDR_OF(set_l2_table_entry_in, in, mac_address); ether_addr_copy(&in_mac_addr[2], mac); err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); if (err != 0) { MPFS_LOCK(dev); clear_bit(index, dev->mpfs.bitmap); MPFS_UNLOCK(dev); } else { *p_index = index; } return (err); } int mlx5_mpfs_del_mac(struct mlx5_core_dev *dev, u32 index) { u32 in[MLX5_ST_SZ_DW(delete_l2_table_entry_in)] = {}; u32 out[MLX5_ST_SZ_DW(delete_l2_table_entry_out)] = {}; int err; if (!MLX5_CAP_GEN(dev, eswitch_flow_table)) { if (index != 0) return (-EINVAL); return (0); } MLX5_SET(delete_l2_table_entry_in, in, opcode, MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY); MLX5_SET(delete_l2_table_entry_in, in, table_index, index); err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); if (err == 0) { MPFS_LOCK(dev); clear_bit(index, dev->mpfs.bitmap); MPFS_UNLOCK(dev); } return (err); } int mlx5_mpfs_init(struct mlx5_core_dev *dev) { spin_lock_init(&dev->mpfs.spinlock); bitmap_zero(dev->mpfs.bitmap, MLX5_MPFS_TABLE_MAX); return (0); } void mlx5_mpfs_destroy(struct mlx5_core_dev *dev) { u32 num; num = bitmap_weight(dev->mpfs.bitmap, MLX5_MPFS_TABLE_MAX); if (num != 0) - dev_err(&dev->pdev->dev, "Leaking %u MPFS MAC table entries\n", num); + mlx5_core_err(dev, "Leaking %u MPFS MAC table entries\n", num); spin_lock_destroy(&dev->mpfs.spinlock); } Index: stable/11/sys/dev/mlx5/mlx5_core/mlx5_port.c =================================================================== --- stable/11/sys/dev/mlx5/mlx5_core/mlx5_port.c (revision 353223) +++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_port.c (revision 353224) @@ -1,1238 +1,1238 @@ /*- * Copyright (c) 2013-2018, Mellanox Technologies, Ltd. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include "mlx5_core.h" int mlx5_core_access_reg(struct mlx5_core_dev *dev, void *data_in, int size_in, void *data_out, int size_out, u16 reg_num, int arg, int write) { int outlen = MLX5_ST_SZ_BYTES(access_register_out) + size_out; int inlen = MLX5_ST_SZ_BYTES(access_register_in) + size_in; int err = -ENOMEM; u32 *out = NULL; u32 *in = NULL; void *data; in = mlx5_vzalloc(inlen); out = mlx5_vzalloc(outlen); if (!in || !out) goto out; data = MLX5_ADDR_OF(access_register_in, in, register_data); memcpy(data, data_in, size_in); MLX5_SET(access_register_in, in, opcode, MLX5_CMD_OP_ACCESS_REG); MLX5_SET(access_register_in, in, op_mod, !write); MLX5_SET(access_register_in, in, argument, arg); MLX5_SET(access_register_in, in, register_id, reg_num); err = mlx5_cmd_exec(dev, in, inlen, out, outlen); if (err) goto out; data = MLX5_ADDR_OF(access_register_out, out, register_data); memcpy(data_out, data, size_out); out: kvfree(out); kvfree(in); return err; } EXPORT_SYMBOL_GPL(mlx5_core_access_reg); int mlx5_query_qcam_reg(struct mlx5_core_dev *mdev, u32 *qcam, u8 feature_group, u8 access_reg_group) { u32 in[MLX5_ST_SZ_DW(qcam_reg)] = {}; int sz = MLX5_ST_SZ_BYTES(qcam_reg); MLX5_SET(qcam_reg, in, feature_group, feature_group); MLX5_SET(qcam_reg, in, access_reg_group, access_reg_group); return mlx5_core_access_reg(mdev, in, sz, qcam, sz, MLX5_REG_QCAM, 0, 0); } EXPORT_SYMBOL_GPL(mlx5_query_qcam_reg); int mlx5_query_pcam_reg(struct mlx5_core_dev *dev, u32 *pcam, u8 feature_group, u8 access_reg_group) { u32 in[MLX5_ST_SZ_DW(pcam_reg)] = {}; int sz = MLX5_ST_SZ_BYTES(pcam_reg); MLX5_SET(pcam_reg, in, feature_group, feature_group); MLX5_SET(pcam_reg, in, access_reg_group, access_reg_group); return mlx5_core_access_reg(dev, in, sz, pcam, sz, MLX5_REG_PCAM, 0, 0); } int mlx5_query_mcam_reg(struct mlx5_core_dev *dev, u32 *mcam, u8 feature_group, u8 access_reg_group) { u32 in[MLX5_ST_SZ_DW(mcam_reg)] = {}; int sz = MLX5_ST_SZ_BYTES(mcam_reg); MLX5_SET(mcam_reg, in, feature_group, feature_group); MLX5_SET(mcam_reg, in, access_reg_group, access_reg_group); return mlx5_core_access_reg(dev, in, sz, mcam, sz, MLX5_REG_MCAM, 0, 0); } struct mlx5_reg_pcap { u8 rsvd0; u8 port_num; u8 rsvd1[2]; __be32 caps_127_96; __be32 caps_95_64; __be32 caps_63_32; __be32 caps_31_0; }; /* This function should be used after setting a port register only */ void mlx5_toggle_port_link(struct mlx5_core_dev *dev) { enum mlx5_port_status ps; mlx5_query_port_admin_status(dev, &ps); mlx5_set_port_status(dev, MLX5_PORT_DOWN); if (ps == MLX5_PORT_UP) mlx5_set_port_status(dev, MLX5_PORT_UP); } EXPORT_SYMBOL_GPL(mlx5_toggle_port_link); int mlx5_set_port_caps(struct mlx5_core_dev *dev, u8 port_num, u32 caps) { struct mlx5_reg_pcap in; struct mlx5_reg_pcap out; int err; memset(&in, 0, sizeof(in)); in.caps_127_96 = cpu_to_be32(caps); in.port_num = port_num; err = mlx5_core_access_reg(dev, &in, sizeof(in), &out, sizeof(out), MLX5_REG_PCAP, 0, 1); return err; } EXPORT_SYMBOL_GPL(mlx5_set_port_caps); int mlx5_query_port_ptys(struct mlx5_core_dev *dev, u32 *ptys, int ptys_size, int proto_mask, u8 local_port) { u32 in[MLX5_ST_SZ_DW(ptys_reg)]; int err; memset(in, 0, sizeof(in)); MLX5_SET(ptys_reg, in, local_port, local_port); MLX5_SET(ptys_reg, in, proto_mask, proto_mask); err = mlx5_core_access_reg(dev, in, sizeof(in), ptys, ptys_size, MLX5_REG_PTYS, 0, 0); return err; } EXPORT_SYMBOL_GPL(mlx5_query_port_ptys); int mlx5_query_port_proto_cap(struct mlx5_core_dev *dev, u32 *proto_cap, int proto_mask) { u32 out[MLX5_ST_SZ_DW(ptys_reg)]; int err; err = mlx5_query_port_ptys(dev, out, sizeof(out), proto_mask, 1); if (err) return err; if (proto_mask == MLX5_PTYS_EN) *proto_cap = MLX5_GET(ptys_reg, out, eth_proto_capability); else *proto_cap = MLX5_GET(ptys_reg, out, ib_proto_capability); return 0; } EXPORT_SYMBOL_GPL(mlx5_query_port_proto_cap); int mlx5_query_port_autoneg(struct mlx5_core_dev *dev, int proto_mask, u8 *an_disable_cap, u8 *an_disable_status) { u32 out[MLX5_ST_SZ_DW(ptys_reg)]; int err; err = mlx5_query_port_ptys(dev, out, sizeof(out), proto_mask, 1); if (err) return err; *an_disable_status = MLX5_GET(ptys_reg, out, an_disable_admin); *an_disable_cap = MLX5_GET(ptys_reg, out, an_disable_cap); return 0; } EXPORT_SYMBOL_GPL(mlx5_query_port_autoneg); int mlx5_set_port_autoneg(struct mlx5_core_dev *dev, bool disable, u32 eth_proto_admin, int proto_mask) { u32 in[MLX5_ST_SZ_DW(ptys_reg)] = {0}; u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {0}; u8 an_disable_cap; u8 an_disable_status; int err; err = mlx5_query_port_autoneg(dev, proto_mask, &an_disable_cap, &an_disable_status); if (err) return err; if (!an_disable_cap) return -EPERM; MLX5_SET(ptys_reg, in, local_port, 1); MLX5_SET(ptys_reg, in, an_disable_admin, disable); MLX5_SET(ptys_reg, in, proto_mask, proto_mask); if (proto_mask == MLX5_PTYS_EN) MLX5_SET(ptys_reg, in, eth_proto_admin, eth_proto_admin); err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_PTYS, 0, 1); return err; } EXPORT_SYMBOL_GPL(mlx5_set_port_autoneg); int mlx5_query_port_proto_admin(struct mlx5_core_dev *dev, u32 *proto_admin, int proto_mask) { u32 out[MLX5_ST_SZ_DW(ptys_reg)]; int err; err = mlx5_query_port_ptys(dev, out, sizeof(out), proto_mask, 1); if (err) return err; if (proto_mask == MLX5_PTYS_EN) *proto_admin = MLX5_GET(ptys_reg, out, eth_proto_admin); else *proto_admin = MLX5_GET(ptys_reg, out, ib_proto_admin); return 0; } EXPORT_SYMBOL_GPL(mlx5_query_port_proto_admin); int mlx5_query_port_eth_proto_oper(struct mlx5_core_dev *dev, u32 *proto_oper, u8 local_port) { u32 out[MLX5_ST_SZ_DW(ptys_reg)]; int err; err = mlx5_query_port_ptys(dev, out, sizeof(out), MLX5_PTYS_EN, local_port); if (err) return err; *proto_oper = MLX5_GET(ptys_reg, out, eth_proto_oper); return 0; } EXPORT_SYMBOL(mlx5_query_port_eth_proto_oper); int mlx5_set_port_proto(struct mlx5_core_dev *dev, u32 proto_admin, int proto_mask, bool ext) { u32 in[MLX5_ST_SZ_DW(ptys_reg)] = {0}; u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {0}; int err; MLX5_SET(ptys_reg, in, local_port, 1); MLX5_SET(ptys_reg, in, proto_mask, proto_mask); if (proto_mask == MLX5_PTYS_EN) { if (ext) MLX5_SET(ptys_reg, in, ext_eth_proto_admin, proto_admin); else MLX5_SET(ptys_reg, in, eth_proto_admin, proto_admin); } else { MLX5_SET(ptys_reg, in, ib_proto_admin, proto_admin); } err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_PTYS, 0, 1); return err; } EXPORT_SYMBOL_GPL(mlx5_set_port_proto); int mlx5_set_port_status(struct mlx5_core_dev *dev, enum mlx5_port_status status) { u32 in[MLX5_ST_SZ_DW(paos_reg)] = {0}; u32 out[MLX5_ST_SZ_DW(paos_reg)] = {0}; int err; MLX5_SET(paos_reg, in, local_port, 1); MLX5_SET(paos_reg, in, admin_status, status); MLX5_SET(paos_reg, in, ase, 1); err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_PAOS, 0, 1); return err; } int mlx5_query_port_status(struct mlx5_core_dev *dev, u8 *status) { u32 in[MLX5_ST_SZ_DW(paos_reg)] = {0}; u32 out[MLX5_ST_SZ_DW(paos_reg)] = {0}; int err; MLX5_SET(paos_reg, in, local_port, 1); err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_PAOS, 0, 0); if (err) return err; *status = MLX5_GET(paos_reg, out, oper_status); return err; } int mlx5_query_port_admin_status(struct mlx5_core_dev *dev, enum mlx5_port_status *status) { u32 in[MLX5_ST_SZ_DW(paos_reg)] = {0}; u32 out[MLX5_ST_SZ_DW(paos_reg)]; int err; MLX5_SET(paos_reg, in, local_port, 1); err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_PAOS, 0, 0); if (err) return err; *status = MLX5_GET(paos_reg, out, admin_status); return 0; } EXPORT_SYMBOL_GPL(mlx5_query_port_admin_status); static int mlx5_query_port_mtu(struct mlx5_core_dev *dev, int *admin_mtu, int *max_mtu, int *oper_mtu) { u32 in[MLX5_ST_SZ_DW(pmtu_reg)] = {0}; u32 out[MLX5_ST_SZ_DW(pmtu_reg)] = {0}; int err; MLX5_SET(pmtu_reg, in, local_port, 1); err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_PMTU, 0, 0); if (err) return err; if (max_mtu) *max_mtu = MLX5_GET(pmtu_reg, out, max_mtu); if (oper_mtu) *oper_mtu = MLX5_GET(pmtu_reg, out, oper_mtu); if (admin_mtu) *admin_mtu = MLX5_GET(pmtu_reg, out, admin_mtu); return err; } int mlx5_set_port_mtu(struct mlx5_core_dev *dev, int mtu) { u32 in[MLX5_ST_SZ_DW(pmtu_reg)] = {0}; u32 out[MLX5_ST_SZ_DW(pmtu_reg)] = {0}; MLX5_SET(pmtu_reg, in, admin_mtu, mtu); MLX5_SET(pmtu_reg, in, local_port, 1); return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_PMTU, 0, 1); } EXPORT_SYMBOL_GPL(mlx5_set_port_mtu); int mlx5_query_port_max_mtu(struct mlx5_core_dev *dev, int *max_mtu) { return mlx5_query_port_mtu(dev, NULL, max_mtu, NULL); } EXPORT_SYMBOL_GPL(mlx5_query_port_max_mtu); int mlx5_set_port_pause_and_pfc(struct mlx5_core_dev *dev, u32 port, u8 rx_pause, u8 tx_pause, u8 pfc_en_rx, u8 pfc_en_tx) { u32 in[MLX5_ST_SZ_DW(pfcc_reg)] = {0}; u32 out[MLX5_ST_SZ_DW(pfcc_reg)] = {0}; if (pfc_en_rx || pfc_en_tx) { /* PFC and global pauseframes are incompatible features */ if (tx_pause || rx_pause) return -EINVAL; } MLX5_SET(pfcc_reg, in, local_port, port); MLX5_SET(pfcc_reg, in, pptx, tx_pause); MLX5_SET(pfcc_reg, in, pprx, rx_pause); MLX5_SET(pfcc_reg, in, pfctx, pfc_en_tx); MLX5_SET(pfcc_reg, in, pfcrx, pfc_en_rx); MLX5_SET(pfcc_reg, in, prio_mask_tx, pfc_en_tx); MLX5_SET(pfcc_reg, in, prio_mask_rx, pfc_en_rx); return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_PFCC, 0, 1); } int mlx5_query_port_pause(struct mlx5_core_dev *dev, u32 port, u32 *rx_pause, u32 *tx_pause) { u32 in[MLX5_ST_SZ_DW(pfcc_reg)] = {0}; u32 out[MLX5_ST_SZ_DW(pfcc_reg)] = {0}; int err; MLX5_SET(pfcc_reg, in, local_port, port); err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_PFCC, 0, 0); if (err) return err; *rx_pause = MLX5_GET(pfcc_reg, out, pprx); *tx_pause = MLX5_GET(pfcc_reg, out, pptx); return 0; } int mlx5_query_port_pfc(struct mlx5_core_dev *dev, u8 *pfc_en_tx, u8 *pfc_en_rx) { u32 in[MLX5_ST_SZ_DW(pfcc_reg)] = {}; u32 out[MLX5_ST_SZ_DW(pfcc_reg)]; int err; MLX5_SET(pfcc_reg, in, local_port, 1); err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_PFCC, 0, 0); if (err) return err; if (pfc_en_tx != NULL) *pfc_en_tx = MLX5_GET(pfcc_reg, out, pfctx); if (pfc_en_rx != NULL) *pfc_en_rx = MLX5_GET(pfcc_reg, out, pfcrx); return 0; } EXPORT_SYMBOL_GPL(mlx5_query_port_pfc); int mlx5_query_port_oper_mtu(struct mlx5_core_dev *dev, int *oper_mtu) { return mlx5_query_port_mtu(dev, NULL, NULL, oper_mtu); } EXPORT_SYMBOL_GPL(mlx5_query_port_oper_mtu); u8 mlx5_is_wol_supported(struct mlx5_core_dev *dev) { u8 wol_supported = 0; if (MLX5_CAP_GEN(dev, wol_s)) wol_supported |= MLX5_WOL_SECURED_MAGIC; if (MLX5_CAP_GEN(dev, wol_g)) wol_supported |= MLX5_WOL_MAGIC; if (MLX5_CAP_GEN(dev, wol_a)) wol_supported |= MLX5_WOL_ARP; if (MLX5_CAP_GEN(dev, wol_b)) wol_supported |= MLX5_WOL_BROADCAST; if (MLX5_CAP_GEN(dev, wol_m)) wol_supported |= MLX5_WOL_MULTICAST; if (MLX5_CAP_GEN(dev, wol_u)) wol_supported |= MLX5_WOL_UNICAST; if (MLX5_CAP_GEN(dev, wol_p)) wol_supported |= MLX5_WOL_PHY_ACTIVITY; return wol_supported; } EXPORT_SYMBOL_GPL(mlx5_is_wol_supported); int mlx5_set_wol(struct mlx5_core_dev *dev, u8 wol_mode) { u32 in[MLX5_ST_SZ_DW(set_wol_rol_in)] = {0}; u32 out[MLX5_ST_SZ_DW(set_wol_rol_out)] = {0}; MLX5_SET(set_wol_rol_in, in, opcode, MLX5_CMD_OP_SET_WOL_ROL); MLX5_SET(set_wol_rol_in, in, wol_mode_valid, 1); MLX5_SET(set_wol_rol_in, in, wol_mode, wol_mode); return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } EXPORT_SYMBOL_GPL(mlx5_set_wol); int mlx5_query_dropless_mode(struct mlx5_core_dev *dev, u16 *timeout) { u32 in[MLX5_ST_SZ_DW(query_delay_drop_params_in)] = {0}; u32 out[MLX5_ST_SZ_DW(query_delay_drop_params_out)] = {0}; int err = 0; MLX5_SET(query_delay_drop_params_in, in, opcode, MLX5_CMD_OP_QUERY_DELAY_DROP_PARAMS); err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); if (err) return err; *timeout = MLX5_GET(query_delay_drop_params_out, out, delay_drop_timeout); return 0; } EXPORT_SYMBOL_GPL(mlx5_query_dropless_mode); int mlx5_set_dropless_mode(struct mlx5_core_dev *dev, u16 timeout) { u32 in[MLX5_ST_SZ_DW(set_delay_drop_params_in)] = {0}; u32 out[MLX5_ST_SZ_DW(set_delay_drop_params_out)] = {0}; MLX5_SET(set_delay_drop_params_in, in, opcode, MLX5_CMD_OP_SET_DELAY_DROP_PARAMS); MLX5_SET(set_delay_drop_params_in, in, delay_drop_timeout, timeout); return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } EXPORT_SYMBOL_GPL(mlx5_set_dropless_mode); int mlx5_core_access_pvlc(struct mlx5_core_dev *dev, struct mlx5_pvlc_reg *pvlc, int write) { int sz = MLX5_ST_SZ_BYTES(pvlc_reg); u8 in[MLX5_ST_SZ_BYTES(pvlc_reg)] = {0}; u8 out[MLX5_ST_SZ_BYTES(pvlc_reg)] = {0}; int err; MLX5_SET(pvlc_reg, in, local_port, pvlc->local_port); if (write) MLX5_SET(pvlc_reg, in, vl_admin, pvlc->vl_admin); err = mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PVLC, 0, !!write); if (err) return err; if (!write) { pvlc->local_port = MLX5_GET(pvlc_reg, out, local_port); pvlc->vl_hw_cap = MLX5_GET(pvlc_reg, out, vl_hw_cap); pvlc->vl_admin = MLX5_GET(pvlc_reg, out, vl_admin); pvlc->vl_operational = MLX5_GET(pvlc_reg, out, vl_operational); } return 0; } EXPORT_SYMBOL_GPL(mlx5_core_access_pvlc); int mlx5_core_access_ptys(struct mlx5_core_dev *dev, struct mlx5_ptys_reg *ptys, int write) { int sz = MLX5_ST_SZ_BYTES(ptys_reg); void *out = NULL; void *in = NULL; int err; in = mlx5_vzalloc(sz); if (!in) return -ENOMEM; out = mlx5_vzalloc(sz); if (!out) { kfree(in); return -ENOMEM; } MLX5_SET(ptys_reg, in, local_port, ptys->local_port); MLX5_SET(ptys_reg, in, proto_mask, ptys->proto_mask); if (write) { MLX5_SET(ptys_reg, in, eth_proto_capability, ptys->eth_proto_cap); MLX5_SET(ptys_reg, in, ib_link_width_capability, ptys->ib_link_width_cap); MLX5_SET(ptys_reg, in, ib_proto_capability, ptys->ib_proto_cap); MLX5_SET(ptys_reg, in, eth_proto_admin, ptys->eth_proto_admin); MLX5_SET(ptys_reg, in, ib_link_width_admin, ptys->ib_link_width_admin); MLX5_SET(ptys_reg, in, ib_proto_admin, ptys->ib_proto_admin); MLX5_SET(ptys_reg, in, eth_proto_oper, ptys->eth_proto_oper); MLX5_SET(ptys_reg, in, ib_link_width_oper, ptys->ib_link_width_oper); MLX5_SET(ptys_reg, in, ib_proto_oper, ptys->ib_proto_oper); MLX5_SET(ptys_reg, in, eth_proto_lp_advertise, ptys->eth_proto_lp_advertise); } err = mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PTYS, 0, !!write); if (err) goto out; if (!write) { ptys->local_port = MLX5_GET(ptys_reg, out, local_port); ptys->proto_mask = MLX5_GET(ptys_reg, out, proto_mask); ptys->eth_proto_cap = MLX5_GET(ptys_reg, out, eth_proto_capability); ptys->ib_link_width_cap = MLX5_GET(ptys_reg, out, ib_link_width_capability); ptys->ib_proto_cap = MLX5_GET(ptys_reg, out, ib_proto_capability); ptys->eth_proto_admin = MLX5_GET(ptys_reg, out, eth_proto_admin); ptys->ib_link_width_admin = MLX5_GET(ptys_reg, out, ib_link_width_admin); ptys->ib_proto_admin = MLX5_GET(ptys_reg, out, ib_proto_admin); ptys->eth_proto_oper = MLX5_GET(ptys_reg, out, eth_proto_oper); ptys->ib_link_width_oper = MLX5_GET(ptys_reg, out, ib_link_width_oper); ptys->ib_proto_oper = MLX5_GET(ptys_reg, out, ib_proto_oper); ptys->eth_proto_lp_advertise = MLX5_GET(ptys_reg, out, eth_proto_lp_advertise); } out: kvfree(in); kvfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_core_access_ptys); -static int mtu_to_ib_mtu(int mtu) +static int mtu_to_ib_mtu(struct mlx5_core_dev *dev, int mtu) { switch (mtu) { case 256: return 1; case 512: return 2; case 1024: return 3; case 2048: return 4; case 4096: return 5; default: - printf("mlx5_core: WARN: ""invalid mtu\n"); + mlx5_core_warn(dev, "invalid mtu\n"); return -1; } } int mlx5_core_access_pmtu(struct mlx5_core_dev *dev, struct mlx5_pmtu_reg *pmtu, int write) { int sz = MLX5_ST_SZ_BYTES(pmtu_reg); void *out = NULL; void *in = NULL; int err; in = mlx5_vzalloc(sz); if (!in) return -ENOMEM; out = mlx5_vzalloc(sz); if (!out) { kfree(in); return -ENOMEM; } MLX5_SET(pmtu_reg, in, local_port, pmtu->local_port); if (write) MLX5_SET(pmtu_reg, in, admin_mtu, pmtu->admin_mtu); err = mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PMTU, 0, !!write); if (err) goto out; if (!write) { pmtu->local_port = MLX5_GET(pmtu_reg, out, local_port); - pmtu->max_mtu = mtu_to_ib_mtu(MLX5_GET(pmtu_reg, out, + pmtu->max_mtu = mtu_to_ib_mtu(dev, MLX5_GET(pmtu_reg, out, max_mtu)); - pmtu->admin_mtu = mtu_to_ib_mtu(MLX5_GET(pmtu_reg, out, + pmtu->admin_mtu = mtu_to_ib_mtu(dev, MLX5_GET(pmtu_reg, out, admin_mtu)); - pmtu->oper_mtu = mtu_to_ib_mtu(MLX5_GET(pmtu_reg, out, + pmtu->oper_mtu = mtu_to_ib_mtu(dev, MLX5_GET(pmtu_reg, out, oper_mtu)); } out: kvfree(in); kvfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_core_access_pmtu); int mlx5_query_module_num(struct mlx5_core_dev *dev, int *module_num) { u32 in[MLX5_ST_SZ_DW(pmlp_reg)] = {0}; u32 out[MLX5_ST_SZ_DW(pmlp_reg)] = {0}; int lane = 0; int err; MLX5_SET(pmlp_reg, in, local_port, 1); err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_PMLP, 0, 0); if (err) return err; lane = MLX5_GET(pmlp_reg, out, lane0_module_mapping); *module_num = lane & MLX5_EEPROM_IDENTIFIER_BYTE_MASK; return 0; } EXPORT_SYMBOL_GPL(mlx5_query_module_num); int mlx5_query_eeprom(struct mlx5_core_dev *dev, int i2c_addr, int page_num, int device_addr, int size, int module_num, u32 *data, int *size_read) { u32 in[MLX5_ST_SZ_DW(mcia_reg)] = {0}; u32 out[MLX5_ST_SZ_DW(mcia_reg)] = {0}; u32 *ptr = (u32 *)MLX5_ADDR_OF(mcia_reg, out, dword_0); int status; int err; size = min_t(int, size, MLX5_EEPROM_MAX_BYTES); MLX5_SET(mcia_reg, in, l, 0); MLX5_SET(mcia_reg, in, module, module_num); MLX5_SET(mcia_reg, in, i2c_device_address, i2c_addr); MLX5_SET(mcia_reg, in, page_number, page_num); MLX5_SET(mcia_reg, in, device_address, device_addr); MLX5_SET(mcia_reg, in, size, size); err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MCIA, 0, 0); if (err) return err; status = MLX5_GET(mcia_reg, out, status); if (status) return status; memcpy(data, ptr, size); *size_read = size; return 0; } EXPORT_SYMBOL_GPL(mlx5_query_eeprom); int mlx5_vxlan_udp_port_add(struct mlx5_core_dev *dev, u16 port) { u32 in[MLX5_ST_SZ_DW(add_vxlan_udp_dport_in)] = {0}; u32 out[MLX5_ST_SZ_DW(add_vxlan_udp_dport_out)] = {0}; int err; MLX5_SET(add_vxlan_udp_dport_in, in, opcode, MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT); MLX5_SET(add_vxlan_udp_dport_in, in, vxlan_udp_port, port); err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); if (err) { mlx5_core_err(dev, "Failed %s, port %u, err - %d", mlx5_command_str(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT), port, err); } return err; } int mlx5_vxlan_udp_port_delete(struct mlx5_core_dev *dev, u16 port) { u32 in[MLX5_ST_SZ_DW(delete_vxlan_udp_dport_in)] = {0}; u32 out[MLX5_ST_SZ_DW(delete_vxlan_udp_dport_out)] = {0}; int err; MLX5_SET(delete_vxlan_udp_dport_in, in, opcode, MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT); MLX5_SET(delete_vxlan_udp_dport_in, in, vxlan_udp_port, port); err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); if (err) { mlx5_core_err(dev, "Failed %s, port %u, err - %d", mlx5_command_str(MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT), port, err); } return err; } int mlx5_query_wol(struct mlx5_core_dev *dev, u8 *wol_mode) { u32 in[MLX5_ST_SZ_DW(query_wol_rol_in)] = {0}; u32 out[MLX5_ST_SZ_DW(query_wol_rol_out)] = {0}; int err; MLX5_SET(query_wol_rol_in, in, opcode, MLX5_CMD_OP_QUERY_WOL_ROL); err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); if (!err) *wol_mode = MLX5_GET(query_wol_rol_out, out, wol_mode); return err; } EXPORT_SYMBOL_GPL(mlx5_query_wol); int mlx5_query_port_cong_status(struct mlx5_core_dev *mdev, int protocol, int priority, int *is_enable) { u32 in[MLX5_ST_SZ_DW(query_cong_status_in)] = {0}; u32 out[MLX5_ST_SZ_DW(query_cong_status_out)] = {0}; int err; *is_enable = 0; MLX5_SET(query_cong_status_in, in, opcode, MLX5_CMD_OP_QUERY_CONG_STATUS); MLX5_SET(query_cong_status_in, in, cong_protocol, protocol); MLX5_SET(query_cong_status_in, in, priority, priority); err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); if (!err) *is_enable = MLX5_GET(query_cong_status_out, out, enable); return err; } int mlx5_modify_port_cong_status(struct mlx5_core_dev *mdev, int protocol, int priority, int enable) { u32 in[MLX5_ST_SZ_DW(modify_cong_status_in)] = {0}; u32 out[MLX5_ST_SZ_DW(modify_cong_status_out)] = {0}; MLX5_SET(modify_cong_status_in, in, opcode, MLX5_CMD_OP_MODIFY_CONG_STATUS); MLX5_SET(modify_cong_status_in, in, cong_protocol, protocol); MLX5_SET(modify_cong_status_in, in, priority, priority); MLX5_SET(modify_cong_status_in, in, enable, enable); return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); } int mlx5_query_port_cong_params(struct mlx5_core_dev *mdev, int protocol, void *out, int out_size) { u32 in[MLX5_ST_SZ_DW(query_cong_params_in)] = {0}; MLX5_SET(query_cong_params_in, in, opcode, MLX5_CMD_OP_QUERY_CONG_PARAMS); MLX5_SET(query_cong_params_in, in, cong_protocol, protocol); return mlx5_cmd_exec(mdev, in, sizeof(in), out, out_size); } static int mlx5_query_port_qetcr_reg(struct mlx5_core_dev *mdev, u32 *out, int outlen) { u32 in[MLX5_ST_SZ_DW(qetc_reg)]; if (!MLX5_CAP_GEN(mdev, ets)) return -ENOTSUPP; memset(in, 0, sizeof(in)); return mlx5_core_access_reg(mdev, in, sizeof(in), out, outlen, MLX5_REG_QETCR, 0, 0); } int mlx5_max_tc(struct mlx5_core_dev *mdev) { u8 num_tc = MLX5_CAP_GEN(mdev, max_tc) ? : 8; return num_tc - 1; } EXPORT_SYMBOL_GPL(mlx5_max_tc); static int mlx5_set_port_qetcr_reg(struct mlx5_core_dev *mdev, u32 *in, int inlen) { u32 out[MLX5_ST_SZ_DW(qetc_reg)]; if (!MLX5_CAP_GEN(mdev, ets)) return -ENOTSUPP; return mlx5_core_access_reg(mdev, in, inlen, out, sizeof(out), MLX5_REG_QETCR, 0, 1); } int mlx5_query_port_tc_rate_limit(struct mlx5_core_dev *mdev, u8 *max_bw_value, u8 *max_bw_units) { u32 out[MLX5_ST_SZ_DW(qetc_reg)]; void *ets_tcn_conf; int err; int i; err = mlx5_query_port_qetcr_reg(mdev, out, sizeof(out)); if (err) return err; for (i = 0; i <= mlx5_max_tc(mdev); i++) { ets_tcn_conf = MLX5_ADDR_OF(qetc_reg, out, tc_configuration[i]); max_bw_value[i] = MLX5_GET(ets_tcn_config_reg, ets_tcn_conf, max_bw_value); max_bw_units[i] = MLX5_GET(ets_tcn_config_reg, ets_tcn_conf, max_bw_units); } return 0; } EXPORT_SYMBOL_GPL(mlx5_query_port_tc_rate_limit); int mlx5_modify_port_tc_rate_limit(struct mlx5_core_dev *mdev, const u8 *max_bw_value, const u8 *max_bw_units) { u32 in[MLX5_ST_SZ_DW(qetc_reg)] = {}; void *ets_tcn_conf; int i; MLX5_SET(qetc_reg, in, port_number, 1); for (i = 0; i <= mlx5_max_tc(mdev); i++) { ets_tcn_conf = MLX5_ADDR_OF(qetc_reg, in, tc_configuration[i]); MLX5_SET(ets_tcn_config_reg, ets_tcn_conf, r, 1); MLX5_SET(ets_tcn_config_reg, ets_tcn_conf, max_bw_units, max_bw_units[i]); MLX5_SET(ets_tcn_config_reg, ets_tcn_conf, max_bw_value, max_bw_value[i]); } return mlx5_set_port_qetcr_reg(mdev, in, sizeof(in)); } EXPORT_SYMBOL_GPL(mlx5_modify_port_tc_rate_limit); int mlx5_query_port_prio_tc(struct mlx5_core_dev *mdev, u8 prio, u8 *tc) { u32 in[MLX5_ST_SZ_DW(qtct_reg)]; u32 out[MLX5_ST_SZ_DW(qtct_reg)]; int err; memset(in, 0, sizeof(in)); memset(out, 0, sizeof(out)); MLX5_SET(qtct_reg, in, port_number, 1); MLX5_SET(qtct_reg, in, prio, prio); err = mlx5_core_access_reg(mdev, in, sizeof(in), out, sizeof(out), MLX5_REG_QTCT, 0, 0); if (!err) *tc = MLX5_GET(qtct_reg, out, tclass); return err; } EXPORT_SYMBOL_GPL(mlx5_query_port_prio_tc); int mlx5_set_port_prio_tc(struct mlx5_core_dev *mdev, int prio_index, const u8 prio_tc) { u32 in[MLX5_ST_SZ_DW(qtct_reg)] = {}; u32 out[MLX5_ST_SZ_DW(qtct_reg)]; int err; if (prio_tc > mlx5_max_tc(mdev)) return -EINVAL; MLX5_SET(qtct_reg, in, prio, prio_index); MLX5_SET(qtct_reg, in, tclass, prio_tc); err = mlx5_core_access_reg(mdev, in, sizeof(in), out, sizeof(out), MLX5_REG_QTCT, 0, 1); return (err); } EXPORT_SYMBOL_GPL(mlx5_set_port_prio_tc); int mlx5_set_port_tc_group(struct mlx5_core_dev *mdev, const u8 *tc_group) { u32 in[MLX5_ST_SZ_DW(qetc_reg)] = {}; int i; for (i = 0; i <= mlx5_max_tc(mdev); i++) { MLX5_SET(qetc_reg, in, tc_configuration[i].g, 1); MLX5_SET(qetc_reg, in, tc_configuration[i].group, tc_group[i]); } return mlx5_set_port_qetcr_reg(mdev, in, sizeof(in)); } EXPORT_SYMBOL_GPL(mlx5_set_port_tc_group); int mlx5_query_port_tc_group(struct mlx5_core_dev *mdev, u8 tc, u8 *tc_group) { u32 out[MLX5_ST_SZ_DW(qetc_reg)]; void *ets_tcn_conf; int err; err = mlx5_query_port_qetcr_reg(mdev, out, sizeof(out)); if (err) return err; ets_tcn_conf = MLX5_ADDR_OF(qetc_reg, out, tc_configuration[tc]); *tc_group = MLX5_GET(ets_tcn_config_reg, ets_tcn_conf, group); return 0; } EXPORT_SYMBOL_GPL(mlx5_query_port_tc_group); int mlx5_set_port_tc_bw_alloc(struct mlx5_core_dev *mdev, const u8 *tc_bw) { u32 in[MLX5_ST_SZ_DW(qetc_reg)] = {}; int i; for (i = 0; i <= mlx5_max_tc(mdev); i++) { MLX5_SET(qetc_reg, in, tc_configuration[i].b, 1); MLX5_SET(qetc_reg, in, tc_configuration[i].bw_allocation, tc_bw[i]); } return mlx5_set_port_qetcr_reg(mdev, in, sizeof(in)); } EXPORT_SYMBOL_GPL(mlx5_set_port_tc_bw_alloc); int mlx5_query_port_tc_bw_alloc(struct mlx5_core_dev *mdev, u8 *bw_pct) { u32 out[MLX5_ST_SZ_DW(qetc_reg)]; void *ets_tcn_conf; int err; int i; err = mlx5_query_port_qetcr_reg(mdev, out, sizeof(out)); if (err) return err; for (i = 0; i <= mlx5_max_tc(mdev); i++) { ets_tcn_conf = MLX5_ADDR_OF(qetc_reg, out, tc_configuration[i]); bw_pct[i] = MLX5_GET(ets_tcn_config_reg, ets_tcn_conf, bw_allocation); } return 0; } EXPORT_SYMBOL_GPL(mlx5_query_port_tc_bw_alloc); int mlx5_modify_port_cong_params(struct mlx5_core_dev *mdev, void *in, int in_size) { u32 out[MLX5_ST_SZ_DW(modify_cong_params_out)] = {0}; MLX5_SET(modify_cong_params_in, in, opcode, MLX5_CMD_OP_MODIFY_CONG_PARAMS); return mlx5_cmd_exec(mdev, in, in_size, out, sizeof(out)); } int mlx5_query_port_cong_statistics(struct mlx5_core_dev *mdev, int clear, void *out, int out_size) { u32 in[MLX5_ST_SZ_DW(query_cong_statistics_in)] = {0}; MLX5_SET(query_cong_statistics_in, in, opcode, MLX5_CMD_OP_QUERY_CONG_STATISTICS); MLX5_SET(query_cong_statistics_in, in, clear, clear); return mlx5_cmd_exec(mdev, in, sizeof(in), out, out_size); } int mlx5_set_diagnostic_params(struct mlx5_core_dev *mdev, void *in, int in_size) { u32 out[MLX5_ST_SZ_DW(set_diagnostic_params_out)] = {0}; MLX5_SET(set_diagnostic_params_in, in, opcode, MLX5_CMD_OP_SET_DIAGNOSTICS); return mlx5_cmd_exec(mdev, in, in_size, out, sizeof(out)); } int mlx5_query_diagnostic_counters(struct mlx5_core_dev *mdev, u8 num_of_samples, u16 sample_index, void *out, int out_size) { u32 in[MLX5_ST_SZ_DW(query_diagnostic_counters_in)] = {0}; MLX5_SET(query_diagnostic_counters_in, in, opcode, MLX5_CMD_OP_QUERY_DIAGNOSTICS); MLX5_SET(query_diagnostic_counters_in, in, num_of_samples, num_of_samples); MLX5_SET(query_diagnostic_counters_in, in, sample_index, sample_index); return mlx5_cmd_exec(mdev, in, sizeof(in), out, out_size); } int mlx5_set_trust_state(struct mlx5_core_dev *mdev, u8 trust_state) { u32 out[MLX5_ST_SZ_DW(qpts_reg)] = {}; u32 in[MLX5_ST_SZ_DW(qpts_reg)] = {}; int err; MLX5_SET(qpts_reg, in, local_port, 1); MLX5_SET(qpts_reg, in, trust_state, trust_state); err = mlx5_core_access_reg(mdev, in, sizeof(in), out, sizeof(out), MLX5_REG_QPTS, 0, 1); return err; } int mlx5_query_trust_state(struct mlx5_core_dev *mdev, u8 *trust_state) { u32 out[MLX5_ST_SZ_DW(qpts_reg)] = {}; u32 in[MLX5_ST_SZ_DW(qpts_reg)] = {}; int err; MLX5_SET(qpts_reg, in, local_port, 1); err = mlx5_core_access_reg(mdev, in, sizeof(in), out, sizeof(out), MLX5_REG_QPTS, 0, 0); if (!err) *trust_state = MLX5_GET(qpts_reg, out, trust_state); return err; } int mlx5_set_dscp2prio(struct mlx5_core_dev *mdev, const u8 *dscp2prio) { int sz = MLX5_ST_SZ_BYTES(qpdpm_reg); void *qpdpm_dscp; void *out; void *in; int err; int i; in = kzalloc(sz, GFP_KERNEL); out = kzalloc(sz, GFP_KERNEL); if (!in || !out) { err = -ENOMEM; goto out; } MLX5_SET(qpdpm_reg, in, local_port, 1); err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_QPDPM, 0, 0); if (err) goto out; memcpy(in, out, sz); MLX5_SET(qpdpm_reg, in, local_port, 1); /* Update the corresponding dscp entry */ for (i = 0; i < MLX5_MAX_SUPPORTED_DSCP; i++) { qpdpm_dscp = MLX5_ADDR_OF(qpdpm_reg, in, dscp[i]); MLX5_SET16(qpdpm_dscp_reg, qpdpm_dscp, prio, dscp2prio[i]); MLX5_SET16(qpdpm_dscp_reg, qpdpm_dscp, e, 1); } err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_QPDPM, 0, 1); out: kfree(in); kfree(out); return err; } int mlx5_query_dscp2prio(struct mlx5_core_dev *mdev, u8 *dscp2prio) { int sz = MLX5_ST_SZ_BYTES(qpdpm_reg); void *qpdpm_dscp; void *out; void *in; int err; int i; in = kzalloc(sz, GFP_KERNEL); out = kzalloc(sz, GFP_KERNEL); if (!in || !out) { err = -ENOMEM; goto out; } MLX5_SET(qpdpm_reg, in, local_port, 1); err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_QPDPM, 0, 0); if (err) goto out; for (i = 0; i < MLX5_MAX_SUPPORTED_DSCP; i++) { qpdpm_dscp = MLX5_ADDR_OF(qpdpm_reg, out, dscp[i]); dscp2prio[i] = MLX5_GET16(qpdpm_dscp_reg, qpdpm_dscp, prio); } out: kfree(in); kfree(out); return err; } int mlx5_query_pddr_range_info(struct mlx5_core_dev *mdev, u8 local_port, u8 *is_er_type) { u32 pddr_reg[MLX5_ST_SZ_DW(pddr_reg)] = {}; int sz = MLX5_ST_SZ_BYTES(pddr_reg); int error; u8 ecc; u8 ci; MLX5_SET(pddr_reg, pddr_reg, local_port, local_port); MLX5_SET(pddr_reg, pddr_reg, page_select, 3 /* module info page */); error = mlx5_core_access_reg(mdev, pddr_reg, sz, pddr_reg, sz, MLX5_ACCESS_REG_SUMMARY_CTRL_ID_PDDR, 0, 0); if (error != 0) return (error); ecc = MLX5_GET(pddr_reg, pddr_reg, page_data.pddr_module_info.ethernet_compliance_code); ci = MLX5_GET(pddr_reg, pddr_reg, page_data.pddr_module_info.cable_identifier); switch (ci) { case 0: /* QSFP28 */ case 1: /* QSFP+ */ *is_er_type = 0; break; case 2: /* SFP28/SFP+ */ case 3: /* QSA (QSFP->SFP) */ *is_er_type = ((ecc & (1 << 7)) != 0); break; default: *is_er_type = 0; break; } return (0); } EXPORT_SYMBOL_GPL(mlx5_query_pddr_range_info); int mlx5_query_mfrl_reg(struct mlx5_core_dev *mdev, u8 *reset_level) { u32 mfrl[MLX5_ST_SZ_DW(mfrl_reg)] = {}; int sz = MLX5_ST_SZ_BYTES(mfrl_reg); int err; err = mlx5_core_access_reg(mdev, mfrl, sz, mfrl, sz, MLX5_REG_MFRL, 0, 0); if (err == 0) *reset_level = MLX5_GET(mfrl_reg, mfrl, reset_level); return (err); } int mlx5_set_mfrl_reg(struct mlx5_core_dev *mdev, u8 reset_level) { u32 mfrl[MLX5_ST_SZ_DW(mfrl_reg)] = {}; int sz = MLX5_ST_SZ_BYTES(mfrl_reg); MLX5_SET(mfrl_reg, mfrl, reset_level, reset_level); return (mlx5_core_access_reg(mdev, mfrl, sz, mfrl, sz, MLX5_REG_MFRL, 0, 1)); } Index: stable/11 =================================================================== --- stable/11 (revision 353223) +++ stable/11 (revision 353224) Property changes on: stable/11 ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r352975