Index: head/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c =================================================================== --- head/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c (revision 341555) +++ head/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c (revision 341556) @@ -1,2717 +1,2718 @@ /*- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include "mlx5_core.h" #include "fs_core.h" #include #include #define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\ sizeof(struct init_tree_node)) #define ADD_PRIO(name_val, flags_val, min_level_val, max_ft_val, caps_val, \ ...) {.type = FS_TYPE_PRIO,\ .name = name_val,\ .min_ft_level = min_level_val,\ .flags = flags_val,\ .max_ft = max_ft_val,\ .caps = caps_val,\ .children = (struct init_tree_node[]) {__VA_ARGS__},\ .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ } #define ADD_FT_PRIO(name_val, flags_val, max_ft_val, ...)\ ADD_PRIO(name_val, flags_val, 0, max_ft_val, {},\ __VA_ARGS__)\ #define ADD_NS(name_val, ...) {.type = FS_TYPE_NAMESPACE,\ .name = name_val,\ .children = (struct init_tree_node[]) {__VA_ARGS__},\ .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \ } #define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\ sizeof(long)) #define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap)) #define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \ .caps = (long[]) {__VA_ARGS__}} #define BYPASS_MAX_FT 5 #define BYPASS_PRIO_MAX_FT 1 #define KERNEL_MAX_FT 3 #define LEFTOVER_MAX_FT 1 #define KENREL_MIN_LEVEL 3 #define LEFTOVER_MIN_LEVEL KENREL_MIN_LEVEL + 1 #define BYPASS_MIN_LEVEL MLX5_NUM_BYPASS_FTS + LEFTOVER_MIN_LEVEL struct node_caps { size_t arr_sz; long *caps; }; struct init_tree_node { enum fs_type type; const char *name; struct init_tree_node *children; int ar_size; struct node_caps caps; u8 flags; int min_ft_level; int prio; int max_ft; } root_fs = { .type = FS_TYPE_NAMESPACE, .name = "root", .ar_size = 3, .children = (struct init_tree_node[]) { ADD_PRIO("by_pass_prio", 0, BYPASS_MIN_LEVEL, 0, FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), FS_CAP(flow_table_properties_nic_receive.modify_root)), ADD_NS("by_pass_ns", ADD_FT_PRIO("prio0", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio1", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio2", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio3", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio4", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio5", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio6", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio7", 0, BYPASS_PRIO_MAX_FT), ADD_FT_PRIO("prio-mcast", 0, BYPASS_PRIO_MAX_FT))), ADD_PRIO("kernel_prio", 0, KENREL_MIN_LEVEL, 0, {}, ADD_NS("kernel_ns", ADD_FT_PRIO("prio_kernel-0", 0, KERNEL_MAX_FT))), ADD_PRIO("leftovers_prio", MLX5_CORE_FS_PRIO_SHARED, LEFTOVER_MIN_LEVEL, 0, FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), FS_CAP(flow_table_properties_nic_receive.modify_root)), ADD_NS("leftover_ns", ADD_FT_PRIO("leftovers_prio-0", MLX5_CORE_FS_PRIO_SHARED, LEFTOVER_MAX_FT))) } }; /* Tree creation functions */ static struct mlx5_flow_root_namespace *find_root(struct fs_base *node) { struct fs_base *parent; /* Make sure we only read it once while we go up the tree */ while ((parent = node->parent)) node = parent; if (node->type != FS_TYPE_NAMESPACE) { printf("mlx5_core: WARN: ""mlx5: flow steering node %s is not in tree or garbaged\n", node->name); return NULL; } return container_of(container_of(node, struct mlx5_flow_namespace, base), struct mlx5_flow_root_namespace, ns); } static inline struct mlx5_core_dev *fs_get_dev(struct fs_base *node) { struct mlx5_flow_root_namespace *root = find_root(node); if (root) return root->dev; return NULL; } static void fs_init_node(struct fs_base *node, unsigned int refcount) { kref_init(&node->refcount); atomic_set(&node->users_refcount, refcount); init_completion(&node->complete); INIT_LIST_HEAD(&node->list); mutex_init(&node->lock); } static void _fs_add_node(struct fs_base *node, const char *name, struct fs_base *parent) { if (parent) atomic_inc(&parent->users_refcount); node->name = kstrdup_const(name, GFP_KERNEL); node->parent = parent; } static void fs_add_node(struct fs_base *node, struct fs_base *parent, const char *name, unsigned int refcount) { fs_init_node(node, refcount); _fs_add_node(node, name, parent); } static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref), bool parent_locked); static void fs_del_dst(struct mlx5_flow_rule *dst); static void _fs_del_ft(struct mlx5_flow_table *ft); static void fs_del_fg(struct mlx5_flow_group *fg); static void fs_del_fte(struct fs_fte *fte); static void cmd_remove_node(struct fs_base *base) { switch (base->type) { case FS_TYPE_FLOW_DEST: fs_del_dst(container_of(base, struct mlx5_flow_rule, base)); break; case FS_TYPE_FLOW_TABLE: _fs_del_ft(container_of(base, struct mlx5_flow_table, base)); break; case FS_TYPE_FLOW_GROUP: fs_del_fg(container_of(base, struct mlx5_flow_group, base)); break; case FS_TYPE_FLOW_ENTRY: fs_del_fte(container_of(base, struct fs_fte, base)); break; default: break; } } static void __fs_remove_node(struct kref *kref) { struct fs_base *node = container_of(kref, struct fs_base, refcount); if (node->parent) mutex_lock(&node->parent->lock); mutex_lock(&node->lock); cmd_remove_node(node); mutex_unlock(&node->lock); complete(&node->complete); if (node->parent) { mutex_unlock(&node->parent->lock); _fs_put(node->parent, _fs_remove_node, false); } } void _fs_remove_node(struct kref *kref) { struct fs_base *node = container_of(kref, struct fs_base, refcount); __fs_remove_node(kref); kfree_const(node->name); kfree(node); } static void fs_get(struct fs_base *node) { atomic_inc(&node->users_refcount); } static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref), bool parent_locked) { struct fs_base *parent_node = node->parent; if (parent_node && !parent_locked) mutex_lock(&parent_node->lock); if (atomic_dec_and_test(&node->users_refcount)) { if (parent_node) { /*remove from parent's list*/ list_del_init(&node->list); mutex_unlock(&parent_node->lock); } kref_put(&node->refcount, kref_cb); if (parent_node && parent_locked) mutex_lock(&parent_node->lock); } else if (parent_node && !parent_locked) { mutex_unlock(&parent_node->lock); } } static void fs_put(struct fs_base *node) { _fs_put(node, __fs_remove_node, false); } static void fs_put_parent_locked(struct fs_base *node) { _fs_put(node, __fs_remove_node, true); } static void fs_remove_node(struct fs_base *node) { fs_put(node); wait_for_completion(&node->complete); kfree_const(node->name); kfree(node); } static void fs_remove_node_parent_locked(struct fs_base *node) { fs_put_parent_locked(node); wait_for_completion(&node->complete); kfree_const(node->name); kfree(node); } static struct fs_fte *fs_alloc_fte(u8 action, u32 flow_tag, u32 *match_value, unsigned int index) { struct fs_fte *fte; fte = kzalloc(sizeof(*fte), GFP_KERNEL); if (!fte) return ERR_PTR(-ENOMEM); memcpy(fte->val, match_value, sizeof(fte->val)); fte->base.type = FS_TYPE_FLOW_ENTRY; fte->dests_size = 0; fte->flow_tag = flow_tag; fte->index = index; INIT_LIST_HEAD(&fte->dests); fte->action = action; return fte; } static struct fs_fte *alloc_star_ft_entry(struct mlx5_flow_table *ft, struct mlx5_flow_group *fg, u32 *match_value, unsigned int index) { int err; struct fs_fte *fte; struct mlx5_flow_rule *dst; if (fg->num_ftes == fg->max_ftes) return ERR_PTR(-ENOSPC); fte = fs_alloc_fte(MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_DEFAULT_FLOW_TAG, match_value, index); if (IS_ERR(fte)) return fte; /*create dst*/ dst = kzalloc(sizeof(*dst), GFP_KERNEL); if (!dst) { err = -ENOMEM; goto free_fte; } fte->base.parent = &fg->base; fte->dests_size = 1; dst->dest_attr.type = MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE; dst->base.parent = &fte->base; list_add(&dst->base.list, &fte->dests); /* assumed that the callee creates the star rules sorted by index */ list_add_tail(&fte->base.list, &fg->ftes); fg->num_ftes++; return fte; free_fte: kfree(fte); return ERR_PTR(err); } /* assume that fte can't be changed */ static void free_star_fte_entry(struct fs_fte *fte) { struct mlx5_flow_group *fg; struct mlx5_flow_rule *dst, *temp; fs_get_parent(fg, fte); list_for_each_entry_safe(dst, temp, &fte->dests, base.list) { fte->dests_size--; list_del(&dst->base.list); kfree(dst); } list_del(&fte->base.list); fg->num_ftes--; kfree(fte); } static struct mlx5_flow_group *fs_alloc_fg(u32 *create_fg_in) { struct mlx5_flow_group *fg; void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, create_fg_in, match_criteria); u8 match_criteria_enable = MLX5_GET(create_flow_group_in, create_fg_in, match_criteria_enable); fg = kzalloc(sizeof(*fg), GFP_KERNEL); if (!fg) return ERR_PTR(-ENOMEM); INIT_LIST_HEAD(&fg->ftes); fg->mask.match_criteria_enable = match_criteria_enable; memcpy(&fg->mask.match_criteria, match_criteria, sizeof(fg->mask.match_criteria)); fg->base.type = FS_TYPE_FLOW_GROUP; fg->start_index = MLX5_GET(create_flow_group_in, create_fg_in, start_flow_index); fg->max_ftes = MLX5_GET(create_flow_group_in, create_fg_in, end_flow_index) - fg->start_index + 1; return fg; } static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio); static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr, struct fs_prio *prio); /* assumed src_ft and dst_ft can't be freed */ static int fs_set_star_rule(struct mlx5_core_dev *dev, struct mlx5_flow_table *src_ft, struct mlx5_flow_table *dst_ft) { struct mlx5_flow_rule *src_dst; struct fs_fte *src_fte; int err = 0; u32 *match_value; int match_len = MLX5_ST_SZ_BYTES(fte_match_param); src_dst = list_first_entry(&src_ft->star_rule.fte->dests, struct mlx5_flow_rule, base.list); match_value = mlx5_vzalloc(match_len); if (!match_value) { mlx5_core_warn(dev, "failed to allocate inbox\n"); return -ENOMEM; } /*Create match context*/ fs_get_parent(src_fte, src_dst); src_dst->dest_attr.ft = dst_ft; if (dst_ft) { err = mlx5_cmd_fs_set_fte(dev, src_ft->vport, &src_fte->status, match_value, src_ft->type, src_ft->id, src_fte->index, src_ft->star_rule.fg->id, src_fte->flow_tag, src_fte->action, src_fte->dests_size, &src_fte->dests); if (err) goto free; fs_get(&dst_ft->base); } else { mlx5_cmd_fs_delete_fte(dev, src_ft->vport, &src_fte->status, src_ft->type, src_ft->id, src_fte->index); } free: kvfree(match_value); return err; } static int connect_prev_fts(struct fs_prio *locked_prio, struct fs_prio *prev_prio, struct mlx5_flow_table *next_ft) { struct mlx5_flow_table *iter; int err = 0; struct mlx5_core_dev *dev = fs_get_dev(&prev_prio->base); if (!dev) return -ENODEV; mutex_lock(&prev_prio->base.lock); fs_for_each_ft(iter, prev_prio) { struct mlx5_flow_rule *src_dst = list_first_entry(&iter->star_rule.fte->dests, struct mlx5_flow_rule, base.list); struct mlx5_flow_table *prev_ft = src_dst->dest_attr.ft; if (prev_ft == next_ft) continue; err = fs_set_star_rule(dev, iter, next_ft); if (err) { mlx5_core_warn(dev, "mlx5: flow steering can't connect prev and next\n"); goto unlock; } else { /* Assume ft's prio is locked */ if (prev_ft) { struct fs_prio *prio; fs_get_parent(prio, prev_ft); if (prio == locked_prio) fs_put_parent_locked(&prev_ft->base); else fs_put(&prev_ft->base); } } } unlock: mutex_unlock(&prev_prio->base.lock); return 0; } static int create_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio) { struct mlx5_flow_group *fg; int err; u32 *fg_in; u32 *match_value; struct mlx5_flow_table *next_ft; struct mlx5_flow_table *prev_ft; struct mlx5_flow_root_namespace *root = find_root(&prio->base); int fg_inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); int match_len = MLX5_ST_SZ_BYTES(fte_match_param); fg_in = mlx5_vzalloc(fg_inlen); if (!fg_in) { mlx5_core_warn(root->dev, "failed to allocate inbox\n"); return -ENOMEM; } match_value = mlx5_vzalloc(match_len); if (!match_value) { mlx5_core_warn(root->dev, "failed to allocate inbox\n"); kvfree(fg_in); return -ENOMEM; } MLX5_SET(create_flow_group_in, fg_in, start_flow_index, ft->max_fte); MLX5_SET(create_flow_group_in, fg_in, end_flow_index, ft->max_fte); fg = fs_alloc_fg(fg_in); if (IS_ERR(fg)) { err = PTR_ERR(fg); goto out; } ft->star_rule.fg = fg; err = mlx5_cmd_fs_create_fg(fs_get_dev(&prio->base), fg_in, ft->vport, ft->type, ft->id, &fg->id); if (err) goto free_fg; ft->star_rule.fte = alloc_star_ft_entry(ft, fg, match_value, ft->max_fte); if (IS_ERR(ft->star_rule.fte)) goto free_star_rule; mutex_lock(&root->fs_chain_lock); next_ft = find_next_ft(prio); err = fs_set_star_rule(root->dev, ft, next_ft); if (err) { mutex_unlock(&root->fs_chain_lock); goto free_star_rule; } if (next_ft) { struct fs_prio *parent; fs_get_parent(parent, next_ft); fs_put(&next_ft->base); } prev_ft = find_prev_ft(ft, prio); if (prev_ft) { struct fs_prio *prev_parent; fs_get_parent(prev_parent, prev_ft); err = connect_prev_fts(NULL, prev_parent, ft); if (err) { mutex_unlock(&root->fs_chain_lock); goto destroy_chained_star_rule; } fs_put(&prev_ft->base); } mutex_unlock(&root->fs_chain_lock); kvfree(fg_in); kvfree(match_value); return 0; destroy_chained_star_rule: fs_set_star_rule(fs_get_dev(&prio->base), ft, NULL); if (next_ft) fs_put(&next_ft->base); free_star_rule: free_star_fte_entry(ft->star_rule.fte); mlx5_cmd_fs_destroy_fg(fs_get_dev(&ft->base), ft->vport, ft->type, ft->id, fg->id); free_fg: kfree(fg); out: kvfree(fg_in); kvfree(match_value); return err; } static void destroy_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio) { int err; struct mlx5_flow_root_namespace *root; struct mlx5_core_dev *dev = fs_get_dev(&prio->base); struct mlx5_flow_table *prev_ft, *next_ft; struct fs_prio *prev_prio; WARN_ON(!dev); root = find_root(&prio->base); if (!root) printf("mlx5_core: ERR: ""mlx5: flow steering failed to find root of priority %s", prio->base.name); /* In order to ensure atomic deletion, first update * prev ft to point on the next ft. */ mutex_lock(&root->fs_chain_lock); prev_ft = find_prev_ft(ft, prio); next_ft = find_next_ft(prio); if (prev_ft) { fs_get_parent(prev_prio, prev_ft); /*Prev is connected to ft, only if ft is the first(last) in the prio*/ err = connect_prev_fts(prio, prev_prio, next_ft); if (err) mlx5_core_warn(root->dev, "flow steering can't connect prev and next of flow table\n"); fs_put(&prev_ft->base); } err = fs_set_star_rule(root->dev, ft, NULL); /*One put is for fs_get in find next ft*/ if (next_ft) { fs_put(&next_ft->base); if (!err) fs_put(&next_ft->base); } mutex_unlock(&root->fs_chain_lock); err = mlx5_cmd_fs_destroy_fg(dev, ft->vport, ft->type, ft->id, ft->star_rule.fg->id); if (err) mlx5_core_warn(dev, "flow steering can't destroy star entry group(index:%d) of ft:%s\n", ft->star_rule.fg->start_index, ft->base.name); free_star_fte_entry(ft->star_rule.fte); kfree(ft->star_rule.fg); ft->star_rule.fg = NULL; } static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns, unsigned int prio) { struct fs_prio *iter_prio; fs_for_each_prio(iter_prio, ns) { if (iter_prio->prio == prio) return iter_prio; } return NULL; } static unsigned int _alloc_new_level(struct fs_prio *prio, struct mlx5_flow_namespace *match); static unsigned int __alloc_new_level(struct mlx5_flow_namespace *ns, struct fs_prio *prio) { unsigned int level = 0; struct fs_prio *p; if (!ns) return 0; mutex_lock(&ns->base.lock); fs_for_each_prio(p, ns) { if (p != prio) level += p->max_ft; else break; } mutex_unlock(&ns->base.lock); fs_get_parent(prio, ns); if (prio) WARN_ON(prio->base.type != FS_TYPE_PRIO); return level + _alloc_new_level(prio, ns); } /* Called under lock of priority, hence locking all upper objects */ static unsigned int _alloc_new_level(struct fs_prio *prio, struct mlx5_flow_namespace *match) { struct mlx5_flow_namespace *ns; struct fs_base *it; unsigned int level = 0; if (!prio) return 0; mutex_lock(&prio->base.lock); fs_for_each_ns_or_ft_reverse(it, prio) { if (it->type == FS_TYPE_NAMESPACE) { struct fs_prio *p; fs_get_obj(ns, it); if (match != ns) { mutex_lock(&ns->base.lock); fs_for_each_prio(p, ns) level += p->max_ft; mutex_unlock(&ns->base.lock); } else { break; } } else { struct mlx5_flow_table *ft; fs_get_obj(ft, it); mutex_unlock(&prio->base.lock); return level + ft->level + 1; } } fs_get_parent(ns, prio); mutex_unlock(&prio->base.lock); return __alloc_new_level(ns, prio) + level; } static unsigned int alloc_new_level(struct fs_prio *prio) { return _alloc_new_level(prio, NULL); } static int update_root_ft_create(struct mlx5_flow_root_namespace *root, struct mlx5_flow_table *ft) { int err = 0; int min_level = INT_MAX; if (root->root_ft) min_level = root->root_ft->level; if (ft->level < min_level) err = mlx5_cmd_update_root_ft(root->dev, ft->type, ft->id); else return err; if (err) mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n", ft->id); else root->root_ft = ft; return err; } static struct mlx5_flow_table *_create_ft_common(struct mlx5_flow_namespace *ns, u16 vport, struct fs_prio *fs_prio, int max_fte, const char *name) { struct mlx5_flow_table *ft; int err; int log_table_sz; int ft_size; char gen_name[20]; struct mlx5_flow_root_namespace *root = find_root(&ns->base); if (!root) { printf("mlx5_core: ERR: ""mlx5: flow steering failed to find root of namespace %s", ns->base.name); return ERR_PTR(-ENODEV); } if (fs_prio->num_ft == fs_prio->max_ft) return ERR_PTR(-ENOSPC); ft = kzalloc(sizeof(*ft), GFP_KERNEL); if (!ft) return ERR_PTR(-ENOMEM); fs_init_node(&ft->base, 1); INIT_LIST_HEAD(&ft->fgs); /* Temporarily WA until we expose the level set in the API */ if (root->table_type == FS_FT_ESW_EGRESS_ACL || root->table_type == FS_FT_ESW_INGRESS_ACL) ft->level = 0; else ft->level = alloc_new_level(fs_prio); ft->base.type = FS_TYPE_FLOW_TABLE; ft->vport = vport; ft->type = root->table_type; /*Two entries are reserved for star rules*/ ft_size = roundup_pow_of_two(max_fte + 2); /*User isn't aware to those rules*/ ft->max_fte = ft_size - 2; log_table_sz = ilog2(ft_size); err = mlx5_cmd_fs_create_ft(root->dev, ft->vport, ft->type, ft->level, log_table_sz, &ft->id); if (err) goto free_ft; err = create_star_rule(ft, fs_prio); if (err) goto del_ft; if ((root->table_type == FS_FT_NIC_RX) && MLX5_CAP_FLOWTABLE(root->dev, flow_table_properties_nic_receive.modify_root)) { err = update_root_ft_create(root, ft); if (err) goto destroy_star_rule; } if (!name || !strlen(name)) { snprintf(gen_name, 20, "flow_table_%u", ft->id); _fs_add_node(&ft->base, gen_name, &fs_prio->base); } else { _fs_add_node(&ft->base, name, &fs_prio->base); } list_add_tail(&ft->base.list, &fs_prio->objs); fs_prio->num_ft++; return ft; destroy_star_rule: destroy_star_rule(ft, fs_prio); del_ft: mlx5_cmd_fs_destroy_ft(root->dev, ft->vport, ft->type, ft->id); free_ft: kfree(ft); return ERR_PTR(err); } static struct mlx5_flow_table *create_ft_common(struct mlx5_flow_namespace *ns, u16 vport, unsigned int prio, int max_fte, const char *name) { struct fs_prio *fs_prio = NULL; fs_prio = find_prio(ns, prio); if (!fs_prio) return ERR_PTR(-EINVAL); return _create_ft_common(ns, vport, fs_prio, max_fte, name); } static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns, struct list_head *start); static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio, struct list_head *start); static struct mlx5_flow_table *mlx5_create_autogrouped_shared_flow_table(struct fs_prio *fs_prio) { struct mlx5_flow_table *ft; ft = find_first_ft_in_prio(fs_prio, &fs_prio->objs); if (ft) { ft->shared_refcount++; return ft; } return NULL; } struct mlx5_flow_table *mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns, int prio, const char *name, int num_flow_table_entries, int max_num_groups) { struct mlx5_flow_table *ft = NULL; struct fs_prio *fs_prio; bool is_shared_prio; fs_prio = find_prio(ns, prio); if (!fs_prio) return ERR_PTR(-EINVAL); is_shared_prio = fs_prio->flags & MLX5_CORE_FS_PRIO_SHARED; if (is_shared_prio) { mutex_lock(&fs_prio->shared_lock); ft = mlx5_create_autogrouped_shared_flow_table(fs_prio); } if (ft) goto return_ft; ft = create_ft_common(ns, 0, prio, num_flow_table_entries, name); if (IS_ERR(ft)) goto return_ft; ft->autogroup.active = true; ft->autogroup.max_types = max_num_groups; if (is_shared_prio) ft->shared_refcount = 1; return_ft: if (is_shared_prio) mutex_unlock(&fs_prio->shared_lock); return ft; } EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table); struct mlx5_flow_table *mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns, u16 vport, int prio, const char *name, int num_flow_table_entries) { return create_ft_common(ns, vport, prio, num_flow_table_entries, name); } EXPORT_SYMBOL(mlx5_create_vport_flow_table); struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns, int prio, const char *name, int num_flow_table_entries) { return create_ft_common(ns, 0, prio, num_flow_table_entries, name); } EXPORT_SYMBOL(mlx5_create_flow_table); static void _fs_del_ft(struct mlx5_flow_table *ft) { int err; struct mlx5_core_dev *dev = fs_get_dev(&ft->base); struct fs_prio *prio; err = mlx5_cmd_fs_destroy_ft(dev, ft->vport, ft->type, ft->id); if (err) mlx5_core_warn(dev, "flow steering can't destroy ft %s\n", ft->base.name); fs_get_parent(prio, ft); prio->num_ft--; } static int update_root_ft_destroy(struct mlx5_flow_root_namespace *root, struct mlx5_flow_table *ft) { int err = 0; struct fs_prio *prio; struct mlx5_flow_table *next_ft = NULL; struct mlx5_flow_table *put_ft = NULL; if (root->root_ft != ft) return 0; fs_get_parent(prio, ft); /*Assuming objs containis only flow tables and * flow tables are sorted by level. */ if (!list_is_last(&ft->base.list, &prio->objs)) { next_ft = list_next_entry(ft, base.list); } else { next_ft = find_next_ft(prio); put_ft = next_ft; } if (next_ft) { err = mlx5_cmd_update_root_ft(root->dev, next_ft->type, next_ft->id); if (err) mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n", ft->id); } if (!err) root->root_ft = next_ft; if (put_ft) fs_put(&put_ft->base); return err; } /*Objects in the same prio are destroyed in the reverse order they were createrd*/ int mlx5_destroy_flow_table(struct mlx5_flow_table *ft) { int err = 0; struct fs_prio *prio; struct mlx5_flow_root_namespace *root; bool is_shared_prio; fs_get_parent(prio, ft); root = find_root(&prio->base); if (!root) { printf("mlx5_core: ERR: ""mlx5: flow steering failed to find root of priority %s", prio->base.name); return -ENODEV; } is_shared_prio = prio->flags & MLX5_CORE_FS_PRIO_SHARED; if (is_shared_prio) { mutex_lock(&prio->shared_lock); if (ft->shared_refcount > 1) { --ft->shared_refcount; fs_put(&ft->base); mutex_unlock(&prio->shared_lock); return 0; } } mutex_lock(&prio->base.lock); mutex_lock(&ft->base.lock); err = update_root_ft_destroy(root, ft); if (err) goto unlock_ft; /* delete two last entries */ destroy_star_rule(ft, prio); mutex_unlock(&ft->base.lock); fs_remove_node_parent_locked(&ft->base); mutex_unlock(&prio->base.lock); if (is_shared_prio) mutex_unlock(&prio->shared_lock); return err; unlock_ft: mutex_unlock(&ft->base.lock); mutex_unlock(&prio->base.lock); if (is_shared_prio) mutex_unlock(&prio->shared_lock); return err; } EXPORT_SYMBOL(mlx5_destroy_flow_table); static struct mlx5_flow_group *fs_create_fg(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft, struct list_head *prev, u32 *fg_in, int refcount) { struct mlx5_flow_group *fg; int err; unsigned int end_index; char name[20]; fg = fs_alloc_fg(fg_in); if (IS_ERR(fg)) return fg; end_index = fg->start_index + fg->max_ftes - 1; err = mlx5_cmd_fs_create_fg(dev, fg_in, ft->vport, ft->type, ft->id, &fg->id); if (err) goto free_fg; mutex_lock(&ft->base.lock); if (ft->autogroup.active) ft->autogroup.num_types++; snprintf(name, sizeof(name), "group_%u", fg->id); /*Add node to tree*/ fs_add_node(&fg->base, &ft->base, name, refcount); /*Add node to group list*/ list_add(&fg->base.list, prev); mutex_unlock(&ft->base.lock); return fg; free_fg: kfree(fg); return ERR_PTR(err); } struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft, u32 *in) { struct mlx5_flow_group *fg; struct mlx5_core_dev *dev = fs_get_dev(&ft->base); if (!dev) return ERR_PTR(-ENODEV); if (ft->autogroup.active) return ERR_PTR(-EPERM); fg = fs_create_fg(dev, ft, ft->fgs.prev, in, 1); return fg; } EXPORT_SYMBOL(mlx5_create_flow_group); /*Group is destoyed when all the rules in the group were removed*/ static void fs_del_fg(struct mlx5_flow_group *fg) { struct mlx5_flow_table *parent_ft; struct mlx5_core_dev *dev; fs_get_parent(parent_ft, fg); dev = fs_get_dev(&parent_ft->base); WARN_ON(!dev); if (parent_ft->autogroup.active) parent_ft->autogroup.num_types--; if (mlx5_cmd_fs_destroy_fg(dev, parent_ft->vport, parent_ft->type, parent_ft->id, fg->id)) mlx5_core_warn(dev, "flow steering can't destroy fg\n"); } void mlx5_destroy_flow_group(struct mlx5_flow_group *fg) { fs_remove_node(&fg->base); } EXPORT_SYMBOL(mlx5_destroy_flow_group); static bool _fs_match_exact_val(void *mask, void *val1, void *val2, size_t size) { unsigned int i; /* TODO: optimize by comparing 64bits when possible */ for (i = 0; i < size; i++, mask++, val1++, val2++) if ((*((u8 *)val1) & (*(u8 *)mask)) != ((*(u8 *)val2) & (*(u8 *)mask))) return false; return true; } bool fs_match_exact_val(struct mlx5_core_fs_mask *mask, void *val1, void *val2) { if (mask->match_criteria_enable & 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS) { void *fte_match1 = MLX5_ADDR_OF(fte_match_param, val1, outer_headers); void *fte_match2 = MLX5_ADDR_OF(fte_match_param, val2, outer_headers); void *fte_mask = MLX5_ADDR_OF(fte_match_param, mask->match_criteria, outer_headers); if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4))) return false; } if (mask->match_criteria_enable & 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) { void *fte_match1 = MLX5_ADDR_OF(fte_match_param, val1, misc_parameters); void *fte_match2 = MLX5_ADDR_OF(fte_match_param, val2, misc_parameters); void *fte_mask = MLX5_ADDR_OF(fte_match_param, mask->match_criteria, misc_parameters); if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, MLX5_ST_SZ_BYTES(fte_match_set_misc))) return false; } if (mask->match_criteria_enable & 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS) { void *fte_match1 = MLX5_ADDR_OF(fte_match_param, val1, inner_headers); void *fte_match2 = MLX5_ADDR_OF(fte_match_param, val2, inner_headers); void *fte_mask = MLX5_ADDR_OF(fte_match_param, mask->match_criteria, inner_headers); if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2, MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4))) return false; } return true; } bool fs_match_exact_mask(u8 match_criteria_enable1, u8 match_criteria_enable2, void *mask1, void *mask2) { return match_criteria_enable1 == match_criteria_enable2 && !memcmp(mask1, mask2, MLX5_ST_SZ_BYTES(fte_match_param)); } static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns, struct list_head *start); static struct mlx5_flow_table *_find_first_ft_in_prio_reverse(struct fs_prio *prio, struct list_head *start) { struct fs_base *it = container_of(start, struct fs_base, list); if (!prio) return NULL; fs_for_each_ns_or_ft_continue_reverse(it, prio) { struct mlx5_flow_namespace *ns; struct mlx5_flow_table *ft; if (it->type == FS_TYPE_FLOW_TABLE) { fs_get_obj(ft, it); fs_get(&ft->base); return ft; } fs_get_obj(ns, it); WARN_ON(ns->base.type != FS_TYPE_NAMESPACE); ft = find_first_ft_in_ns_reverse(ns, &ns->prios); if (ft) return ft; } return NULL; } static struct mlx5_flow_table *find_first_ft_in_prio_reverse(struct fs_prio *prio, struct list_head *start) { struct mlx5_flow_table *ft; if (!prio) return NULL; mutex_lock(&prio->base.lock); ft = _find_first_ft_in_prio_reverse(prio, start); mutex_unlock(&prio->base.lock); return ft; } static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns, struct list_head *start) { struct fs_prio *prio; if (!ns) return NULL; fs_get_obj(prio, container_of(start, struct fs_base, list)); mutex_lock(&ns->base.lock); fs_for_each_prio_continue_reverse(prio, ns) { struct mlx5_flow_table *ft; ft = find_first_ft_in_prio_reverse(prio, &prio->objs); if (ft) { mutex_unlock(&ns->base.lock); return ft; } } mutex_unlock(&ns->base.lock); return NULL; } /* Returned a held ft, assumed curr is protected, assumed curr's parent is * locked */ static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr, struct fs_prio *prio) { struct mlx5_flow_table *ft = NULL; struct fs_base *curr_base; if (!curr) return NULL; /* prio has either namespace or flow-tables, but not both */ if (!list_empty(&prio->objs) && list_first_entry(&prio->objs, struct mlx5_flow_table, base.list) != curr) return NULL; while (!ft && prio) { struct mlx5_flow_namespace *ns; fs_get_parent(ns, prio); ft = find_first_ft_in_ns_reverse(ns, &prio->base.list); curr_base = &ns->base; fs_get_parent(prio, ns); if (prio && !ft) ft = find_first_ft_in_prio_reverse(prio, &curr_base->list); } return ft; } static struct mlx5_flow_table *_find_first_ft_in_prio(struct fs_prio *prio, struct list_head *start) { struct fs_base *it = container_of(start, struct fs_base, list); if (!prio) return NULL; fs_for_each_ns_or_ft_continue(it, prio) { struct mlx5_flow_namespace *ns; struct mlx5_flow_table *ft; if (it->type == FS_TYPE_FLOW_TABLE) { fs_get_obj(ft, it); fs_get(&ft->base); return ft; } fs_get_obj(ns, it); WARN_ON(ns->base.type != FS_TYPE_NAMESPACE); ft = find_first_ft_in_ns(ns, &ns->prios); if (ft) return ft; } return NULL; } static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio, struct list_head *start) { struct mlx5_flow_table *ft; if (!prio) return NULL; mutex_lock(&prio->base.lock); ft = _find_first_ft_in_prio(prio, start); mutex_unlock(&prio->base.lock); return ft; } static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns, struct list_head *start) { struct fs_prio *prio; if (!ns) return NULL; fs_get_obj(prio, container_of(start, struct fs_base, list)); mutex_lock(&ns->base.lock); fs_for_each_prio_continue(prio, ns) { struct mlx5_flow_table *ft; ft = find_first_ft_in_prio(prio, &prio->objs); if (ft) { mutex_unlock(&ns->base.lock); return ft; } } mutex_unlock(&ns->base.lock); return NULL; } /* returned a held ft, assumed curr is protected, assumed curr's parent is * locked */ static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio) { struct mlx5_flow_table *ft = NULL; struct fs_base *curr_base; while (!ft && prio) { struct mlx5_flow_namespace *ns; fs_get_parent(ns, prio); ft = find_first_ft_in_ns(ns, &prio->base.list); curr_base = &ns->base; fs_get_parent(prio, ns); if (!ft && prio) ft = _find_first_ft_in_prio(prio, &curr_base->list); } return ft; } /* called under ft mutex lock */ static struct mlx5_flow_group *create_autogroup(struct mlx5_flow_table *ft, u8 match_criteria_enable, u32 *match_criteria) { unsigned int group_size; unsigned int candidate_index = 0; unsigned int candidate_group_num = 0; struct mlx5_flow_group *g; struct mlx5_flow_group *ret; struct list_head *prev = &ft->fgs; struct mlx5_core_dev *dev; u32 *in; int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); void *match_criteria_addr; if (!ft->autogroup.active) return ERR_PTR(-ENOENT); dev = fs_get_dev(&ft->base); if (!dev) return ERR_PTR(-ENODEV); in = mlx5_vzalloc(inlen); if (!in) { mlx5_core_warn(dev, "failed to allocate inbox\n"); return ERR_PTR(-ENOMEM); } if (ft->autogroup.num_types < ft->autogroup.max_types) group_size = ft->max_fte / (ft->autogroup.max_types + 1); else group_size = 1; if (group_size == 0) { mlx5_core_warn(dev, "flow steering can't create group size of 0\n"); ret = ERR_PTR(-EINVAL); goto out; } /* sorted by start_index */ fs_for_each_fg(g, ft) { candidate_group_num++; if (candidate_index + group_size > g->start_index) candidate_index = g->start_index + g->max_ftes; else break; prev = &g->base.list; } if (candidate_index + group_size > ft->max_fte) { ret = ERR_PTR(-ENOSPC); goto out; } MLX5_SET(create_flow_group_in, in, match_criteria_enable, match_criteria_enable); MLX5_SET(create_flow_group_in, in, start_flow_index, candidate_index); MLX5_SET(create_flow_group_in, in, end_flow_index, candidate_index + group_size - 1); match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); memcpy(match_criteria_addr, match_criteria, MLX5_ST_SZ_BYTES(fte_match_param)); ret = fs_create_fg(dev, ft, prev, in, 0); out: kvfree(in); return ret; } static struct mlx5_flow_namespace *get_ns_with_notifiers(struct fs_base *node) { struct mlx5_flow_namespace *ns = NULL; while (node && (node->type != FS_TYPE_NAMESPACE || list_empty(&container_of(node, struct mlx5_flow_namespace, base)->list_notifiers))) node = node->parent; if (node) fs_get_obj(ns, node); return ns; } /*Assumption- fte is locked*/ static void call_to_add_rule_notifiers(struct mlx5_flow_rule *dst, struct fs_fte *fte) { struct mlx5_flow_namespace *ns; struct mlx5_flow_handler *iter_handler; struct fs_client_priv_data *iter_client; void *data; bool is_new_rule = list_first_entry(&fte->dests, struct mlx5_flow_rule, base.list) == dst; int err; ns = get_ns_with_notifiers(&fte->base); if (!ns) return; down_read(&ns->notifiers_rw_sem); list_for_each_entry(iter_handler, &ns->list_notifiers, list) { if (iter_handler->add_dst_cb) { data = NULL; mutex_lock(&dst->clients_lock); list_for_each_entry( iter_client, &dst->clients_data, list) { if (iter_client->fs_handler == iter_handler) { data = iter_client->client_dst_data; break; } } mutex_unlock(&dst->clients_lock); err = iter_handler->add_dst_cb(dst, is_new_rule, NULL, iter_handler->client_context); if (err) break; } } up_read(&ns->notifiers_rw_sem); } static void call_to_del_rule_notifiers(struct mlx5_flow_rule *dst, struct fs_fte *fte) { struct mlx5_flow_namespace *ns; struct mlx5_flow_handler *iter_handler; struct fs_client_priv_data *iter_client; void *data; bool ctx_changed = (fte->dests_size == 0); ns = get_ns_with_notifiers(&fte->base); if (!ns) return; down_read(&ns->notifiers_rw_sem); list_for_each_entry(iter_handler, &ns->list_notifiers, list) { data = NULL; mutex_lock(&dst->clients_lock); list_for_each_entry(iter_client, &dst->clients_data, list) { if (iter_client->fs_handler == iter_handler) { data = iter_client->client_dst_data; break; } } mutex_unlock(&dst->clients_lock); if (iter_handler->del_dst_cb) { iter_handler->del_dst_cb(dst, ctx_changed, data, iter_handler->client_context); } } up_read(&ns->notifiers_rw_sem); } /* fte should not be deleted while calling this function */ static struct mlx5_flow_rule *_fs_add_dst_fte(struct fs_fte *fte, struct mlx5_flow_group *fg, struct mlx5_flow_destination *dest) { struct mlx5_flow_table *ft; struct mlx5_flow_rule *dst; int err; dst = kzalloc(sizeof(*dst), GFP_KERNEL); if (!dst) return ERR_PTR(-ENOMEM); memcpy(&dst->dest_attr, dest, sizeof(*dest)); dst->base.type = FS_TYPE_FLOW_DEST; INIT_LIST_HEAD(&dst->clients_data); mutex_init(&dst->clients_lock); fs_get_parent(ft, fg); /*Add dest to dests list- added as first element after the head*/ list_add_tail(&dst->base.list, &fte->dests); fte->dests_size++; err = mlx5_cmd_fs_set_fte(fs_get_dev(&ft->base), ft->vport, &fte->status, fte->val, ft->type, ft->id, fte->index, fg->id, fte->flow_tag, fte->action, fte->dests_size, &fte->dests); if (err) goto free_dst; list_del(&dst->base.list); return dst; free_dst: list_del(&dst->base.list); kfree(dst); fte->dests_size--; return ERR_PTR(err); } static char *get_dest_name(struct mlx5_flow_destination *dest) { char *name = kzalloc(sizeof(char) * 20, GFP_KERNEL); switch (dest->type) { case MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE: snprintf(name, 20, "dest_%s_%u", "flow_table", dest->ft->id); return name; case MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT: snprintf(name, 20, "dest_%s_%u", "vport", dest->vport_num); return name; case MLX5_FLOW_CONTEXT_DEST_TYPE_TIR: snprintf(name, 20, "dest_%s_%u", "tir", dest->tir_num); return name; + default: + kfree(name); + return NULL; } - - return NULL; } /* assumed fg is locked */ static unsigned int fs_get_free_fg_index(struct mlx5_flow_group *fg, struct list_head **prev) { struct fs_fte *fte; unsigned int start = fg->start_index; if (prev) *prev = &fg->ftes; /* assumed list is sorted by index */ fs_for_each_fte(fte, fg) { if (fte->index != start) return start; start++; if (prev) *prev = &fte->base.list; } return start; } static struct fs_fte *fs_create_fte(struct mlx5_flow_group *fg, u32 *match_value, u8 action, u32 flow_tag, struct list_head **prev) { struct fs_fte *fte; int index = 0; index = fs_get_free_fg_index(fg, prev); fte = fs_alloc_fte(action, flow_tag, match_value, index); if (IS_ERR(fte)) return fte; return fte; } static void add_rule_to_tree(struct mlx5_flow_rule *rule, struct fs_fte *fte) { char *dest_name; dest_name = get_dest_name(&rule->dest_attr); fs_add_node(&rule->base, &fte->base, dest_name, 1); /* re-add to list, since fs_add_node reset our list */ list_add_tail(&rule->base.list, &fte->dests); kfree(dest_name); call_to_add_rule_notifiers(rule, fte); } static void fs_del_dst(struct mlx5_flow_rule *dst) { struct mlx5_flow_table *ft; struct mlx5_flow_group *fg; struct fs_fte *fte; u32 *match_value; struct mlx5_core_dev *dev = fs_get_dev(&dst->base); int match_len = MLX5_ST_SZ_BYTES(fte_match_param); int err; WARN_ON(!dev); match_value = mlx5_vzalloc(match_len); if (!match_value) { mlx5_core_warn(dev, "failed to allocate inbox\n"); return; } fs_get_parent(fte, dst); fs_get_parent(fg, fte); mutex_lock(&fg->base.lock); memcpy(match_value, fte->val, sizeof(fte->val)); /* ft can't be changed as fg is locked */ fs_get_parent(ft, fg); list_del(&dst->base.list); fte->dests_size--; if (fte->dests_size) { err = mlx5_cmd_fs_set_fte(dev, ft->vport, &fte->status, match_value, ft->type, ft->id, fte->index, fg->id, fte->flow_tag, fte->action, fte->dests_size, &fte->dests); if (err) { mlx5_core_warn(dev, "%s can't delete dst %s\n", __func__, dst->base.name); goto err; } } call_to_del_rule_notifiers(dst, fte); err: mutex_unlock(&fg->base.lock); kvfree(match_value); } static void fs_del_fte(struct fs_fte *fte) { struct mlx5_flow_table *ft; struct mlx5_flow_group *fg; int err; struct mlx5_core_dev *dev; fs_get_parent(fg, fte); fs_get_parent(ft, fg); dev = fs_get_dev(&ft->base); WARN_ON(!dev); err = mlx5_cmd_fs_delete_fte(dev, ft->vport, &fte->status, ft->type, ft->id, fte->index); if (err) mlx5_core_warn(dev, "flow steering can't delete fte %s\n", fte->base.name); fg->num_ftes--; } /* assuming parent fg is locked */ /* Add dst algorithm */ static struct mlx5_flow_rule *fs_add_dst_fg(struct mlx5_flow_group *fg, u32 *match_value, u8 action, u32 flow_tag, struct mlx5_flow_destination *dest) { struct fs_fte *fte; struct mlx5_flow_rule *dst; struct mlx5_flow_table *ft; struct list_head *prev; char fte_name[20]; mutex_lock(&fg->base.lock); fs_for_each_fte(fte, fg) { /* TODO: Check of size against PRM max size */ mutex_lock(&fte->base.lock); if (fs_match_exact_val(&fg->mask, match_value, &fte->val) && action == fte->action && flow_tag == fte->flow_tag) { dst = _fs_add_dst_fte(fte, fg, dest); mutex_unlock(&fte->base.lock); if (IS_ERR(dst)) goto unlock_fg; goto add_rule; } mutex_unlock(&fte->base.lock); } fs_get_parent(ft, fg); if (fg->num_ftes == fg->max_ftes) { dst = ERR_PTR(-ENOSPC); goto unlock_fg; } fte = fs_create_fte(fg, match_value, action, flow_tag, &prev); if (IS_ERR(fte)) { dst = (void *)fte; goto unlock_fg; } dst = _fs_add_dst_fte(fte, fg, dest); if (IS_ERR(dst)) { kfree(fte); goto unlock_fg; } fg->num_ftes++; snprintf(fte_name, sizeof(fte_name), "fte%u", fte->index); /* Add node to tree */ fs_add_node(&fte->base, &fg->base, fte_name, 0); list_add(&fte->base.list, prev); add_rule: add_rule_to_tree(dst, fte); unlock_fg: mutex_unlock(&fg->base.lock); return dst; } static struct mlx5_flow_rule *fs_add_dst_ft(struct mlx5_flow_table *ft, u8 match_criteria_enable, u32 *match_criteria, u32 *match_value, u8 action, u32 flow_tag, struct mlx5_flow_destination *dest) { /*? where dst_entry is allocated*/ struct mlx5_flow_group *g; struct mlx5_flow_rule *dst; fs_get(&ft->base); mutex_lock(&ft->base.lock); fs_for_each_fg(g, ft) if (fs_match_exact_mask(g->mask.match_criteria_enable, match_criteria_enable, g->mask.match_criteria, match_criteria)) { mutex_unlock(&ft->base.lock); dst = fs_add_dst_fg(g, match_value, action, flow_tag, dest); if (PTR_ERR(dst) && PTR_ERR(dst) != -ENOSPC) goto unlock; } mutex_unlock(&ft->base.lock); g = create_autogroup(ft, match_criteria_enable, match_criteria); if (IS_ERR(g)) { dst = (void *)g; goto unlock; } dst = fs_add_dst_fg(g, match_value, action, flow_tag, dest); if (IS_ERR(dst)) { /* Remove assumes refcount > 0 and autogroup creates a group * with a refcount = 0. */ fs_get(&g->base); fs_remove_node(&g->base); goto unlock; } unlock: fs_put(&ft->base); return dst; } struct mlx5_flow_rule * mlx5_add_flow_rule(struct mlx5_flow_table *ft, u8 match_criteria_enable, u32 *match_criteria, u32 *match_value, u32 action, u32 flow_tag, struct mlx5_flow_destination *dest) { struct mlx5_flow_rule *dst; struct mlx5_flow_namespace *ns; ns = get_ns_with_notifiers(&ft->base); if (ns) down_read(&ns->dests_rw_sem); dst = fs_add_dst_ft(ft, match_criteria_enable, match_criteria, match_value, action, flow_tag, dest); if (ns) up_read(&ns->dests_rw_sem); return dst; } EXPORT_SYMBOL(mlx5_add_flow_rule); void mlx5_del_flow_rule(struct mlx5_flow_rule *dst) { struct mlx5_flow_namespace *ns; ns = get_ns_with_notifiers(&dst->base); if (ns) down_read(&ns->dests_rw_sem); fs_remove_node(&dst->base); if (ns) up_read(&ns->dests_rw_sem); } EXPORT_SYMBOL(mlx5_del_flow_rule); #define MLX5_CORE_FS_ROOT_NS_NAME "root" #define MLX5_CORE_FS_ESW_EGRESS_ACL "esw_egress_root" #define MLX5_CORE_FS_ESW_INGRESS_ACL "esw_ingress_root" #define MLX5_CORE_FS_FDB_ROOT_NS_NAME "fdb_root" #define MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME "sniffer_rx_root" #define MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME "sniffer_tx_root" #define MLX5_CORE_FS_PRIO_MAX_FT 4 #define MLX5_CORE_FS_PRIO_MAX_NS 1 static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns, unsigned prio, int max_ft, const char *name, u8 flags) { struct fs_prio *fs_prio; fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL); if (!fs_prio) return ERR_PTR(-ENOMEM); fs_prio->base.type = FS_TYPE_PRIO; fs_add_node(&fs_prio->base, &ns->base, name, 1); fs_prio->max_ft = max_ft; fs_prio->max_ns = MLX5_CORE_FS_PRIO_MAX_NS; fs_prio->prio = prio; fs_prio->flags = flags; list_add_tail(&fs_prio->base.list, &ns->prios); INIT_LIST_HEAD(&fs_prio->objs); mutex_init(&fs_prio->shared_lock); return fs_prio; } static void cleanup_root_ns(struct mlx5_core_dev *dev) { struct mlx5_flow_root_namespace *root_ns = dev->root_ns; struct fs_prio *iter_prio; if (!root_ns) return; /* stage 1 */ fs_for_each_prio(iter_prio, &root_ns->ns) { struct mlx5_flow_namespace *iter_ns; fs_for_each_ns(iter_ns, iter_prio) { while (!list_empty(&iter_ns->prios)) { struct fs_base *iter_prio2 = list_first_entry(&iter_ns->prios, struct fs_base, list); fs_remove_node(iter_prio2); } } } /* stage 2 */ fs_for_each_prio(iter_prio, &root_ns->ns) { while (!list_empty(&iter_prio->objs)) { struct fs_base *iter_ns = list_first_entry(&iter_prio->objs, struct fs_base, list); fs_remove_node(iter_ns); } } /* stage 3 */ while (!list_empty(&root_ns->ns.prios)) { struct fs_base *iter_prio = list_first_entry(&root_ns->ns.prios, struct fs_base, list); fs_remove_node(iter_prio); } fs_remove_node(&root_ns->ns.base); dev->root_ns = NULL; } static void cleanup_single_prio_root_ns(struct mlx5_core_dev *dev, struct mlx5_flow_root_namespace *root_ns) { struct fs_base *prio; if (!root_ns) return; if (!list_empty(&root_ns->ns.prios)) { prio = list_first_entry(&root_ns->ns.prios, struct fs_base, list); fs_remove_node(prio); } fs_remove_node(&root_ns->ns.base); root_ns = NULL; } void mlx5_cleanup_fs(struct mlx5_core_dev *dev) { cleanup_root_ns(dev); cleanup_single_prio_root_ns(dev, dev->sniffer_rx_root_ns); cleanup_single_prio_root_ns(dev, dev->sniffer_tx_root_ns); cleanup_single_prio_root_ns(dev, dev->fdb_root_ns); cleanup_single_prio_root_ns(dev, dev->esw_egress_root_ns); cleanup_single_prio_root_ns(dev, dev->esw_ingress_root_ns); } static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace *ns) { ns->base.type = FS_TYPE_NAMESPACE; init_rwsem(&ns->dests_rw_sem); init_rwsem(&ns->notifiers_rw_sem); INIT_LIST_HEAD(&ns->prios); INIT_LIST_HEAD(&ns->list_notifiers); return ns; } static struct mlx5_flow_root_namespace *create_root_ns(struct mlx5_core_dev *dev, enum fs_ft_type table_type, char *name) { struct mlx5_flow_root_namespace *root_ns; struct mlx5_flow_namespace *ns; /* create the root namespace */ root_ns = mlx5_vzalloc(sizeof(*root_ns)); if (!root_ns) goto err; root_ns->dev = dev; root_ns->table_type = table_type; mutex_init(&root_ns->fs_chain_lock); ns = &root_ns->ns; fs_init_namespace(ns); fs_add_node(&ns->base, NULL, name, 1); return root_ns; err: return NULL; } static int init_fdb_root_ns(struct mlx5_core_dev *dev) { struct fs_prio *prio; dev->fdb_root_ns = create_root_ns(dev, FS_FT_FDB, MLX5_CORE_FS_FDB_ROOT_NS_NAME); if (!dev->fdb_root_ns) return -ENOMEM; /* create 1 prio*/ prio = fs_create_prio(&dev->fdb_root_ns->ns, 0, 1, "fdb_prio", 0); if (IS_ERR(prio)) return PTR_ERR(prio); else return 0; } #define MAX_VPORTS 128 static int init_egress_acl_root_ns(struct mlx5_core_dev *dev) { struct fs_prio *prio; dev->esw_egress_root_ns = create_root_ns(dev, FS_FT_ESW_EGRESS_ACL, MLX5_CORE_FS_ESW_EGRESS_ACL); if (!dev->esw_egress_root_ns) return -ENOMEM; /* create 1 prio*/ prio = fs_create_prio(&dev->esw_egress_root_ns->ns, 0, MAX_VPORTS, "esw_egress_prio", 0); if (IS_ERR(prio)) return PTR_ERR(prio); else return 0; } static int init_ingress_acl_root_ns(struct mlx5_core_dev *dev) { struct fs_prio *prio; dev->esw_ingress_root_ns = create_root_ns(dev, FS_FT_ESW_INGRESS_ACL, MLX5_CORE_FS_ESW_INGRESS_ACL); if (!dev->esw_ingress_root_ns) return -ENOMEM; /* create 1 prio*/ prio = fs_create_prio(&dev->esw_ingress_root_ns->ns, 0, MAX_VPORTS, "esw_ingress_prio", 0); if (IS_ERR(prio)) return PTR_ERR(prio); else return 0; } static int init_sniffer_rx_root_ns(struct mlx5_core_dev *dev) { struct fs_prio *prio; dev->sniffer_rx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_RX, MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME); if (!dev->sniffer_rx_root_ns) return -ENOMEM; /* create 1 prio*/ prio = fs_create_prio(&dev->sniffer_rx_root_ns->ns, 0, 1, "sniffer_prio", 0); if (IS_ERR(prio)) return PTR_ERR(prio); else return 0; } static int init_sniffer_tx_root_ns(struct mlx5_core_dev *dev) { struct fs_prio *prio; dev->sniffer_tx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_TX, MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME); if (!dev->sniffer_tx_root_ns) return -ENOMEM; /* create 1 prio*/ prio = fs_create_prio(&dev->sniffer_tx_root_ns->ns, 0, 1, "sniffer_prio", 0); if (IS_ERR(prio)) return PTR_ERR(prio); else return 0; } static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio, const char *name) { struct mlx5_flow_namespace *ns; ns = kzalloc(sizeof(*ns), GFP_KERNEL); if (!ns) return ERR_PTR(-ENOMEM); fs_init_namespace(ns); fs_add_node(&ns->base, &prio->base, name, 1); list_add_tail(&ns->base.list, &prio->objs); return ns; } #define FLOW_TABLE_BIT_SZ 1 #define GET_FLOW_TABLE_CAP(dev, offset) \ ((be32_to_cpu(*((__be32 *)(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) + \ offset / 32)) >> \ (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ) static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps) { int i; for (i = 0; i < caps->arr_sz; i++) { if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i])) return false; } return true; } static int _init_root_tree(struct mlx5_core_dev *dev, int max_ft_level, struct init_tree_node *node, struct fs_base *base_parent, struct init_tree_node *tree_parent) { struct mlx5_flow_namespace *fs_ns; struct fs_prio *fs_prio; int priority; struct fs_base *base; int i; int err = 0; if (node->type == FS_TYPE_PRIO) { if ((node->min_ft_level > max_ft_level) || !has_required_caps(dev, &node->caps)) goto out; fs_get_obj(fs_ns, base_parent); priority = node - tree_parent->children; fs_prio = fs_create_prio(fs_ns, priority, node->max_ft, node->name, node->flags); if (IS_ERR(fs_prio)) { err = PTR_ERR(fs_prio); goto out; } base = &fs_prio->base; } else if (node->type == FS_TYPE_NAMESPACE) { fs_get_obj(fs_prio, base_parent); fs_ns = fs_create_namespace(fs_prio, node->name); if (IS_ERR(fs_ns)) { err = PTR_ERR(fs_ns); goto out; } base = &fs_ns->base; } else { return -EINVAL; } for (i = 0; i < node->ar_size; i++) { err = _init_root_tree(dev, max_ft_level, &node->children[i], base, node); if (err) break; } out: return err; } static int init_root_tree(struct mlx5_core_dev *dev, int max_ft_level, struct init_tree_node *node, struct fs_base *parent) { int i; struct mlx5_flow_namespace *fs_ns; int err = 0; fs_get_obj(fs_ns, parent); for (i = 0; i < node->ar_size; i++) { err = _init_root_tree(dev, max_ft_level, &node->children[i], &fs_ns->base, node); if (err) break; } return err; } static int sum_max_ft_in_prio(struct fs_prio *prio); static int sum_max_ft_in_ns(struct mlx5_flow_namespace *ns) { struct fs_prio *prio; int sum = 0; fs_for_each_prio(prio, ns) { sum += sum_max_ft_in_prio(prio); } return sum; } static int sum_max_ft_in_prio(struct fs_prio *prio) { int sum = 0; struct fs_base *it; struct mlx5_flow_namespace *ns; if (prio->max_ft) return prio->max_ft; fs_for_each_ns_or_ft(it, prio) { if (it->type == FS_TYPE_FLOW_TABLE) continue; fs_get_obj(ns, it); sum += sum_max_ft_in_ns(ns); } prio->max_ft = sum; return sum; } static void set_max_ft(struct mlx5_flow_namespace *ns) { struct fs_prio *prio; if (!ns) return; fs_for_each_prio(prio, ns) sum_max_ft_in_prio(prio); } static int init_root_ns(struct mlx5_core_dev *dev) { int max_ft_level = MLX5_CAP_FLOWTABLE(dev, flow_table_properties_nic_receive. max_ft_level); dev->root_ns = create_root_ns(dev, FS_FT_NIC_RX, MLX5_CORE_FS_ROOT_NS_NAME); if (IS_ERR_OR_NULL(dev->root_ns)) goto err; if (init_root_tree(dev, max_ft_level, &root_fs, &dev->root_ns->ns.base)) goto err; set_max_ft(&dev->root_ns->ns); return 0; err: return -ENOMEM; } u8 mlx5_get_match_criteria_enable(struct mlx5_flow_rule *rule) { struct fs_base *pbase; struct mlx5_flow_group *fg; pbase = rule->base.parent; WARN_ON(!pbase); pbase = pbase->parent; WARN_ON(!pbase); fs_get_obj(fg, pbase); return fg->mask.match_criteria_enable; } void mlx5_get_match_value(u32 *match_value, struct mlx5_flow_rule *rule) { struct fs_base *pbase; struct fs_fte *fte; pbase = rule->base.parent; WARN_ON(!pbase); fs_get_obj(fte, pbase); memcpy(match_value, fte->val, sizeof(fte->val)); } void mlx5_get_match_criteria(u32 *match_criteria, struct mlx5_flow_rule *rule) { struct fs_base *pbase; struct mlx5_flow_group *fg; pbase = rule->base.parent; WARN_ON(!pbase); pbase = pbase->parent; WARN_ON(!pbase); fs_get_obj(fg, pbase); memcpy(match_criteria, &fg->mask.match_criteria, sizeof(fg->mask.match_criteria)); } int mlx5_init_fs(struct mlx5_core_dev *dev) { int err; if (MLX5_CAP_GEN(dev, nic_flow_table)) { err = init_root_ns(dev); if (err) goto err; } err = init_fdb_root_ns(dev); if (err) goto err; err = init_egress_acl_root_ns(dev); if (err) goto err; err = init_ingress_acl_root_ns(dev); if (err) goto err; err = init_sniffer_tx_root_ns(dev); if (err) goto err; err = init_sniffer_rx_root_ns(dev); if (err) goto err; return 0; err: mlx5_cleanup_fs(dev); return err; } struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type) { struct mlx5_flow_root_namespace *root_ns = dev->root_ns; int prio; static struct fs_prio *fs_prio; struct mlx5_flow_namespace *ns; switch (type) { case MLX5_FLOW_NAMESPACE_BYPASS: prio = 0; break; case MLX5_FLOW_NAMESPACE_KERNEL: prio = 1; break; case MLX5_FLOW_NAMESPACE_LEFTOVERS: prio = 2; break; case MLX5_FLOW_NAMESPACE_FDB: if (dev->fdb_root_ns) return &dev->fdb_root_ns->ns; else return NULL; case MLX5_FLOW_NAMESPACE_ESW_EGRESS: if (dev->esw_egress_root_ns) return &dev->esw_egress_root_ns->ns; else return NULL; case MLX5_FLOW_NAMESPACE_ESW_INGRESS: if (dev->esw_ingress_root_ns) return &dev->esw_ingress_root_ns->ns; else return NULL; case MLX5_FLOW_NAMESPACE_SNIFFER_RX: if (dev->sniffer_rx_root_ns) return &dev->sniffer_rx_root_ns->ns; else return NULL; case MLX5_FLOW_NAMESPACE_SNIFFER_TX: if (dev->sniffer_tx_root_ns) return &dev->sniffer_tx_root_ns->ns; else return NULL; default: return NULL; } if (!root_ns) return NULL; fs_prio = find_prio(&root_ns->ns, prio); if (!fs_prio) return NULL; ns = list_first_entry(&fs_prio->objs, typeof(*ns), base.list); return ns; } EXPORT_SYMBOL(mlx5_get_flow_namespace); int mlx5_set_rule_private_data(struct mlx5_flow_rule *rule, struct mlx5_flow_handler *fs_handler, void *client_data) { struct fs_client_priv_data *priv_data; mutex_lock(&rule->clients_lock); /*Check that hanlder isn't exists in the list already*/ list_for_each_entry(priv_data, &rule->clients_data, list) { if (priv_data->fs_handler == fs_handler) { priv_data->client_dst_data = client_data; goto unlock; } } priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL); if (!priv_data) { mutex_unlock(&rule->clients_lock); return -ENOMEM; } priv_data->client_dst_data = client_data; priv_data->fs_handler = fs_handler; list_add(&priv_data->list, &rule->clients_data); unlock: mutex_unlock(&rule->clients_lock); return 0; } static int remove_from_clients(struct mlx5_flow_rule *rule, bool ctx_changed, void *client_data, void *context) { struct fs_client_priv_data *iter_client; struct fs_client_priv_data *temp_client; struct mlx5_flow_handler *handler = (struct mlx5_flow_handler*)context; mutex_lock(&rule->clients_lock); list_for_each_entry_safe(iter_client, temp_client, &rule->clients_data, list) { if (iter_client->fs_handler == handler) { list_del(&iter_client->list); kfree(iter_client); break; } } mutex_unlock(&rule->clients_lock); return 0; } struct mlx5_flow_handler *mlx5_register_rule_notifier(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type ns_type, rule_event_fn add_cb, rule_event_fn del_cb, void *context) { struct mlx5_flow_namespace *ns; struct mlx5_flow_handler *handler; ns = mlx5_get_flow_namespace(dev, ns_type); if (!ns) return ERR_PTR(-EINVAL); handler = kzalloc(sizeof(*handler), GFP_KERNEL); if (!handler) return ERR_PTR(-ENOMEM); handler->add_dst_cb = add_cb; handler->del_dst_cb = del_cb; handler->client_context = context; handler->ns = ns; down_write(&ns->notifiers_rw_sem); list_add_tail(&handler->list, &ns->list_notifiers); up_write(&ns->notifiers_rw_sem); return handler; } static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns, rule_event_fn add_rule_cb, void *context); void mlx5_unregister_rule_notifier(struct mlx5_flow_handler *handler) { struct mlx5_flow_namespace *ns = handler->ns; /*Remove from dst's clients*/ down_write(&ns->dests_rw_sem); down_write(&ns->notifiers_rw_sem); iterate_rules_in_ns(ns, remove_from_clients, handler); list_del(&handler->list); up_write(&ns->notifiers_rw_sem); up_write(&ns->dests_rw_sem); kfree(handler); } static void iterate_rules_in_ft(struct mlx5_flow_table *ft, rule_event_fn add_rule_cb, void *context) { struct mlx5_flow_group *iter_fg; struct fs_fte *iter_fte; struct mlx5_flow_rule *iter_rule; int err = 0; bool is_new_rule; mutex_lock(&ft->base.lock); fs_for_each_fg(iter_fg, ft) { mutex_lock(&iter_fg->base.lock); fs_for_each_fte(iter_fte, iter_fg) { mutex_lock(&iter_fte->base.lock); is_new_rule = true; fs_for_each_dst(iter_rule, iter_fte) { fs_get(&iter_rule->base); err = add_rule_cb(iter_rule, is_new_rule, NULL, context); fs_put_parent_locked(&iter_rule->base); if (err) break; is_new_rule = false; } mutex_unlock(&iter_fte->base.lock); if (err) break; } mutex_unlock(&iter_fg->base.lock); if (err) break; } mutex_unlock(&ft->base.lock); } static void iterate_rules_in_prio(struct fs_prio *prio, rule_event_fn add_rule_cb, void *context) { struct fs_base *it; mutex_lock(&prio->base.lock); fs_for_each_ns_or_ft(it, prio) { if (it->type == FS_TYPE_FLOW_TABLE) { struct mlx5_flow_table *ft; fs_get_obj(ft, it); iterate_rules_in_ft(ft, add_rule_cb, context); } else { struct mlx5_flow_namespace *ns; fs_get_obj(ns, it); iterate_rules_in_ns(ns, add_rule_cb, context); } } mutex_unlock(&prio->base.lock); } static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns, rule_event_fn add_rule_cb, void *context) { struct fs_prio *iter_prio; mutex_lock(&ns->base.lock); fs_for_each_prio(iter_prio, ns) { iterate_rules_in_prio(iter_prio, add_rule_cb, context); } mutex_unlock(&ns->base.lock); } void mlx5_flow_iterate_existing_rules(struct mlx5_flow_namespace *ns, rule_event_fn add_rule_cb, void *context) { down_write(&ns->dests_rw_sem); down_read(&ns->notifiers_rw_sem); iterate_rules_in_ns(ns, add_rule_cb, context); up_read(&ns->notifiers_rw_sem); up_write(&ns->dests_rw_sem); } void mlx5_del_flow_rules_list(struct mlx5_flow_rules_list *rules_list) { struct mlx5_flow_rule_node *iter_node; struct mlx5_flow_rule_node *temp_node; list_for_each_entry_safe(iter_node, temp_node, &rules_list->head, list) { list_del(&iter_node->list); kfree(iter_node); } kfree(rules_list); } #define ROCEV1_ETHERTYPE 0x8915 static int set_rocev1_rules(struct list_head *rules_list) { struct mlx5_flow_rule_node *rocev1_rule; rocev1_rule = kzalloc(sizeof(*rocev1_rule), GFP_KERNEL); if (!rocev1_rule) return -ENOMEM; rocev1_rule->match_criteria_enable = 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_criteria, ethertype, 0xffff); MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_value, ethertype, ROCEV1_ETHERTYPE); list_add_tail(&rocev1_rule->list, rules_list); return 0; } #define ROCEV2_UDP_PORT 4791 static int set_rocev2_rules(struct list_head *rules_list) { struct mlx5_flow_rule_node *ipv4_rule; struct mlx5_flow_rule_node *ipv6_rule; ipv4_rule = kzalloc(sizeof(*ipv4_rule), GFP_KERNEL); if (!ipv4_rule) return -ENOMEM; ipv6_rule = kzalloc(sizeof(*ipv6_rule), GFP_KERNEL); if (!ipv6_rule) { kfree(ipv4_rule); return -ENOMEM; } ipv4_rule->match_criteria_enable = 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ethertype, 0xffff); MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ethertype, 0x0800); MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ip_protocol, 0xff); MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ip_protocol, IPPROTO_UDP); MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, udp_dport, 0xffff); MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, udp_dport, ROCEV2_UDP_PORT); ipv6_rule->match_criteria_enable = 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS; MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ethertype, 0xffff); MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ethertype, 0x86dd); MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ip_protocol, 0xff); MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ip_protocol, IPPROTO_UDP); MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, udp_dport, 0xffff); MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, udp_dport, ROCEV2_UDP_PORT); list_add_tail(&ipv4_rule->list, rules_list); list_add_tail(&ipv6_rule->list, rules_list); return 0; } struct mlx5_flow_rules_list *get_roce_flow_rules(u8 roce_mode) { int err = 0; struct mlx5_flow_rules_list *rules_list = kzalloc(sizeof(*rules_list), GFP_KERNEL); if (!rules_list) return NULL; INIT_LIST_HEAD(&rules_list->head); if (roce_mode & MLX5_ROCE_VERSION_1_CAP) { err = set_rocev1_rules(&rules_list->head); if (err) goto free_list; } if (roce_mode & MLX5_ROCE_VERSION_2_CAP) err = set_rocev2_rules(&rules_list->head); if (err) goto free_list; return rules_list; free_list: mlx5_del_flow_rules_list(rules_list); return NULL; } Index: head/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c =================================================================== --- head/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c (revision 341555) +++ head/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c (revision 341556) @@ -1,609 +1,610 @@ /*- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include #include "mlx5_core.h" CTASSERT((uintptr_t)PAGE_MASK > (uintptr_t)PAGE_SIZE); struct mlx5_pages_req { struct mlx5_core_dev *dev; u16 func_id; s32 npages; struct work_struct work; }; enum { MAX_RECLAIM_TIME_MSECS = 5000, }; static void mlx5_fwp_load_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { struct mlx5_fw_page *fwp; uint8_t owned; fwp = (struct mlx5_fw_page *)arg; owned = MLX5_DMA_OWNED(fwp->dev); if (!owned) MLX5_DMA_LOCK(fwp->dev); if (error == 0) { KASSERT(nseg == 1, ("Number of segments is different from 1")); fwp->dma_addr = segs->ds_addr; fwp->load_done = MLX5_LOAD_ST_SUCCESS; } else { fwp->load_done = MLX5_LOAD_ST_FAILURE; } MLX5_DMA_DONE(fwp->dev); if (!owned) MLX5_DMA_UNLOCK(fwp->dev); } void mlx5_fwp_flush(struct mlx5_fw_page *fwp) { unsigned num = fwp->numpages; while (num--) bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_PREWRITE); } void mlx5_fwp_invalidate(struct mlx5_fw_page *fwp) { unsigned num = fwp->numpages; while (num--) { bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_POSTREAD); bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_PREREAD); } } struct mlx5_fw_page * mlx5_fwp_alloc(struct mlx5_core_dev *dev, gfp_t flags, unsigned num) { struct mlx5_fw_page *fwp; unsigned x; int err; /* check for special case */ if (num == 0) { fwp = kzalloc(sizeof(*fwp), flags); if (fwp != NULL) fwp->dev = dev; return (fwp); } /* we need sleeping context for this function */ if (flags & M_NOWAIT) return (NULL); fwp = kzalloc(sizeof(*fwp) * num, flags); /* serialize loading the DMA map(s) */ sx_xlock(&dev->cmd.dma_sx); for (x = 0; x != num; x++) { /* store pointer to MLX5 core device */ fwp[x].dev = dev; /* store number of pages left from the array */ fwp[x].numpages = num - x; /* allocate memory */ err = bus_dmamem_alloc(dev->cmd.dma_tag, &fwp[x].virt_addr, BUS_DMA_WAITOK | BUS_DMA_COHERENT, &fwp[x].dma_map); if (err != 0) goto failure; /* load memory into DMA */ MLX5_DMA_LOCK(dev); (void) bus_dmamap_load( dev->cmd.dma_tag, fwp[x].dma_map, fwp[x].virt_addr, MLX5_ADAPTER_PAGE_SIZE, &mlx5_fwp_load_mem_cb, fwp + x, BUS_DMA_WAITOK | BUS_DMA_COHERENT); while (fwp[x].load_done == MLX5_LOAD_ST_NONE) MLX5_DMA_WAIT(dev); MLX5_DMA_UNLOCK(dev); /* check for error */ if (fwp[x].load_done != MLX5_LOAD_ST_SUCCESS) { bus_dmamem_free(dev->cmd.dma_tag, fwp[x].virt_addr, fwp[x].dma_map); goto failure; } } sx_xunlock(&dev->cmd.dma_sx); return (fwp); failure: while (x--) { bus_dmamap_unload(dev->cmd.dma_tag, fwp[x].dma_map); bus_dmamem_free(dev->cmd.dma_tag, fwp[x].virt_addr, fwp[x].dma_map); } sx_xunlock(&dev->cmd.dma_sx); + kfree(fwp); return (NULL); } void mlx5_fwp_free(struct mlx5_fw_page *fwp) { struct mlx5_core_dev *dev; unsigned num; /* be NULL safe */ if (fwp == NULL) return; /* check for special case */ if (fwp->numpages == 0) { kfree(fwp); return; } num = fwp->numpages; dev = fwp->dev; while (num--) { bus_dmamap_unload(dev->cmd.dma_tag, fwp[num].dma_map); bus_dmamem_free(dev->cmd.dma_tag, fwp[num].virt_addr, fwp[num].dma_map); } kfree(fwp); } u64 mlx5_fwp_get_dma(struct mlx5_fw_page *fwp, size_t offset) { size_t index = (offset / MLX5_ADAPTER_PAGE_SIZE); KASSERT(index < fwp->numpages, ("Invalid offset: %lld", (long long)offset)); return ((fwp + index)->dma_addr + (offset % MLX5_ADAPTER_PAGE_SIZE)); } void * mlx5_fwp_get_virt(struct mlx5_fw_page *fwp, size_t offset) { size_t index = (offset / MLX5_ADAPTER_PAGE_SIZE); KASSERT(index < fwp->numpages, ("Invalid offset: %lld", (long long)offset)); return ((char *)(fwp + index)->virt_addr + (offset % MLX5_ADAPTER_PAGE_SIZE)); } static int mlx5_insert_fw_page_locked(struct mlx5_core_dev *dev, struct mlx5_fw_page *nfp) { struct rb_root *root = &dev->priv.page_root; struct rb_node **new = &root->rb_node; struct rb_node *parent = NULL; struct mlx5_fw_page *tfp; while (*new) { parent = *new; tfp = rb_entry(parent, struct mlx5_fw_page, rb_node); if (tfp->dma_addr < nfp->dma_addr) new = &parent->rb_left; else if (tfp->dma_addr > nfp->dma_addr) new = &parent->rb_right; else return (-EEXIST); } rb_link_node(&nfp->rb_node, parent, new); rb_insert_color(&nfp->rb_node, root); return (0); } static struct mlx5_fw_page * mlx5_remove_fw_page_locked(struct mlx5_core_dev *dev, bus_addr_t addr) { struct rb_root *root = &dev->priv.page_root; struct rb_node *tmp = root->rb_node; struct mlx5_fw_page *result = NULL; struct mlx5_fw_page *tfp; while (tmp) { tfp = rb_entry(tmp, struct mlx5_fw_page, rb_node); if (tfp->dma_addr < addr) { tmp = tmp->rb_left; } else if (tfp->dma_addr > addr) { tmp = tmp->rb_right; } else { rb_erase(&tfp->rb_node, &dev->priv.page_root); result = tfp; break; } } return (result); } static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr, u16 func_id) { struct mlx5_fw_page *fwp; int err; fwp = mlx5_fwp_alloc(dev, GFP_KERNEL, 1); if (fwp == NULL) return (-ENOMEM); fwp->func_id = func_id; MLX5_DMA_LOCK(dev); err = mlx5_insert_fw_page_locked(dev, fwp); MLX5_DMA_UNLOCK(dev); if (err != 0) { mlx5_fwp_free(fwp); } else { /* make sure cached data is cleaned */ mlx5_fwp_invalidate(fwp); /* store DMA address */ *addr = fwp->dma_addr; } return (err); } static void free_4k(struct mlx5_core_dev *dev, u64 addr) { struct mlx5_fw_page *fwp; MLX5_DMA_LOCK(dev); fwp = mlx5_remove_fw_page_locked(dev, addr); MLX5_DMA_UNLOCK(dev); if (fwp == NULL) { mlx5_core_warn(dev, "Cannot free 4K page at 0x%llx\n", (long long)addr); return; } mlx5_fwp_free(fwp); } static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, s32 *npages, int boot) { u32 in[MLX5_ST_SZ_DW(query_pages_in)] = {0}; u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {0}; int err; MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES); MLX5_SET(query_pages_in, in, op_mod, boot ? MLX5_QUERY_PAGES_IN_OP_MOD_BOOT_PAGES : MLX5_QUERY_PAGES_IN_OP_MOD_INIT_PAGES); err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); if (err) return err; *npages = MLX5_GET(query_pages_out, out, num_pages); *func_id = MLX5_GET(query_pages_out, out, function_id); return 0; } static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, int notify_fail) { u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0}; int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); u64 addr; int err; u32 *in, *nin; int i = 0; inlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_in, pas[0]); in = mlx5_vzalloc(inlen); if (!in) { mlx5_core_warn(dev, "vzalloc failed %d\n", inlen); err = -ENOMEM; goto out_alloc; } for (i = 0; i < npages; i++) { err = alloc_4k(dev, &addr, func_id); if (err) goto out_alloc; MLX5_ARRAY_SET64(manage_pages_in, in, pas, i, addr); } MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_GIVE); MLX5_SET(manage_pages_in, in, function_id, func_id); MLX5_SET(manage_pages_in, in, input_num_entries, npages); err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); if (err) { mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", func_id, npages, err); goto out_alloc; } dev->priv.fw_pages += npages; dev->priv.pages_per_func[func_id] += npages; mlx5_core_dbg(dev, "err %d\n", err); goto out_free; out_alloc: if (notify_fail) { nin = mlx5_vzalloc(inlen); if (!nin) goto out_4k; memset(&out, 0, sizeof(out)); MLX5_SET(manage_pages_in, nin, opcode, MLX5_CMD_OP_MANAGE_PAGES); MLX5_SET(manage_pages_in, nin, op_mod, MLX5_PAGES_CANT_GIVE); MLX5_SET(manage_pages_in, nin, function_id, func_id); if (mlx5_cmd_exec(dev, nin, inlen, out, sizeof(out))) mlx5_core_warn(dev, "page notify failed\n"); kvfree(nin); } out_4k: for (i--; i >= 0; i--) free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i])); out_free: kvfree(in); return err; } static int reclaim_pages_cmd(struct mlx5_core_dev *dev, u32 *in, int in_size, u32 *out, int out_size) { struct mlx5_fw_page *fwp; struct rb_node *p; u32 func_id; u32 npages; u32 i = 0; if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) return mlx5_cmd_exec(dev, in, in_size, out, out_size); /* No hard feelings, we want our pages back! */ npages = MLX5_GET(manage_pages_in, in, input_num_entries); func_id = MLX5_GET(manage_pages_in, in, function_id); p = rb_first(&dev->priv.page_root); while (p && i < npages) { fwp = rb_entry(p, struct mlx5_fw_page, rb_node); p = rb_next(p); if (fwp->func_id != func_id) continue; MLX5_ARRAY_SET64(manage_pages_out, out, pas, i, fwp->dma_addr); i++; } MLX5_SET(manage_pages_out, out, output_num_entries, i); return 0; } static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, int *nclaimed) { int outlen = MLX5_ST_SZ_BYTES(manage_pages_out); u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {0}; int num_claimed; u32 *out; int err; int i; if (nclaimed) *nclaimed = 0; outlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); out = mlx5_vzalloc(outlen); if (!out) return -ENOMEM; MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_TAKE); MLX5_SET(manage_pages_in, in, function_id, func_id); MLX5_SET(manage_pages_in, in, input_num_entries, npages); mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen); err = reclaim_pages_cmd(dev, in, sizeof(in), out, outlen); if (err) { mlx5_core_err(dev, "failed reclaiming pages\n"); goto out_free; } num_claimed = MLX5_GET(manage_pages_out, out, output_num_entries); if (nclaimed) *nclaimed = num_claimed; dev->priv.fw_pages -= num_claimed; dev->priv.pages_per_func[func_id] -= num_claimed; for (i = 0; i < num_claimed; i++) free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i])); out_free: kvfree(out); return err; } static void pages_work_handler(struct work_struct *work) { struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work); struct mlx5_core_dev *dev = req->dev; int err = 0; if (req->npages < 0) err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL); else if (req->npages > 0) err = give_pages(dev, req->func_id, req->npages, 1); if (err) mlx5_core_warn(dev, "%s fail %d\n", req->npages < 0 ? "reclaim" : "give", err); kfree(req); } void mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id, s32 npages) { struct mlx5_pages_req *req; req = kzalloc(sizeof(*req), GFP_ATOMIC); if (!req) { mlx5_core_warn(dev, "failed to allocate pages request\n"); return; } req->dev = dev; req->func_id = func_id; req->npages = npages; INIT_WORK(&req->work, pages_work_handler); if (!queue_work(dev->priv.pg_wq, &req->work)) mlx5_core_warn(dev, "failed to queue pages handler work\n"); } int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) { u16 uninitialized_var(func_id); s32 uninitialized_var(npages); int err; err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot); if (err) return err; mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n", npages, boot ? "boot" : "init", func_id); return give_pages(dev, func_id, npages, 0); } enum { MLX5_BLKS_FOR_RECLAIM_PAGES = 12 }; s64 mlx5_wait_for_reclaim_vfs_pages(struct mlx5_core_dev *dev) { int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); s64 prevpages = 0; s64 npages = 0; while (!time_after(jiffies, end)) { /* exclude own function, VFs only */ npages = dev->priv.fw_pages - dev->priv.pages_per_func[0]; if (!npages) break; if (npages != prevpages) end = end + msecs_to_jiffies(100); prevpages = npages; msleep(1); } if (npages) mlx5_core_warn(dev, "FW did not return all VFs pages, will cause to memory leak\n"); return -npages; } static int optimal_reclaimed_pages(void) { struct mlx5_cmd_prot_block *block; struct mlx5_cmd_layout *lay; int ret; ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) - MLX5_ST_SZ_BYTES(manage_pages_out)) / MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); return ret; } int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) { int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); struct mlx5_fw_page *fwp; struct rb_node *p; int nclaimed = 0; int err; do { p = rb_first(&dev->priv.page_root); if (p) { fwp = rb_entry(p, struct mlx5_fw_page, rb_node); err = reclaim_pages(dev, fwp->func_id, optimal_reclaimed_pages(), &nclaimed); if (err) { mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", err); return err; } if (nclaimed) end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); } if (time_after(jiffies, end)) { mlx5_core_warn(dev, "FW did not return all pages. giving up...\n"); break; } } while (p); return 0; } void mlx5_pagealloc_init(struct mlx5_core_dev *dev) { dev->priv.page_root = RB_ROOT; } void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) { /* nothing */ } int mlx5_pagealloc_start(struct mlx5_core_dev *dev) { dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator"); if (!dev->priv.pg_wq) return -ENOMEM; return 0; } void mlx5_pagealloc_stop(struct mlx5_core_dev *dev) { destroy_workqueue(dev->priv.pg_wq); } Index: head/sys/dev/mlx5/mlx5_core/mlx5_vport.c =================================================================== --- head/sys/dev/mlx5/mlx5_core/mlx5_vport.c (revision 341555) +++ head/sys/dev/mlx5/mlx5_core/mlx5_vport.c (revision 341556) @@ -1,1790 +1,1790 @@ /*- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include "mlx5_core.h" static int mlx5_modify_nic_vport_context(struct mlx5_core_dev *mdev, void *in, int inlen); static int _mlx5_query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport, u32 *out, int outlen) { int err; u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {0}; MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE); MLX5_SET(query_vport_state_in, in, op_mod, opmod); MLX5_SET(query_vport_state_in, in, vport_number, vport); if (vport) MLX5_SET(query_vport_state_in, in, other_vport, 1); err = mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen); if (err) mlx5_core_warn(mdev, "MLX5_CMD_OP_QUERY_VPORT_STATE failed\n"); return err; } u8 mlx5_query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport) { u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {0}; _mlx5_query_vport_state(mdev, opmod, vport, out, sizeof(out)); return MLX5_GET(query_vport_state_out, out, state); } EXPORT_SYMBOL_GPL(mlx5_query_vport_state); u8 mlx5_query_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport) { u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {0}; _mlx5_query_vport_state(mdev, opmod, vport, out, sizeof(out)); return MLX5_GET(query_vport_state_out, out, admin_state); } EXPORT_SYMBOL(mlx5_query_vport_admin_state); int mlx5_modify_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport, u8 state) { u32 in[MLX5_ST_SZ_DW(modify_vport_state_in)] = {0}; u32 out[MLX5_ST_SZ_DW(modify_vport_state_out)] = {0}; int err; MLX5_SET(modify_vport_state_in, in, opcode, MLX5_CMD_OP_MODIFY_VPORT_STATE); MLX5_SET(modify_vport_state_in, in, op_mod, opmod); MLX5_SET(modify_vport_state_in, in, vport_number, vport); if (vport) MLX5_SET(modify_vport_state_in, in, other_vport, 1); MLX5_SET(modify_vport_state_in, in, admin_state, state); err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); if (err) mlx5_core_warn(mdev, "MLX5_CMD_OP_MODIFY_VPORT_STATE failed\n"); return err; } EXPORT_SYMBOL(mlx5_modify_vport_admin_state); static int mlx5_query_nic_vport_context(struct mlx5_core_dev *mdev, u16 vport, u32 *out, int outlen) { u32 in[MLX5_ST_SZ_DW(query_nic_vport_context_in)] = {0}; MLX5_SET(query_nic_vport_context_in, in, opcode, MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT); MLX5_SET(query_nic_vport_context_in, in, vport_number, vport); if (vport) MLX5_SET(query_nic_vport_context_in, in, other_vport, 1); return mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen); } static u32 mlx5_vport_max_q_counter_allocator(struct mlx5_core_dev *mdev, int client_id) { switch (client_id) { case MLX5_INTERFACE_PROTOCOL_IB: return (MLX5_CAP_GEN(mdev, max_qp_cnt) - MLX5_QCOUNTER_SETS_NETDEV); case MLX5_INTERFACE_PROTOCOL_ETH: return MLX5_QCOUNTER_SETS_NETDEV; default: mlx5_core_warn(mdev, "Unknown Client: %d\n", client_id); return 0; } } int mlx5_vport_alloc_q_counter(struct mlx5_core_dev *mdev, int client_id, u16 *counter_set_id) { u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {0}; u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {0}; int err; if (mdev->num_q_counter_allocated[client_id] > mlx5_vport_max_q_counter_allocator(mdev, client_id)) return -EINVAL; MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); if (!err) *counter_set_id = MLX5_GET(alloc_q_counter_out, out, counter_set_id); mdev->num_q_counter_allocated[client_id]++; return err; } int mlx5_vport_dealloc_q_counter(struct mlx5_core_dev *mdev, int client_id, u16 counter_set_id) { u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {0}; u32 out[MLX5_ST_SZ_DW(dealloc_q_counter_out)] = {0}; int err; if (mdev->num_q_counter_allocated[client_id] <= 0) return -EINVAL; MLX5_SET(dealloc_q_counter_in, in, opcode, MLX5_CMD_OP_DEALLOC_Q_COUNTER); MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter_set_id); err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); mdev->num_q_counter_allocated[client_id]--; return err; } int mlx5_vport_query_q_counter(struct mlx5_core_dev *mdev, u16 counter_set_id, int reset, void *out, int out_size) { u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {0}; MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER); MLX5_SET(query_q_counter_in, in, clear, reset); MLX5_SET(query_q_counter_in, in, counter_set_id, counter_set_id); return mlx5_cmd_exec(mdev, in, sizeof(in), out, out_size); } int mlx5_vport_query_out_of_rx_buffer(struct mlx5_core_dev *mdev, u16 counter_set_id, u32 *out_of_rx_buffer) { u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {0}; int err; err = mlx5_vport_query_q_counter(mdev, counter_set_id, 0, out, sizeof(out)); if (err) return err; *out_of_rx_buffer = MLX5_GET(query_q_counter_out, out, out_of_buffer); return err; } int mlx5_query_nic_vport_min_inline(struct mlx5_core_dev *mdev, u16 vport, u8 *min_inline) { u32 out[MLX5_ST_SZ_DW(query_nic_vport_context_out)] = {0}; int err; err = mlx5_query_nic_vport_context(mdev, vport, out, sizeof(out)); if (!err) *min_inline = MLX5_GET(query_nic_vport_context_out, out, nic_vport_context.min_wqe_inline_mode); return err; } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_min_inline); void mlx5_query_min_inline(struct mlx5_core_dev *mdev, u8 *min_inline_mode) { switch (MLX5_CAP_ETH(mdev, wqe_inline_mode)) { case MLX5_CAP_INLINE_MODE_L2: *min_inline_mode = MLX5_INLINE_MODE_L2; break; case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: mlx5_query_nic_vport_min_inline(mdev, 0, min_inline_mode); break; case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: *min_inline_mode = MLX5_INLINE_MODE_NONE; break; } } EXPORT_SYMBOL_GPL(mlx5_query_min_inline); int mlx5_modify_nic_vport_min_inline(struct mlx5_core_dev *mdev, u16 vport, u8 min_inline) { u32 in[MLX5_ST_SZ_DW(modify_nic_vport_context_in)] = {0}; int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); void *nic_vport_ctx; MLX5_SET(modify_nic_vport_context_in, in, field_select.min_wqe_inline_mode, 1); MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport); MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1); nic_vport_ctx = MLX5_ADDR_OF(modify_nic_vport_context_in, in, nic_vport_context); MLX5_SET(nic_vport_context, nic_vport_ctx, min_wqe_inline_mode, min_inline); return mlx5_modify_nic_vport_context(mdev, in, inlen); } EXPORT_SYMBOL_GPL(mlx5_modify_nic_vport_min_inline); int mlx5_query_nic_vport_mac_address(struct mlx5_core_dev *mdev, u16 vport, u8 *addr) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); u8 *out_addr; int err; out = mlx5_vzalloc(outlen); if (!out) return -ENOMEM; out_addr = MLX5_ADDR_OF(query_nic_vport_context_out, out, nic_vport_context.permanent_address); err = mlx5_query_nic_vport_context(mdev, vport, out, outlen); if (err) goto out; ether_addr_copy(addr, &out_addr[2]); out: kvfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_mac_address); int mlx5_modify_nic_vport_mac_address(struct mlx5_core_dev *mdev, u16 vport, u8 *addr) { void *in; int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); int err; void *nic_vport_ctx; u8 *perm_mac; in = mlx5_vzalloc(inlen); if (!in) { mlx5_core_warn(mdev, "failed to allocate inbox\n"); return -ENOMEM; } MLX5_SET(modify_nic_vport_context_in, in, field_select.permanent_address, 1); MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport); if (vport) MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1); nic_vport_ctx = MLX5_ADDR_OF(modify_nic_vport_context_in, in, nic_vport_context); perm_mac = MLX5_ADDR_OF(nic_vport_context, nic_vport_ctx, permanent_address); ether_addr_copy(&perm_mac[2], addr); err = mlx5_modify_nic_vport_context(mdev, in, inlen); kvfree(in); return err; } EXPORT_SYMBOL(mlx5_modify_nic_vport_mac_address); int mlx5_query_nic_vport_system_image_guid(struct mlx5_core_dev *mdev, u64 *system_image_guid) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); int err; out = mlx5_vzalloc(outlen); if (!out) return -ENOMEM; err = mlx5_query_nic_vport_context(mdev, 0, out, outlen); if (err) goto out; *system_image_guid = MLX5_GET64(query_nic_vport_context_out, out, nic_vport_context.system_image_guid); out: kvfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_system_image_guid); int mlx5_query_nic_vport_node_guid(struct mlx5_core_dev *mdev, u64 *node_guid) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); int err; out = mlx5_vzalloc(outlen); if (!out) return -ENOMEM; err = mlx5_query_nic_vport_context(mdev, 0, out, outlen); if (err) goto out; *node_guid = MLX5_GET64(query_nic_vport_context_out, out, nic_vport_context.node_guid); out: kvfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_node_guid); static int mlx5_query_nic_vport_port_guid(struct mlx5_core_dev *mdev, u64 *port_guid) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); int err; out = mlx5_vzalloc(outlen); if (!out) return -ENOMEM; err = mlx5_query_nic_vport_context(mdev, 0, out, outlen); if (err) goto out; *port_guid = MLX5_GET64(query_nic_vport_context_out, out, nic_vport_context.port_guid); out: kvfree(out); return err; } int mlx5_query_nic_vport_qkey_viol_cntr(struct mlx5_core_dev *mdev, u16 *qkey_viol_cntr) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); int err; out = mlx5_vzalloc(outlen); if (!out) return -ENOMEM; err = mlx5_query_nic_vport_context(mdev, 0, out, outlen); if (err) goto out; *qkey_viol_cntr = MLX5_GET(query_nic_vport_context_out, out, nic_vport_context.qkey_violation_counter); out: kvfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_qkey_viol_cntr); static int mlx5_modify_nic_vport_context(struct mlx5_core_dev *mdev, void *in, int inlen) { u32 out[MLX5_ST_SZ_DW(modify_nic_vport_context_out)] = {0}; MLX5_SET(modify_nic_vport_context_in, in, opcode, MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT); return mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); } static int mlx5_nic_vport_enable_disable_roce(struct mlx5_core_dev *mdev, int enable_disable) { void *in; int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); int err; in = mlx5_vzalloc(inlen); if (!in) { mlx5_core_warn(mdev, "failed to allocate inbox\n"); return -ENOMEM; } MLX5_SET(modify_nic_vport_context_in, in, field_select.roce_en, 1); MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.roce_en, enable_disable); err = mlx5_modify_nic_vport_context(mdev, in, inlen); kvfree(in); return err; } int mlx5_set_nic_vport_current_mac(struct mlx5_core_dev *mdev, int vport, bool other_vport, u8 *addr) { void *in; int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in) + MLX5_ST_SZ_BYTES(mac_address_layout); u8 *mac_layout; u8 *mac_ptr; int err; in = mlx5_vzalloc(inlen); if (!in) { mlx5_core_warn(mdev, "failed to allocate inbox\n"); return -ENOMEM; } MLX5_SET(modify_nic_vport_context_in, in, opcode, MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT); MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport); MLX5_SET(modify_nic_vport_context_in, in, other_vport, other_vport); MLX5_SET(modify_nic_vport_context_in, in, field_select.addresses_list, 1); MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.allowed_list_type, MLX5_NIC_VPORT_LIST_TYPE_UC); MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.allowed_list_size, 1); mac_layout = (u8 *)MLX5_ADDR_OF(modify_nic_vport_context_in, in, nic_vport_context.current_uc_mac_address); mac_ptr = (u8 *)MLX5_ADDR_OF(mac_address_layout, mac_layout, mac_addr_47_32); ether_addr_copy(mac_ptr, addr); err = mlx5_modify_nic_vport_context(mdev, in, inlen); kvfree(in); return err; } EXPORT_SYMBOL_GPL(mlx5_set_nic_vport_current_mac); int mlx5_modify_nic_vport_node_guid(struct mlx5_core_dev *mdev, u32 vport, u64 node_guid) { void *in; int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); int err; void *nic_vport_context; if (!vport) return -EINVAL; if (!MLX5_CAP_GEN(mdev, vport_group_manager)) return -EPERM; if (!MLX5_CAP_ESW(mdev, nic_vport_node_guid_modify)) return -ENOTSUPP; in = mlx5_vzalloc(inlen); if (!in) { mlx5_core_warn(mdev, "failed to allocate inbox\n"); return -ENOMEM; } MLX5_SET(modify_nic_vport_context_in, in, field_select.node_guid, 1); MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport); MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1); nic_vport_context = MLX5_ADDR_OF(modify_nic_vport_context_in, in, nic_vport_context); MLX5_SET64(nic_vport_context, nic_vport_context, node_guid, node_guid); err = mlx5_modify_nic_vport_context(mdev, in, inlen); kvfree(in); return err; } EXPORT_SYMBOL(mlx5_modify_nic_vport_node_guid); int mlx5_modify_nic_vport_port_guid(struct mlx5_core_dev *mdev, u32 vport, u64 port_guid) { void *in; int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); int err; void *nic_vport_context; if (!vport) return -EINVAL; if (!MLX5_CAP_GEN(mdev, vport_group_manager)) return -EPERM; if (!MLX5_CAP_ESW(mdev, nic_vport_port_guid_modify)) return -ENOTSUPP; in = mlx5_vzalloc(inlen); if (!in) { mlx5_core_warn(mdev, "failed to allocate inbox\n"); return -ENOMEM; } MLX5_SET(modify_nic_vport_context_in, in, field_select.port_guid, 1); MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport); MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1); nic_vport_context = MLX5_ADDR_OF(modify_nic_vport_context_in, in, nic_vport_context); MLX5_SET64(nic_vport_context, nic_vport_context, port_guid, port_guid); err = mlx5_modify_nic_vport_context(mdev, in, inlen); kvfree(in); return err; } EXPORT_SYMBOL(mlx5_modify_nic_vport_port_guid); int mlx5_set_nic_vport_vlan_list(struct mlx5_core_dev *dev, u16 vport, u16 *vlan_list, int list_len) { void *in, *ctx; int i, err; int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in) + MLX5_ST_SZ_BYTES(vlan_layout) * (int)list_len; int max_list_size = 1 << MLX5_CAP_GEN_MAX(dev, log_max_vlan_list); if (list_len > max_list_size) { mlx5_core_warn(dev, "Requested list size (%d) > (%d) max_list_size\n", list_len, max_list_size); return -ENOSPC; } in = mlx5_vzalloc(inlen); if (!in) { mlx5_core_warn(dev, "failed to allocate inbox\n"); return -ENOMEM; } MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport); if (vport) MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1); MLX5_SET(modify_nic_vport_context_in, in, field_select.addresses_list, 1); ctx = MLX5_ADDR_OF(modify_nic_vport_context_in, in, nic_vport_context); MLX5_SET(nic_vport_context, ctx, allowed_list_type, MLX5_NIC_VPORT_LIST_TYPE_VLAN); MLX5_SET(nic_vport_context, ctx, allowed_list_size, list_len); for (i = 0; i < list_len; i++) { u8 *vlan_lout = MLX5_ADDR_OF(nic_vport_context, ctx, current_uc_mac_address[i]); MLX5_SET(vlan_layout, vlan_lout, vlan, vlan_list[i]); } err = mlx5_modify_nic_vport_context(dev, in, inlen); kvfree(in); return err; } EXPORT_SYMBOL_GPL(mlx5_set_nic_vport_vlan_list); int mlx5_set_nic_vport_mc_list(struct mlx5_core_dev *mdev, int vport, u64 *addr_list, size_t addr_list_len) { void *in, *ctx; int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in) + MLX5_ST_SZ_BYTES(mac_address_layout) * (int)addr_list_len; int err; size_t i; int max_list_sz = 1 << MLX5_CAP_GEN_MAX(mdev, log_max_current_mc_list); if ((int)addr_list_len > max_list_sz) { mlx5_core_warn(mdev, "Requested list size (%d) > (%d) max_list_size\n", (int)addr_list_len, max_list_sz); return -ENOSPC; } in = mlx5_vzalloc(inlen); if (!in) { mlx5_core_warn(mdev, "failed to allocate inbox\n"); return -ENOMEM; } MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport); if (vport) MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1); MLX5_SET(modify_nic_vport_context_in, in, field_select.addresses_list, 1); ctx = MLX5_ADDR_OF(modify_nic_vport_context_in, in, nic_vport_context); MLX5_SET(nic_vport_context, ctx, allowed_list_type, MLX5_NIC_VPORT_LIST_TYPE_MC); MLX5_SET(nic_vport_context, ctx, allowed_list_size, addr_list_len); for (i = 0; i < addr_list_len; i++) { u8 *mac_lout = (u8 *)MLX5_ADDR_OF(nic_vport_context, ctx, current_uc_mac_address[i]); u8 *mac_ptr = (u8 *)MLX5_ADDR_OF(mac_address_layout, mac_lout, mac_addr_47_32); ether_addr_copy(mac_ptr, (u8 *)&addr_list[i]); } err = mlx5_modify_nic_vport_context(mdev, in, inlen); kvfree(in); return err; } EXPORT_SYMBOL_GPL(mlx5_set_nic_vport_mc_list); int mlx5_set_nic_vport_promisc(struct mlx5_core_dev *mdev, int vport, bool promisc_mc, bool promisc_uc, bool promisc_all) { u8 in[MLX5_ST_SZ_BYTES(modify_nic_vport_context_in)]; u8 *ctx = MLX5_ADDR_OF(modify_nic_vport_context_in, in, nic_vport_context); memset(in, 0, MLX5_ST_SZ_BYTES(modify_nic_vport_context_in)); MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport); if (vport) MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1); MLX5_SET(modify_nic_vport_context_in, in, field_select.promisc, 1); if (promisc_mc) MLX5_SET(nic_vport_context, ctx, promisc_mc, 1); if (promisc_uc) MLX5_SET(nic_vport_context, ctx, promisc_uc, 1); if (promisc_all) MLX5_SET(nic_vport_context, ctx, promisc_all, 1); return mlx5_modify_nic_vport_context(mdev, in, sizeof(in)); } EXPORT_SYMBOL_GPL(mlx5_set_nic_vport_promisc); int mlx5_query_nic_vport_mac_list(struct mlx5_core_dev *dev, u16 vport, enum mlx5_list_type list_type, u8 addr_list[][ETH_ALEN], int *list_size) { u32 in[MLX5_ST_SZ_DW(query_nic_vport_context_in)] = {0}; void *nic_vport_ctx; int max_list_size; int req_list_size; int out_sz; void *out; int err; int i; req_list_size = *list_size; max_list_size = (list_type == MLX5_NIC_VPORT_LIST_TYPE_UC) ? 1 << MLX5_CAP_GEN_MAX(dev, log_max_current_uc_list) : 1 << MLX5_CAP_GEN_MAX(dev, log_max_current_mc_list); if (req_list_size > max_list_size) { mlx5_core_warn(dev, "Requested list size (%d) > (%d) max_list_size\n", req_list_size, max_list_size); req_list_size = max_list_size; } out_sz = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in) + req_list_size * MLX5_ST_SZ_BYTES(mac_address_layout); out = kzalloc(out_sz, GFP_KERNEL); if (!out) return -ENOMEM; MLX5_SET(query_nic_vport_context_in, in, opcode, MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT); MLX5_SET(query_nic_vport_context_in, in, allowed_list_type, list_type); MLX5_SET(query_nic_vport_context_in, in, vport_number, vport); if (vport) MLX5_SET(query_nic_vport_context_in, in, other_vport, 1); err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); if (err) goto out; nic_vport_ctx = MLX5_ADDR_OF(query_nic_vport_context_out, out, nic_vport_context); req_list_size = MLX5_GET(nic_vport_context, nic_vport_ctx, allowed_list_size); *list_size = req_list_size; for (i = 0; i < req_list_size; i++) { u8 *mac_addr = MLX5_ADDR_OF(nic_vport_context, nic_vport_ctx, current_uc_mac_address[i]) + 2; ether_addr_copy(addr_list[i], mac_addr); } out: kfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_mac_list); int mlx5_modify_nic_vport_mac_list(struct mlx5_core_dev *dev, enum mlx5_list_type list_type, u8 addr_list[][ETH_ALEN], int list_size) { u32 out[MLX5_ST_SZ_DW(modify_nic_vport_context_out)] = {0}; void *nic_vport_ctx; int max_list_size; int in_sz; void *in; int err; int i; max_list_size = list_type == MLX5_NIC_VPORT_LIST_TYPE_UC ? 1 << MLX5_CAP_GEN(dev, log_max_current_uc_list) : 1 << MLX5_CAP_GEN(dev, log_max_current_mc_list); if (list_size > max_list_size) return -ENOSPC; in_sz = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in) + list_size * MLX5_ST_SZ_BYTES(mac_address_layout); in = kzalloc(in_sz, GFP_KERNEL); if (!in) return -ENOMEM; MLX5_SET(modify_nic_vport_context_in, in, opcode, MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT); MLX5_SET(modify_nic_vport_context_in, in, field_select.addresses_list, 1); nic_vport_ctx = MLX5_ADDR_OF(modify_nic_vport_context_in, in, nic_vport_context); MLX5_SET(nic_vport_context, nic_vport_ctx, allowed_list_type, list_type); MLX5_SET(nic_vport_context, nic_vport_ctx, allowed_list_size, list_size); for (i = 0; i < list_size; i++) { u8 *curr_mac = MLX5_ADDR_OF(nic_vport_context, nic_vport_ctx, current_uc_mac_address[i]) + 2; ether_addr_copy(curr_mac, addr_list[i]); } err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out)); kfree(in); return err; } EXPORT_SYMBOL_GPL(mlx5_modify_nic_vport_mac_list); int mlx5_query_nic_vport_vlans(struct mlx5_core_dev *dev, u16 vport, u16 vlans[], int *size) { u32 in[MLX5_ST_SZ_DW(query_nic_vport_context_in)] = {0}; void *nic_vport_ctx; int req_list_size; int max_list_size; int out_sz; void *out; int err; int i; req_list_size = *size; max_list_size = 1 << MLX5_CAP_GEN(dev, log_max_vlan_list); if (req_list_size > max_list_size) { mlx5_core_warn(dev, "Requested list size (%d) > (%d) max list size\n", req_list_size, max_list_size); req_list_size = max_list_size; } out_sz = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in) + req_list_size * MLX5_ST_SZ_BYTES(vlan_layout); out = kzalloc(out_sz, GFP_KERNEL); if (!out) return -ENOMEM; MLX5_SET(query_nic_vport_context_in, in, opcode, MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT); MLX5_SET(query_nic_vport_context_in, in, allowed_list_type, MLX5_NIC_VPORT_CONTEXT_ALLOWED_LIST_TYPE_VLAN_LIST); MLX5_SET(query_nic_vport_context_in, in, vport_number, vport); if (vport) MLX5_SET(query_nic_vport_context_in, in, other_vport, 1); err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); if (err) goto out; nic_vport_ctx = MLX5_ADDR_OF(query_nic_vport_context_out, out, nic_vport_context); req_list_size = MLX5_GET(nic_vport_context, nic_vport_ctx, allowed_list_size); *size = req_list_size; for (i = 0; i < req_list_size; i++) { void *vlan_addr = MLX5_ADDR_OF(nic_vport_context, nic_vport_ctx, current_uc_mac_address[i]); vlans[i] = MLX5_GET(vlan_layout, vlan_addr, vlan); } out: kfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_vlans); int mlx5_modify_nic_vport_vlans(struct mlx5_core_dev *dev, u16 vlans[], int list_size) { u32 out[MLX5_ST_SZ_DW(modify_nic_vport_context_out)] = {0}; void *nic_vport_ctx; int max_list_size; int in_sz; void *in; int err; int i; max_list_size = 1 << MLX5_CAP_GEN(dev, log_max_vlan_list); if (list_size > max_list_size) return -ENOSPC; in_sz = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in) + list_size * MLX5_ST_SZ_BYTES(vlan_layout); in = kzalloc(in_sz, GFP_KERNEL); if (!in) return -ENOMEM; MLX5_SET(modify_nic_vport_context_in, in, opcode, MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT); MLX5_SET(modify_nic_vport_context_in, in, field_select.addresses_list, 1); nic_vport_ctx = MLX5_ADDR_OF(modify_nic_vport_context_in, in, nic_vport_context); MLX5_SET(nic_vport_context, nic_vport_ctx, allowed_list_type, MLX5_NIC_VPORT_LIST_TYPE_VLAN); MLX5_SET(nic_vport_context, nic_vport_ctx, allowed_list_size, list_size); for (i = 0; i < list_size; i++) { void *vlan_addr = MLX5_ADDR_OF(nic_vport_context, nic_vport_ctx, current_uc_mac_address[i]); MLX5_SET(vlan_layout, vlan_addr, vlan, vlans[i]); } err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out)); kfree(in); return err; } EXPORT_SYMBOL_GPL(mlx5_modify_nic_vport_vlans); int mlx5_query_nic_vport_roce_en(struct mlx5_core_dev *mdev, u8 *enable) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); int err; out = kzalloc(outlen, GFP_KERNEL); if (!out) return -ENOMEM; err = mlx5_query_nic_vport_context(mdev, 0, out, outlen); if (err) goto out; *enable = MLX5_GET(query_nic_vport_context_out, out, nic_vport_context.roce_en); out: kfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_roce_en); int mlx5_set_nic_vport_permanent_mac(struct mlx5_core_dev *mdev, int vport, u8 *addr) { void *in; int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); u8 *mac_ptr; int err; in = mlx5_vzalloc(inlen); if (!in) { mlx5_core_warn(mdev, "failed to allocate inbox\n"); return -ENOMEM; } MLX5_SET(modify_nic_vport_context_in, in, opcode, MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT); MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport); MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1); MLX5_SET(modify_nic_vport_context_in, in, field_select.permanent_address, 1); mac_ptr = (u8 *)MLX5_ADDR_OF(modify_nic_vport_context_in, in, nic_vport_context.permanent_address.mac_addr_47_32); ether_addr_copy(mac_ptr, addr); err = mlx5_modify_nic_vport_context(mdev, in, inlen); kvfree(in); return err; } EXPORT_SYMBOL_GPL(mlx5_set_nic_vport_permanent_mac); int mlx5_nic_vport_enable_roce(struct mlx5_core_dev *mdev) { return mlx5_nic_vport_enable_disable_roce(mdev, 1); } EXPORT_SYMBOL_GPL(mlx5_nic_vport_enable_roce); int mlx5_nic_vport_disable_roce(struct mlx5_core_dev *mdev) { return mlx5_nic_vport_enable_disable_roce(mdev, 0); } EXPORT_SYMBOL_GPL(mlx5_nic_vport_disable_roce); int mlx5_core_query_vport_counter(struct mlx5_core_dev *dev, u8 other_vport, int vf, u8 port_num, void *out, size_t out_sz) { int in_sz = MLX5_ST_SZ_BYTES(query_vport_counter_in); int is_group_manager; void *in; int err; is_group_manager = MLX5_CAP_GEN(dev, vport_group_manager); in = mlx5_vzalloc(in_sz); if (!in) { err = -ENOMEM; return err; } MLX5_SET(query_vport_counter_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_COUNTER); if (other_vport) { if (is_group_manager) { MLX5_SET(query_vport_counter_in, in, other_vport, 1); MLX5_SET(query_vport_counter_in, in, vport_number, vf + 1); } else { err = -EPERM; goto free; } } if (MLX5_CAP_GEN(dev, num_ports) == 2) MLX5_SET(query_vport_counter_in, in, port_num, port_num); err = mlx5_cmd_exec(dev, in, in_sz, out, out_sz); free: kvfree(in); return err; } EXPORT_SYMBOL_GPL(mlx5_core_query_vport_counter); int mlx5_query_hca_vport_context(struct mlx5_core_dev *mdev, u8 port_num, u8 vport_num, u32 *out, int outlen) { u32 in[MLX5_ST_SZ_DW(query_hca_vport_context_in)] = {0}; int is_group_manager; is_group_manager = MLX5_CAP_GEN(mdev, vport_group_manager); MLX5_SET(query_hca_vport_context_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT); if (vport_num) { if (is_group_manager) { MLX5_SET(query_hca_vport_context_in, in, other_vport, 1); MLX5_SET(query_hca_vport_context_in, in, vport_number, vport_num); } else { return -EPERM; } } if (MLX5_CAP_GEN(mdev, num_ports) == 2) MLX5_SET(query_hca_vport_context_in, in, port_num, port_num); return mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen); } int mlx5_query_hca_vport_system_image_guid(struct mlx5_core_dev *mdev, u64 *system_image_guid) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_hca_vport_context_out); int err; out = mlx5_vzalloc(outlen); if (!out) return -ENOMEM; err = mlx5_query_hca_vport_context(mdev, 1, 0, out, outlen); if (err) goto out; *system_image_guid = MLX5_GET64(query_hca_vport_context_out, out, hca_vport_context.system_image_guid); out: kvfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_hca_vport_system_image_guid); int mlx5_query_hca_vport_node_guid(struct mlx5_core_dev *mdev, u64 *node_guid) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_hca_vport_context_out); int err; out = mlx5_vzalloc(outlen); if (!out) return -ENOMEM; err = mlx5_query_hca_vport_context(mdev, 1, 0, out, outlen); if (err) goto out; *node_guid = MLX5_GET64(query_hca_vport_context_out, out, hca_vport_context.node_guid); out: kvfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_hca_vport_node_guid); static int mlx5_query_hca_vport_port_guid(struct mlx5_core_dev *mdev, u64 *port_guid) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_hca_vport_context_out); int err; out = mlx5_vzalloc(outlen); if (!out) return -ENOMEM; err = mlx5_query_hca_vport_context(mdev, 1, 0, out, outlen); if (err) goto out; *port_guid = MLX5_GET64(query_hca_vport_context_out, out, hca_vport_context.port_guid); out: kvfree(out); return err; } int mlx5_query_hca_vport_gid(struct mlx5_core_dev *dev, u8 port_num, u16 vport_num, u16 gid_index, union ib_gid *gid) { int in_sz = MLX5_ST_SZ_BYTES(query_hca_vport_gid_in); int out_sz = MLX5_ST_SZ_BYTES(query_hca_vport_gid_out); int is_group_manager; void *out = NULL; void *in = NULL; union ib_gid *tmp; int tbsz; int nout; int err; is_group_manager = MLX5_CAP_GEN(dev, vport_group_manager); tbsz = mlx5_get_gid_table_len(MLX5_CAP_GEN(dev, gid_table_size)); if (gid_index > tbsz && gid_index != 0xffff) return -EINVAL; if (gid_index == 0xffff) nout = tbsz; else nout = 1; out_sz += nout * sizeof(*gid); in = mlx5_vzalloc(in_sz); out = mlx5_vzalloc(out_sz); if (!in || !out) { err = -ENOMEM; goto out; } MLX5_SET(query_hca_vport_gid_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_VPORT_GID); if (vport_num) { if (is_group_manager) { MLX5_SET(query_hca_vport_gid_in, in, vport_number, vport_num); MLX5_SET(query_hca_vport_gid_in, in, other_vport, 1); } else { err = -EPERM; goto out; } } MLX5_SET(query_hca_vport_gid_in, in, gid_index, gid_index); if (MLX5_CAP_GEN(dev, num_ports) == 2) MLX5_SET(query_hca_vport_gid_in, in, port_num, port_num); err = mlx5_cmd_exec(dev, in, in_sz, out, out_sz); if (err) goto out; tmp = (union ib_gid *)MLX5_ADDR_OF(query_hca_vport_gid_out, out, gid); gid->global.subnet_prefix = tmp->global.subnet_prefix; gid->global.interface_id = tmp->global.interface_id; out: kvfree(in); kvfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_hca_vport_gid); int mlx5_query_hca_vport_pkey(struct mlx5_core_dev *dev, u8 other_vport, u8 port_num, u16 vf_num, u16 pkey_index, u16 *pkey) { int in_sz = MLX5_ST_SZ_BYTES(query_hca_vport_pkey_in); int out_sz = MLX5_ST_SZ_BYTES(query_hca_vport_pkey_out); int is_group_manager; void *out = NULL; void *in = NULL; void *pkarr; int nout; int tbsz; int err; int i; is_group_manager = MLX5_CAP_GEN(dev, vport_group_manager); tbsz = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)); if (pkey_index > tbsz && pkey_index != 0xffff) return -EINVAL; if (pkey_index == 0xffff) nout = tbsz; else nout = 1; out_sz += nout * MLX5_ST_SZ_BYTES(pkey); in = kzalloc(in_sz, GFP_KERNEL); out = kzalloc(out_sz, GFP_KERNEL); MLX5_SET(query_hca_vport_pkey_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY); if (other_vport) { if (is_group_manager) { MLX5_SET(query_hca_vport_pkey_in, in, vport_number, vf_num); MLX5_SET(query_hca_vport_pkey_in, in, other_vport, 1); } else { err = -EPERM; goto out; } } MLX5_SET(query_hca_vport_pkey_in, in, pkey_index, pkey_index); if (MLX5_CAP_GEN(dev, num_ports) == 2) MLX5_SET(query_hca_vport_pkey_in, in, port_num, port_num); err = mlx5_cmd_exec(dev, in, in_sz, out, out_sz); if (err) goto out; pkarr = MLX5_ADDR_OF(query_hca_vport_pkey_out, out, pkey); for (i = 0; i < nout; i++, pkey++, pkarr += MLX5_ST_SZ_BYTES(pkey)) *pkey = MLX5_GET_PR(pkey, pkarr, pkey); out: kfree(in); kfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_hca_vport_pkey); static int mlx5_query_hca_min_wqe_header(struct mlx5_core_dev *mdev, int *min_header) { u32 *out; u32 outlen = MLX5_ST_SZ_BYTES(query_hca_vport_context_out); int err; out = mlx5_vzalloc(outlen); if (!out) return -ENOMEM; err = mlx5_query_hca_vport_context(mdev, 1, 0, out, outlen); if (err) goto out; *min_header = MLX5_GET(query_hca_vport_context_out, out, hca_vport_context.min_wqe_inline_mode); out: kvfree(out); return err; } static int mlx5_modify_eswitch_vport_context(struct mlx5_core_dev *mdev, u16 vport, void *in, int inlen) { u32 out[MLX5_ST_SZ_DW(modify_esw_vport_context_out)] = {0}; int err; MLX5_SET(modify_esw_vport_context_in, in, vport_number, vport); if (vport) MLX5_SET(modify_esw_vport_context_in, in, other_vport, 1); MLX5_SET(modify_esw_vport_context_in, in, opcode, MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT); err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); if (err) mlx5_core_warn(mdev, "MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT failed\n"); return err; } int mlx5_set_eswitch_cvlan_info(struct mlx5_core_dev *mdev, u8 vport, u8 insert_mode, u8 strip_mode, u16 vlan, u8 cfi, u8 pcp) { u32 in[MLX5_ST_SZ_DW(modify_esw_vport_context_in)]; memset(in, 0, sizeof(in)); if (insert_mode != MLX5_MODIFY_ESW_VPORT_CONTEXT_CVLAN_INSERT_NONE) { MLX5_SET(modify_esw_vport_context_in, in, esw_vport_context.cvlan_cfi, cfi); MLX5_SET(modify_esw_vport_context_in, in, esw_vport_context.cvlan_pcp, pcp); MLX5_SET(modify_esw_vport_context_in, in, esw_vport_context.cvlan_id, vlan); } MLX5_SET(modify_esw_vport_context_in, in, esw_vport_context.vport_cvlan_insert, insert_mode); MLX5_SET(modify_esw_vport_context_in, in, esw_vport_context.vport_cvlan_strip, strip_mode); MLX5_SET(modify_esw_vport_context_in, in, field_select, MLX5_MODIFY_ESW_VPORT_CONTEXT_FIELD_SELECT_CVLAN_STRIP | MLX5_MODIFY_ESW_VPORT_CONTEXT_FIELD_SELECT_CVLAN_INSERT); return mlx5_modify_eswitch_vport_context(mdev, vport, in, sizeof(in)); } EXPORT_SYMBOL_GPL(mlx5_set_eswitch_cvlan_info); int mlx5_query_vport_mtu(struct mlx5_core_dev *mdev, int *mtu) { u32 *out; u32 outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); int err; out = mlx5_vzalloc(outlen); if (!out) return -ENOMEM; err = mlx5_query_nic_vport_context(mdev, 0, out, outlen); if (err) goto out; *mtu = MLX5_GET(query_nic_vport_context_out, out, nic_vport_context.mtu); out: kvfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_vport_mtu); int mlx5_set_vport_mtu(struct mlx5_core_dev *mdev, int mtu) { u32 *in; u32 inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); int err; in = mlx5_vzalloc(inlen); if (!in) return -ENOMEM; MLX5_SET(modify_nic_vport_context_in, in, field_select.mtu, 1); MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.mtu, mtu); err = mlx5_modify_nic_vport_context(mdev, in, inlen); kvfree(in); return err; } EXPORT_SYMBOL_GPL(mlx5_set_vport_mtu); static int mlx5_query_vport_min_wqe_header(struct mlx5_core_dev *mdev, int *min_header) { u32 *out; u32 outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); int err; out = mlx5_vzalloc(outlen); if (!out) return -ENOMEM; err = mlx5_query_nic_vport_context(mdev, 0, out, outlen); if (err) goto out; *min_header = MLX5_GET(query_nic_vport_context_out, out, nic_vport_context.min_wqe_inline_mode); out: kvfree(out); return err; } int mlx5_set_vport_min_wqe_header(struct mlx5_core_dev *mdev, u8 vport, int min_header) { u32 *in; u32 inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); int err; in = mlx5_vzalloc(inlen); if (!in) return -ENOMEM; MLX5_SET(modify_nic_vport_context_in, in, field_select.min_wqe_inline_mode, 1); MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.min_wqe_inline_mode, min_header); MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport); MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1); err = mlx5_modify_nic_vport_context(mdev, in, inlen); kvfree(in); return err; } EXPORT_SYMBOL_GPL(mlx5_set_vport_min_wqe_header); int mlx5_query_min_wqe_header(struct mlx5_core_dev *dev, int *min_header) { switch (MLX5_CAP_GEN(dev, port_type)) { case MLX5_CMD_HCA_CAP_PORT_TYPE_IB: return mlx5_query_hca_min_wqe_header(dev, min_header); case MLX5_CMD_HCA_CAP_PORT_TYPE_ETHERNET: return mlx5_query_vport_min_wqe_header(dev, min_header); default: return -EINVAL; } } EXPORT_SYMBOL_GPL(mlx5_query_min_wqe_header); int mlx5_query_nic_vport_promisc(struct mlx5_core_dev *mdev, u16 vport, int *promisc_uc, int *promisc_mc, int *promisc_all) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); int err; out = kzalloc(outlen, GFP_KERNEL); if (!out) return -ENOMEM; err = mlx5_query_nic_vport_context(mdev, vport, out, outlen); if (err) goto out; *promisc_uc = MLX5_GET(query_nic_vport_context_out, out, nic_vport_context.promisc_uc); *promisc_mc = MLX5_GET(query_nic_vport_context_out, out, nic_vport_context.promisc_mc); *promisc_all = MLX5_GET(query_nic_vport_context_out, out, nic_vport_context.promisc_all); out: kfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_promisc); int mlx5_modify_nic_vport_promisc(struct mlx5_core_dev *mdev, int promisc_uc, int promisc_mc, int promisc_all) { void *in; int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); int err; in = mlx5_vzalloc(inlen); if (!in) { mlx5_core_err(mdev, "failed to allocate inbox\n"); return -ENOMEM; } MLX5_SET(modify_nic_vport_context_in, in, field_select.promisc, 1); MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.promisc_uc, promisc_uc); MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.promisc_mc, promisc_mc); MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.promisc_all, promisc_all); err = mlx5_modify_nic_vport_context(mdev, in, inlen); kvfree(in); return err; } EXPORT_SYMBOL_GPL(mlx5_modify_nic_vport_promisc); int mlx5_nic_vport_modify_local_lb(struct mlx5_core_dev *mdev, enum mlx5_local_lb_selection selection, u8 value) { void *in; int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); int err; in = mlx5_vzalloc(inlen); if (!in) { mlx5_core_warn(mdev, "failed to allocate inbox\n"); return -ENOMEM; } MLX5_SET(modify_nic_vport_context_in, in, vport_number, 0); if (selection == MLX5_LOCAL_MC_LB) { MLX5_SET(modify_nic_vport_context_in, in, field_select.disable_mc_local_lb, 1); MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.disable_mc_local_lb, value); } else { MLX5_SET(modify_nic_vport_context_in, in, field_select.disable_uc_local_lb, 1); MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.disable_uc_local_lb, value); } err = mlx5_modify_nic_vport_context(mdev, in, inlen); kvfree(in); return err; } EXPORT_SYMBOL_GPL(mlx5_nic_vport_modify_local_lb); int mlx5_nic_vport_query_local_lb(struct mlx5_core_dev *mdev, enum mlx5_local_lb_selection selection, u8 *value) { void *out; int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); int err; out = kzalloc(outlen, GFP_KERNEL); if (!out) return -ENOMEM; err = mlx5_query_nic_vport_context(mdev, 0, out, outlen); if (err) goto done; if (selection == MLX5_LOCAL_MC_LB) *value = MLX5_GET(query_nic_vport_context_out, out, nic_vport_context.disable_mc_local_lb); else *value = MLX5_GET(query_nic_vport_context_out, out, nic_vport_context.disable_uc_local_lb); done: kfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_nic_vport_query_local_lb); int mlx5_query_vport_counter(struct mlx5_core_dev *dev, u8 port_num, u16 vport_num, void *out, int out_size) { int in_sz = MLX5_ST_SZ_BYTES(query_vport_counter_in); int is_group_manager; void *in; int err; is_group_manager = MLX5_CAP_GEN(dev, vport_group_manager); in = mlx5_vzalloc(in_sz); if (!in) return -ENOMEM; MLX5_SET(query_vport_counter_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_COUNTER); if (vport_num) { if (is_group_manager) { MLX5_SET(query_vport_counter_in, in, other_vport, 1); MLX5_SET(query_vport_counter_in, in, vport_number, vport_num); } else { err = -EPERM; goto ex; } } if (MLX5_CAP_GEN(dev, num_ports) == 2) MLX5_SET(query_vport_counter_in, in, port_num, port_num); err = mlx5_cmd_exec(dev, in, in_sz, out, out_size); - kvfree(in); ex: + kvfree(in); return err; } EXPORT_SYMBOL_GPL(mlx5_query_vport_counter); int mlx5_get_vport_counters(struct mlx5_core_dev *dev, u8 port_num, struct mlx5_vport_counters *vc) { int out_sz = MLX5_ST_SZ_BYTES(query_vport_counter_out); void *out; int err; out = mlx5_vzalloc(out_sz); if (!out) return -ENOMEM; err = mlx5_query_vport_counter(dev, port_num, 0, out, out_sz); if (err) goto ex; vc->received_errors.packets = MLX5_GET64(query_vport_counter_out, out, received_errors.packets); vc->received_errors.octets = MLX5_GET64(query_vport_counter_out, out, received_errors.octets); vc->transmit_errors.packets = MLX5_GET64(query_vport_counter_out, out, transmit_errors.packets); vc->transmit_errors.octets = MLX5_GET64(query_vport_counter_out, out, transmit_errors.octets); vc->received_ib_unicast.packets = MLX5_GET64(query_vport_counter_out, out, received_ib_unicast.packets); vc->received_ib_unicast.octets = MLX5_GET64(query_vport_counter_out, out, received_ib_unicast.octets); vc->transmitted_ib_unicast.packets = MLX5_GET64(query_vport_counter_out, out, transmitted_ib_unicast.packets); vc->transmitted_ib_unicast.octets = MLX5_GET64(query_vport_counter_out, out, transmitted_ib_unicast.octets); vc->received_ib_multicast.packets = MLX5_GET64(query_vport_counter_out, out, received_ib_multicast.packets); vc->received_ib_multicast.octets = MLX5_GET64(query_vport_counter_out, out, received_ib_multicast.octets); vc->transmitted_ib_multicast.packets = MLX5_GET64(query_vport_counter_out, out, transmitted_ib_multicast.packets); vc->transmitted_ib_multicast.octets = MLX5_GET64(query_vport_counter_out, out, transmitted_ib_multicast.octets); vc->received_eth_broadcast.packets = MLX5_GET64(query_vport_counter_out, out, received_eth_broadcast.packets); vc->received_eth_broadcast.octets = MLX5_GET64(query_vport_counter_out, out, received_eth_broadcast.octets); vc->transmitted_eth_broadcast.packets = MLX5_GET64(query_vport_counter_out, out, transmitted_eth_broadcast.packets); vc->transmitted_eth_broadcast.octets = MLX5_GET64(query_vport_counter_out, out, transmitted_eth_broadcast.octets); vc->received_eth_unicast.octets = MLX5_GET64(query_vport_counter_out, out, received_eth_unicast.octets); vc->received_eth_unicast.packets = MLX5_GET64(query_vport_counter_out, out, received_eth_unicast.packets); vc->transmitted_eth_unicast.octets = MLX5_GET64(query_vport_counter_out, out, transmitted_eth_unicast.octets); vc->transmitted_eth_unicast.packets = MLX5_GET64(query_vport_counter_out, out, transmitted_eth_unicast.packets); vc->received_eth_multicast.octets = MLX5_GET64(query_vport_counter_out, out, received_eth_multicast.octets); vc->received_eth_multicast.packets = MLX5_GET64(query_vport_counter_out, out, received_eth_multicast.packets); vc->transmitted_eth_multicast.octets = MLX5_GET64(query_vport_counter_out, out, transmitted_eth_multicast.octets); vc->transmitted_eth_multicast.packets = MLX5_GET64(query_vport_counter_out, out, transmitted_eth_multicast.packets); ex: kvfree(out); return err; } int mlx5_query_vport_system_image_guid(struct mlx5_core_dev *dev, u64 *sys_image_guid) { switch (MLX5_CAP_GEN(dev, port_type)) { case MLX5_CMD_HCA_CAP_PORT_TYPE_IB: return mlx5_query_hca_vport_system_image_guid(dev, sys_image_guid); case MLX5_CMD_HCA_CAP_PORT_TYPE_ETHERNET: return mlx5_query_nic_vport_system_image_guid(dev, sys_image_guid); default: return -EINVAL; } } EXPORT_SYMBOL_GPL(mlx5_query_vport_system_image_guid); int mlx5_query_vport_node_guid(struct mlx5_core_dev *dev, u64 *node_guid) { switch (MLX5_CAP_GEN(dev, port_type)) { case MLX5_CMD_HCA_CAP_PORT_TYPE_IB: return mlx5_query_hca_vport_node_guid(dev, node_guid); case MLX5_CMD_HCA_CAP_PORT_TYPE_ETHERNET: return mlx5_query_nic_vport_node_guid(dev, node_guid); default: return -EINVAL; } } EXPORT_SYMBOL_GPL(mlx5_query_vport_node_guid); int mlx5_query_vport_port_guid(struct mlx5_core_dev *dev, u64 *port_guid) { switch (MLX5_CAP_GEN(dev, port_type)) { case MLX5_CMD_HCA_CAP_PORT_TYPE_IB: return mlx5_query_hca_vport_port_guid(dev, port_guid); case MLX5_CMD_HCA_CAP_PORT_TYPE_ETHERNET: return mlx5_query_nic_vport_port_guid(dev, port_guid); default: return -EINVAL; } } EXPORT_SYMBOL_GPL(mlx5_query_vport_port_guid); int mlx5_query_hca_vport_state(struct mlx5_core_dev *dev, u8 *vport_state) { u32 *out; int outlen = MLX5_ST_SZ_BYTES(query_hca_vport_context_out); int err; out = mlx5_vzalloc(outlen); if (!out) return -ENOMEM; err = mlx5_query_hca_vport_context(dev, 1, 0, out, outlen); if (err) goto out; *vport_state = MLX5_GET(query_hca_vport_context_out, out, hca_vport_context.vport_state); out: kvfree(out); return err; } EXPORT_SYMBOL_GPL(mlx5_query_hca_vport_state); int mlx5_core_query_ib_ppcnt(struct mlx5_core_dev *dev, u8 port_num, void *out, size_t sz) { u32 *in; int err; in = mlx5_vzalloc(sz); if (!in) { err = -ENOMEM; return err; } MLX5_SET(ppcnt_reg, in, local_port, port_num); MLX5_SET(ppcnt_reg, in, grp, MLX5_INFINIBAND_PORT_COUNTERS_GROUP); err = mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0); kvfree(in); return err; }