diff --git a/sys/dev/extres/clk/clk.c b/sys/dev/extres/clk/clk.c index b26158922d74..b1522eb1d4c2 100644 --- a/sys/dev/extres/clk/clk.c +++ b/sys/dev/extres/clk/clk.c @@ -1,1644 +1,1644 @@ /*- * Copyright 2016 Michal Meloun * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #endif #include SYSCTL_NODE(_hw, OID_AUTO, clock, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Clocks"); MALLOC_DEFINE(M_CLOCK, "clocks", "Clock framework"); /* Forward declarations. */ struct clk; struct clknodenode; struct clkdom; typedef TAILQ_HEAD(clknode_list, clknode) clknode_list_t; typedef TAILQ_HEAD(clkdom_list, clkdom) clkdom_list_t; /* Default clock methods. */ static int clknode_method_init(struct clknode *clk, device_t dev); static int clknode_method_recalc_freq(struct clknode *clk, uint64_t *freq); static int clknode_method_set_freq(struct clknode *clk, uint64_t fin, uint64_t *fout, int flags, int *stop); static int clknode_method_set_gate(struct clknode *clk, bool enable); static int clknode_method_set_mux(struct clknode *clk, int idx); /* * Clock controller methods. */ static clknode_method_t clknode_methods[] = { CLKNODEMETHOD(clknode_init, clknode_method_init), CLKNODEMETHOD(clknode_recalc_freq, clknode_method_recalc_freq), CLKNODEMETHOD(clknode_set_freq, clknode_method_set_freq), CLKNODEMETHOD(clknode_set_gate, clknode_method_set_gate), CLKNODEMETHOD(clknode_set_mux, clknode_method_set_mux), CLKNODEMETHOD_END }; DEFINE_CLASS_0(clknode, clknode_class, clknode_methods, 0); /* * Clock node - basic element for modeling SOC clock graph. It holds the clock * provider's data about the clock, and the links for the clock's membership in * various lists. */ struct clknode { KOBJ_FIELDS; /* Clock nodes topology. */ struct clkdom *clkdom; /* Owning clock domain */ TAILQ_ENTRY(clknode) clkdom_link; /* Domain list entry */ TAILQ_ENTRY(clknode) clklist_link; /* Global list entry */ /* String based parent list. */ const char **parent_names; /* Array of parent names */ int parent_cnt; /* Number of parents */ int parent_idx; /* Parent index or -1 */ /* Cache for already resolved names. */ struct clknode **parents; /* Array of potential parents */ struct clknode *parent; /* Current parent */ /* Parent/child relationship links. */ clknode_list_t children; /* List of our children */ TAILQ_ENTRY(clknode) sibling_link; /* Our entry in parent's list */ /* Details of this device. */ void *softc; /* Instance softc */ const char *name; /* Globally unique name */ intptr_t id; /* Per domain unique id */ int flags; /* CLK_FLAG_* */ struct sx lock; /* Lock for this clock */ int ref_cnt; /* Reference counter */ int enable_cnt; /* Enabled counter */ /* Cached values. */ uint64_t freq; /* Actual frequency */ struct sysctl_ctx_list sysctl_ctx; }; /* * Per consumer data, information about how a consumer is using a clock node. * A pointer to this structure is used as a handle in the consumer interface. */ struct clk { device_t dev; struct clknode *clknode; int enable_cnt; }; /* * Clock domain - a group of clocks provided by one clock device. */ struct clkdom { device_t dev; /* Link to provider device */ TAILQ_ENTRY(clkdom) link; /* Global domain list entry */ clknode_list_t clknode_list; /* All clocks in the domain */ #ifdef FDT clknode_ofw_mapper_func *ofw_mapper; /* Find clock using FDT xref */ #endif }; /* * The system-wide list of clock domains. */ static clkdom_list_t clkdom_list = TAILQ_HEAD_INITIALIZER(clkdom_list); /* * Each clock node is linked on a system-wide list and can be searched by name. */ static clknode_list_t clknode_list = TAILQ_HEAD_INITIALIZER(clknode_list); /* * Locking - we use three levels of locking: * - First, topology lock is taken. This one protect all lists. * - Second level is per clknode lock. It protects clknode data. * - Third level is outside of this file, it protect clock device registers. * First two levels use sleepable locks; clock device can use mutex or sx lock. */ static struct sx clk_topo_lock; SX_SYSINIT(clock_topology, &clk_topo_lock, "Clock topology lock"); #define CLK_TOPO_SLOCK() sx_slock(&clk_topo_lock) #define CLK_TOPO_XLOCK() sx_xlock(&clk_topo_lock) #define CLK_TOPO_UNLOCK() sx_unlock(&clk_topo_lock) #define CLK_TOPO_ASSERT() sx_assert(&clk_topo_lock, SA_LOCKED) #define CLK_TOPO_XASSERT() sx_assert(&clk_topo_lock, SA_XLOCKED) #define CLKNODE_SLOCK(_sc) sx_slock(&((_sc)->lock)) #define CLKNODE_XLOCK(_sc) sx_xlock(&((_sc)->lock)) #define CLKNODE_UNLOCK(_sc) sx_unlock(&((_sc)->lock)) static void clknode_adjust_parent(struct clknode *clknode, int idx); enum clknode_sysctl_type { CLKNODE_SYSCTL_PARENT, CLKNODE_SYSCTL_PARENTS_LIST, CLKNODE_SYSCTL_CHILDREN_LIST, }; static int clknode_sysctl(SYSCTL_HANDLER_ARGS); static int clkdom_sysctl(SYSCTL_HANDLER_ARGS); static void clknode_finish(void *dummy); SYSINIT(clknode_finish, SI_SUB_LAST, SI_ORDER_ANY, clknode_finish, NULL); /* * Default clock methods for base class. */ static int clknode_method_init(struct clknode *clknode, device_t dev) { return (0); } static int clknode_method_recalc_freq(struct clknode *clknode, uint64_t *freq) { return (0); } static int clknode_method_set_freq(struct clknode *clknode, uint64_t fin, uint64_t *fout, int flags, int *stop) { *stop = 0; return (0); } static int clknode_method_set_gate(struct clknode *clk, bool enable) { return (0); } static int clknode_method_set_mux(struct clknode *clk, int idx) { return (0); } /* * Internal functions. */ /* * Duplicate an array of parent names. * * Compute total size and allocate a single block which holds both the array of * pointers to strings and the copied strings themselves. Returns a pointer to * the start of the block where the array of copied string pointers lives. * * XXX Revisit this, no need for the DECONST stuff. */ static const char ** strdup_list(const char **names, int num) { size_t len, slen; const char **outptr, *ptr; int i; len = sizeof(char *) * num; for (i = 0; i < num; i++) { if (names[i] == NULL) continue; slen = strlen(names[i]); if (slen == 0) panic("Clock parent names array have empty string"); len += slen + 1; } outptr = malloc(len, M_CLOCK, M_WAITOK | M_ZERO); ptr = (char *)(outptr + num); for (i = 0; i < num; i++) { if (names[i] == NULL) continue; outptr[i] = ptr; slen = strlen(names[i]) + 1; bcopy(names[i], __DECONST(void *, outptr[i]), slen); ptr += slen; } return (outptr); } /* * Recompute the cached frequency for this node and all its children. */ static int clknode_refresh_cache(struct clknode *clknode, uint64_t freq) { int rv; struct clknode *entry; CLK_TOPO_XASSERT(); /* Compute generated frequency. */ rv = CLKNODE_RECALC_FREQ(clknode, &freq); if (rv != 0) { /* XXX If an error happens while refreshing children * this leaves the world in a partially-updated state. * Panic for now. */ panic("clknode_refresh_cache failed for '%s'\n", clknode->name); return (rv); } /* Refresh cache for this node. */ clknode->freq = freq; /* Refresh cache for all children. */ TAILQ_FOREACH(entry, &(clknode->children), sibling_link) { rv = clknode_refresh_cache(entry, freq); if (rv != 0) return (rv); } return (0); } /* * Public interface. */ struct clknode * clknode_find_by_name(const char *name) { struct clknode *entry; CLK_TOPO_ASSERT(); TAILQ_FOREACH(entry, &clknode_list, clklist_link) { if (strcmp(entry->name, name) == 0) return (entry); } return (NULL); } struct clknode * clknode_find_by_id(struct clkdom *clkdom, intptr_t id) { struct clknode *entry; CLK_TOPO_ASSERT(); TAILQ_FOREACH(entry, &clkdom->clknode_list, clkdom_link) { if (entry->id == id) return (entry); } return (NULL); } /* -------------------------------------------------------------------------- */ /* * Clock domain functions */ /* Find clock domain associated to device in global list. */ struct clkdom * clkdom_get_by_dev(const device_t dev) { struct clkdom *entry; CLK_TOPO_ASSERT(); TAILQ_FOREACH(entry, &clkdom_list, link) { if (entry->dev == dev) return (entry); } return (NULL); } #ifdef FDT /* Default DT mapper. */ static int clknode_default_ofw_map(struct clkdom *clkdom, uint32_t ncells, phandle_t *cells, struct clknode **clk) { CLK_TOPO_ASSERT(); if (ncells == 0) *clk = clknode_find_by_id(clkdom, 1); else if (ncells == 1) *clk = clknode_find_by_id(clkdom, cells[0]); else return (ERANGE); if (*clk == NULL) return (ENXIO); return (0); } #endif /* * Create a clock domain. Returns with the topo lock held. */ struct clkdom * clkdom_create(device_t dev) { struct clkdom *clkdom; clkdom = malloc(sizeof(struct clkdom), M_CLOCK, M_WAITOK | M_ZERO); clkdom->dev = dev; TAILQ_INIT(&clkdom->clknode_list); #ifdef FDT clkdom->ofw_mapper = clknode_default_ofw_map; #endif SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "clocks", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, clkdom, 0, clkdom_sysctl, "A", "Clock list for the domain"); return (clkdom); } void clkdom_unlock(struct clkdom *clkdom) { CLK_TOPO_UNLOCK(); } void clkdom_xlock(struct clkdom *clkdom) { CLK_TOPO_XLOCK(); } /* * Finalize initialization of clock domain. Releases topo lock. * * XXX Revisit failure handling. */ int clkdom_finit(struct clkdom *clkdom) { struct clknode *clknode; int i, rv; #ifdef FDT phandle_t node; if ((node = ofw_bus_get_node(clkdom->dev)) == -1) { device_printf(clkdom->dev, "%s called on not ofw based device\n", __func__); return (ENXIO); } #endif rv = 0; /* Make clock domain globally visible. */ CLK_TOPO_XLOCK(); TAILQ_INSERT_TAIL(&clkdom_list, clkdom, link); #ifdef FDT OF_device_register_xref(OF_xref_from_node(node), clkdom->dev); #endif /* Register all clock names into global list. */ TAILQ_FOREACH(clknode, &clkdom->clknode_list, clkdom_link) { TAILQ_INSERT_TAIL(&clknode_list, clknode, clklist_link); } /* * At this point all domain nodes must be registered and all * parents must be valid. */ TAILQ_FOREACH(clknode, &clkdom->clknode_list, clkdom_link) { if (clknode->parent_cnt == 0) continue; for (i = 0; i < clknode->parent_cnt; i++) { if (clknode->parents[i] != NULL) continue; if (clknode->parent_names[i] == NULL) continue; clknode->parents[i] = clknode_find_by_name( clknode->parent_names[i]); if (clknode->parents[i] == NULL) { device_printf(clkdom->dev, "Clock %s have unknown parent: %s\n", clknode->name, clknode->parent_names[i]); rv = ENODEV; } } /* If parent index is not set yet... */ if (clknode->parent_idx == CLKNODE_IDX_NONE) { device_printf(clkdom->dev, "Clock %s have not set parent idx\n", clknode->name); rv = ENXIO; continue; } if (clknode->parents[clknode->parent_idx] == NULL) { device_printf(clkdom->dev, "Clock %s have unknown parent(idx %d): %s\n", clknode->name, clknode->parent_idx, clknode->parent_names[clknode->parent_idx]); rv = ENXIO; continue; } clknode_adjust_parent(clknode, clknode->parent_idx); } CLK_TOPO_UNLOCK(); return (rv); } /* Dump clock domain. */ void clkdom_dump(struct clkdom * clkdom) { struct clknode *clknode; int rv; uint64_t freq; CLK_TOPO_SLOCK(); TAILQ_FOREACH(clknode, &clkdom->clknode_list, clkdom_link) { rv = clknode_get_freq(clknode, &freq); printf("Clock: %s, parent: %s(%d), freq: %ju\n", clknode->name, clknode->parent == NULL ? "(NULL)" : clknode->parent->name, clknode->parent_idx, (uintmax_t)((rv == 0) ? freq: rv)); } CLK_TOPO_UNLOCK(); } /* * Create and initialize clock object, but do not register it. */ struct clknode * clknode_create(struct clkdom * clkdom, clknode_class_t clknode_class, const struct clknode_init_def *def) { struct clknode *clknode; struct sysctl_oid *clknode_oid; bool replaced; KASSERT(def->name != NULL, ("clock name is NULL")); KASSERT(def->name[0] != '\0', ("clock name is empty")); if (def->flags & CLK_NODE_LINKED) { KASSERT(def->parent_cnt == 0, ("Linked clock must not have parents")); KASSERT(clknode_class->size== 0, ("Linked clock cannot have own softc")); } /* Process duplicated clocks */ CLK_TOPO_SLOCK(); clknode = clknode_find_by_name(def->name); CLK_TOPO_UNLOCK(); if (clknode != NULL) { if (!(clknode->flags & CLK_NODE_LINKED) && def->flags & CLK_NODE_LINKED) { /* * New clock is linked and real already exists. * Do nothing and return real node. It is in right * domain, enqueued in right lists and fully initialized. */ return (clknode); } else if (clknode->flags & CLK_NODE_LINKED && !(def->flags & CLK_NODE_LINKED)) { /* * New clock is real but linked already exists. * Remove old linked node from originating domain * (real clock must be owned by another) and from * global names link (it will be added back into it * again in following clknode_register()). Then reuse * original clknode structure and reinitialize it * with new dat. By this, all lists containing this * node remains valid, but the new node virtually * replace the linked one. */ KASSERT(clkdom != clknode->clkdom, ("linked clock must be from another " "domain that real one")); TAILQ_REMOVE(&clkdom->clknode_list, clknode, clkdom_link); TAILQ_REMOVE(&clknode_list, clknode, clklist_link); replaced = true; } else if (clknode->flags & CLK_NODE_LINKED && def->flags & CLK_NODE_LINKED) { /* * Both clocks are linked. * Return old one, so we hold only one copy od link. */ return (clknode); } else { /* Both clocks are real */ panic("Duplicated clock registration: %s\n", def->name); } } else { /* Create clknode object and initialize it. */ clknode = malloc(sizeof(struct clknode), M_CLOCK, M_WAITOK | M_ZERO); sx_init(&clknode->lock, "Clocknode lock"); TAILQ_INIT(&clknode->children); replaced = false; } kobj_init((kobj_t)clknode, (kobj_class_t)clknode_class); /* Allocate softc if required. */ if (clknode_class->size > 0) { clknode->softc = malloc(clknode_class->size, M_CLOCK, M_WAITOK | M_ZERO); } /* Prepare array for ptrs to parent clocks. */ clknode->parents = malloc(sizeof(struct clknode *) * def->parent_cnt, M_CLOCK, M_WAITOK | M_ZERO); /* Copy all strings unless they're flagged as static. */ if (def->flags & CLK_NODE_STATIC_STRINGS) { clknode->name = def->name; clknode->parent_names = def->parent_names; } else { clknode->name = strdup(def->name, M_CLOCK); clknode->parent_names = strdup_list(def->parent_names, def->parent_cnt); } /* Rest of init. */ clknode->id = def->id; clknode->clkdom = clkdom; clknode->flags = def->flags; clknode->parent_cnt = def->parent_cnt; clknode->parent = NULL; clknode->parent_idx = CLKNODE_IDX_NONE; if (replaced) return (clknode); sysctl_ctx_init(&clknode->sysctl_ctx); clknode_oid = SYSCTL_ADD_NODE(&clknode->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_hw_clock), OID_AUTO, clknode->name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "A clock node"); SYSCTL_ADD_U64(&clknode->sysctl_ctx, SYSCTL_CHILDREN(clknode_oid), OID_AUTO, "frequency", CTLFLAG_RD, &clknode->freq, 0, "The clock frequency"); SYSCTL_ADD_PROC(&clknode->sysctl_ctx, SYSCTL_CHILDREN(clknode_oid), OID_AUTO, "parent", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, clknode, CLKNODE_SYSCTL_PARENT, clknode_sysctl, "A", "The clock parent"); SYSCTL_ADD_PROC(&clknode->sysctl_ctx, SYSCTL_CHILDREN(clknode_oid), OID_AUTO, "parents", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, clknode, CLKNODE_SYSCTL_PARENTS_LIST, clknode_sysctl, "A", "The clock parents list"); SYSCTL_ADD_PROC(&clknode->sysctl_ctx, SYSCTL_CHILDREN(clknode_oid), OID_AUTO, "childrens", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, clknode, CLKNODE_SYSCTL_CHILDREN_LIST, clknode_sysctl, "A", "The clock childrens list"); SYSCTL_ADD_INT(&clknode->sysctl_ctx, SYSCTL_CHILDREN(clknode_oid), OID_AUTO, "enable_cnt", CTLFLAG_RD, &clknode->enable_cnt, 0, "The clock enable counter"); return (clknode); } /* * Register clock object into clock domain hierarchy. */ struct clknode * clknode_register(struct clkdom * clkdom, struct clknode *clknode) { int rv; /* Skip already registered linked node */ if (clknode->flags & CLK_NODE_REGISTERED) return(clknode); rv = CLKNODE_INIT(clknode, clknode_get_device(clknode)); if (rv != 0) { printf(" CLKNODE_INIT failed: %d\n", rv); return (NULL); } TAILQ_INSERT_TAIL(&clkdom->clknode_list, clknode, clkdom_link); clknode->flags |= CLK_NODE_REGISTERED; return (clknode); } static void clknode_finish(void *dummy) { struct clknode *clknode; CLK_TOPO_SLOCK(); TAILQ_FOREACH(clknode, &clknode_list, clklist_link) { if (clknode->flags & CLK_NODE_LINKED) printf("Unresolved linked clock found: %s\n", clknode->name); } CLK_TOPO_UNLOCK(); } /* * Clock providers interface. */ /* * Reparent clock node. */ static void clknode_adjust_parent(struct clknode *clknode, int idx) { CLK_TOPO_XASSERT(); if (clknode->parent_cnt == 0) return; if ((idx == CLKNODE_IDX_NONE) || (idx >= clknode->parent_cnt)) panic("%s: Invalid parent index %d for clock %s", __func__, idx, clknode->name); if (clknode->parents[idx] == NULL) panic("%s: Invalid parent index %d for clock %s", __func__, idx, clknode->name); /* Remove me from old children list. */ if (clknode->parent != NULL) { TAILQ_REMOVE(&clknode->parent->children, clknode, sibling_link); } /* Insert into children list of new parent. */ clknode->parent_idx = idx; clknode->parent = clknode->parents[idx]; TAILQ_INSERT_TAIL(&clknode->parent->children, clknode, sibling_link); } /* * Set parent index - init function. */ void clknode_init_parent_idx(struct clknode *clknode, int idx) { if (clknode->parent_cnt == 0) { clknode->parent_idx = CLKNODE_IDX_NONE; clknode->parent = NULL; return; } if ((idx == CLKNODE_IDX_NONE) || (idx >= clknode->parent_cnt) || (clknode->parent_names[idx] == NULL)) panic("%s: Invalid parent index %d for clock %s", __func__, idx, clknode->name); clknode->parent_idx = idx; } int clknode_set_parent_by_idx(struct clknode *clknode, int idx) { int rv; uint64_t freq; int oldidx; /* We have exclusive topology lock, node lock is not needed. */ CLK_TOPO_XASSERT(); if (clknode->parent_cnt == 0) return (0); if (clknode->parent_idx == idx) return (0); oldidx = clknode->parent_idx; clknode_adjust_parent(clknode, idx); rv = CLKNODE_SET_MUX(clknode, idx); if (rv != 0) { clknode_adjust_parent(clknode, oldidx); return (rv); } rv = clknode_get_freq(clknode->parent, &freq); if (rv != 0) return (rv); rv = clknode_refresh_cache(clknode, freq); return (rv); } int clknode_set_parent_by_name(struct clknode *clknode, const char *name) { int rv; uint64_t freq; int oldidx, idx; /* We have exclusive topology lock, node lock is not needed. */ CLK_TOPO_XASSERT(); if (clknode->parent_cnt == 0) return (0); /* * If this node doesnt have mux, then passthrough request to parent. * This feature is used in clock domain initialization and allows us to * set clock source and target frequency on the tail node of the clock * chain. */ if (clknode->parent_cnt == 1) { rv = clknode_set_parent_by_name(clknode->parent, name); return (rv); } for (idx = 0; idx < clknode->parent_cnt; idx++) { if (clknode->parent_names[idx] == NULL) continue; if (strcmp(clknode->parent_names[idx], name) == 0) break; } if (idx >= clknode->parent_cnt) { return (ENXIO); } if (clknode->parent_idx == idx) return (0); oldidx = clknode->parent_idx; clknode_adjust_parent(clknode, idx); rv = CLKNODE_SET_MUX(clknode, idx); if (rv != 0) { clknode_adjust_parent(clknode, oldidx); CLKNODE_UNLOCK(clknode); return (rv); } rv = clknode_get_freq(clknode->parent, &freq); if (rv != 0) return (rv); rv = clknode_refresh_cache(clknode, freq); return (rv); } struct clknode * clknode_get_parent(struct clknode *clknode) { return (clknode->parent); } const char * clknode_get_name(struct clknode *clknode) { return (clknode->name); } const char ** clknode_get_parent_names(struct clknode *clknode) { return (clknode->parent_names); } int clknode_get_parents_num(struct clknode *clknode) { return (clknode->parent_cnt); } int clknode_get_parent_idx(struct clknode *clknode) { return (clknode->parent_idx); } int clknode_get_flags(struct clknode *clknode) { return (clknode->flags); } void * clknode_get_softc(struct clknode *clknode) { return (clknode->softc); } device_t clknode_get_device(struct clknode *clknode) { return (clknode->clkdom->dev); } #ifdef FDT void clkdom_set_ofw_mapper(struct clkdom * clkdom, clknode_ofw_mapper_func *map) { clkdom->ofw_mapper = map; } #endif /* * Real consumers executive */ int clknode_get_freq(struct clknode *clknode, uint64_t *freq) { int rv; CLK_TOPO_ASSERT(); /* Use cached value, if it exists. */ *freq = clknode->freq; if (*freq != 0) return (0); /* Get frequency from parent, if the clock has a parent. */ if (clknode->parent_cnt > 0) { rv = clknode_get_freq(clknode->parent, freq); if (rv != 0) { return (rv); } } /* And recalculate my output frequency. */ CLKNODE_XLOCK(clknode); rv = CLKNODE_RECALC_FREQ(clknode, freq); if (rv != 0) { CLKNODE_UNLOCK(clknode); printf("Cannot get frequency for clk: %s, error: %d\n", clknode->name, rv); return (rv); } /* Save new frequency to cache. */ clknode->freq = *freq; CLKNODE_UNLOCK(clknode); return (0); } int clknode_set_freq(struct clknode *clknode, uint64_t freq, int flags, int enablecnt) { int rv, done; uint64_t parent_freq; /* We have exclusive topology lock, node lock is not needed. */ CLK_TOPO_XASSERT(); /* Check for no change */ if (clknode->freq == freq) return (0); parent_freq = 0; /* * We can set frequency only if * clock is disabled * OR * clock is glitch free and is enabled by calling consumer only */ if ((flags & CLK_SET_DRYRUN) == 0 && clknode->enable_cnt > 1 && clknode->enable_cnt > enablecnt && (clknode->flags & CLK_NODE_GLITCH_FREE) == 0) { return (EBUSY); } /* Get frequency from parent, if the clock has a parent. */ if (clknode->parent_cnt > 0) { rv = clknode_get_freq(clknode->parent, &parent_freq); if (rv != 0) { return (rv); } } /* Set frequency for this clock. */ rv = CLKNODE_SET_FREQ(clknode, parent_freq, &freq, flags, &done); if (rv != 0) { printf("Cannot set frequency for clk: %s, error: %d\n", clknode->name, rv); if ((flags & CLK_SET_DRYRUN) == 0) clknode_refresh_cache(clknode, parent_freq); return (rv); } if (done) { /* Success - invalidate frequency cache for all children. */ if ((flags & CLK_SET_DRYRUN) == 0) { clknode->freq = freq; /* Clock might have reparent during set_freq */ if (clknode->parent_cnt > 0) { rv = clknode_get_freq(clknode->parent, &parent_freq); if (rv != 0) { return (rv); } } clknode_refresh_cache(clknode, parent_freq); } } else if (clknode->parent != NULL) { /* Nothing changed, pass request to parent. */ rv = clknode_set_freq(clknode->parent, freq, flags, enablecnt); } else { /* End of chain without action. */ printf("Cannot set frequency for clk: %s, end of chain\n", clknode->name); rv = ENXIO; } return (rv); } int clknode_enable(struct clknode *clknode) { int rv; CLK_TOPO_ASSERT(); /* Enable clock for each node in chain, starting from source. */ if (clknode->parent_cnt > 0) { rv = clknode_enable(clknode->parent); if (rv != 0) { return (rv); } } /* Handle this node */ CLKNODE_XLOCK(clknode); if (clknode->enable_cnt == 0) { rv = CLKNODE_SET_GATE(clknode, 1); if (rv != 0) { CLKNODE_UNLOCK(clknode); return (rv); } } clknode->enable_cnt++; CLKNODE_UNLOCK(clknode); return (0); } int clknode_disable(struct clknode *clknode) { int rv; CLK_TOPO_ASSERT(); rv = 0; CLKNODE_XLOCK(clknode); /* Disable clock for each node in chain, starting from consumer. */ if ((clknode->enable_cnt == 1) && ((clknode->flags & CLK_NODE_CANNOT_STOP) == 0)) { rv = CLKNODE_SET_GATE(clknode, 0); if (rv != 0) { CLKNODE_UNLOCK(clknode); return (rv); } } clknode->enable_cnt--; CLKNODE_UNLOCK(clknode); if (clknode->parent_cnt > 0) { rv = clknode_disable(clknode->parent); } return (rv); } int clknode_stop(struct clknode *clknode, int depth) { int rv; CLK_TOPO_ASSERT(); rv = 0; CLKNODE_XLOCK(clknode); /* The first node cannot be enabled. */ if ((clknode->enable_cnt != 0) && (depth == 0)) { CLKNODE_UNLOCK(clknode); return (EBUSY); } /* Stop clock for each node in chain, starting from consumer. */ if ((clknode->enable_cnt == 0) && ((clknode->flags & CLK_NODE_CANNOT_STOP) == 0)) { rv = CLKNODE_SET_GATE(clknode, 0); if (rv != 0) { CLKNODE_UNLOCK(clknode); return (rv); } } CLKNODE_UNLOCK(clknode); if (clknode->parent_cnt > 0) rv = clknode_stop(clknode->parent, depth + 1); return (rv); } /* -------------------------------------------------------------------------- * * Clock consumers interface. * */ /* Helper function for clk_get*() */ static clk_t clk_create(struct clknode *clknode, device_t dev) { struct clk *clk; CLK_TOPO_ASSERT(); clk = malloc(sizeof(struct clk), M_CLOCK, M_WAITOK); clk->dev = dev; clk->clknode = clknode; clk->enable_cnt = 0; clknode->ref_cnt++; return (clk); } int clk_get_freq(clk_t clk, uint64_t *freq) { int rv; struct clknode *clknode; clknode = clk->clknode; KASSERT(clknode->ref_cnt > 0, ("Attempt to access unreferenced clock: %s\n", clknode->name)); CLK_TOPO_SLOCK(); rv = clknode_get_freq(clknode, freq); CLK_TOPO_UNLOCK(); return (rv); } int clk_set_freq(clk_t clk, uint64_t freq, int flags) { int rv; struct clknode *clknode; flags &= CLK_SET_USER_MASK; clknode = clk->clknode; KASSERT(clknode->ref_cnt > 0, ("Attempt to access unreferenced clock: %s\n", clknode->name)); CLK_TOPO_XLOCK(); rv = clknode_set_freq(clknode, freq, flags, clk->enable_cnt); CLK_TOPO_UNLOCK(); return (rv); } int clk_test_freq(clk_t clk, uint64_t freq, int flags) { int rv; struct clknode *clknode; flags &= CLK_SET_USER_MASK; clknode = clk->clknode; KASSERT(clknode->ref_cnt > 0, ("Attempt to access unreferenced clock: %s\n", clknode->name)); CLK_TOPO_XLOCK(); rv = clknode_set_freq(clknode, freq, flags | CLK_SET_DRYRUN, 0); CLK_TOPO_UNLOCK(); return (rv); } int clk_get_parent(clk_t clk, clk_t *parent) { struct clknode *clknode; struct clknode *parentnode; clknode = clk->clknode; KASSERT(clknode->ref_cnt > 0, ("Attempt to access unreferenced clock: %s\n", clknode->name)); CLK_TOPO_SLOCK(); parentnode = clknode_get_parent(clknode); if (parentnode == NULL) { CLK_TOPO_UNLOCK(); return (ENODEV); } *parent = clk_create(parentnode, clk->dev); CLK_TOPO_UNLOCK(); return (0); } int clk_set_parent_by_clk(clk_t clk, clk_t parent) { int rv; struct clknode *clknode; struct clknode *parentnode; clknode = clk->clknode; parentnode = parent->clknode; KASSERT(clknode->ref_cnt > 0, ("Attempt to access unreferenced clock: %s\n", clknode->name)); KASSERT(parentnode->ref_cnt > 0, ("Attempt to access unreferenced clock: %s\n", clknode->name)); CLK_TOPO_XLOCK(); rv = clknode_set_parent_by_name(clknode, parentnode->name); CLK_TOPO_UNLOCK(); return (rv); } int clk_enable(clk_t clk) { int rv; struct clknode *clknode; clknode = clk->clknode; KASSERT(clknode->ref_cnt > 0, ("Attempt to access unreferenced clock: %s\n", clknode->name)); CLK_TOPO_SLOCK(); rv = clknode_enable(clknode); if (rv == 0) clk->enable_cnt++; CLK_TOPO_UNLOCK(); return (rv); } int clk_disable(clk_t clk) { int rv; struct clknode *clknode; clknode = clk->clknode; KASSERT(clknode->ref_cnt > 0, ("Attempt to access unreferenced clock: %s\n", clknode->name)); KASSERT(clk->enable_cnt > 0, ("Attempt to disable already disabled clock: %s\n", clknode->name)); CLK_TOPO_SLOCK(); rv = clknode_disable(clknode); if (rv == 0) clk->enable_cnt--; CLK_TOPO_UNLOCK(); return (rv); } int clk_stop(clk_t clk) { int rv; struct clknode *clknode; clknode = clk->clknode; KASSERT(clknode->ref_cnt > 0, ("Attempt to access unreferenced clock: %s\n", clknode->name)); KASSERT(clk->enable_cnt == 0, ("Attempt to stop already enabled clock: %s\n", clknode->name)); CLK_TOPO_SLOCK(); rv = clknode_stop(clknode, 0); CLK_TOPO_UNLOCK(); return (rv); } int clk_release(clk_t clk) { struct clknode *clknode; clknode = clk->clknode; KASSERT(clknode->ref_cnt > 0, ("Attempt to access unreferenced clock: %s\n", clknode->name)); CLK_TOPO_SLOCK(); while (clk->enable_cnt > 0) { clknode_disable(clknode); clk->enable_cnt--; } CLKNODE_XLOCK(clknode); clknode->ref_cnt--; CLKNODE_UNLOCK(clknode); CLK_TOPO_UNLOCK(); free(clk, M_CLOCK); return (0); } const char * clk_get_name(clk_t clk) { const char *name; struct clknode *clknode; clknode = clk->clknode; KASSERT(clknode->ref_cnt > 0, ("Attempt to access unreferenced clock: %s\n", clknode->name)); name = clknode_get_name(clknode); return (name); } int clk_get_by_name(device_t dev, const char *name, clk_t *clk) { struct clknode *clknode; CLK_TOPO_SLOCK(); clknode = clknode_find_by_name(name); if (clknode == NULL) { CLK_TOPO_UNLOCK(); return (ENODEV); } *clk = clk_create(clknode, dev); CLK_TOPO_UNLOCK(); return (0); } int clk_get_by_id(device_t dev, struct clkdom *clkdom, intptr_t id, clk_t *clk) { struct clknode *clknode; CLK_TOPO_SLOCK(); clknode = clknode_find_by_id(clkdom, id); if (clknode == NULL) { CLK_TOPO_UNLOCK(); return (ENODEV); } *clk = clk_create(clknode, dev); CLK_TOPO_UNLOCK(); return (0); } #ifdef FDT static void clk_set_assigned_parent(device_t dev, clk_t clk, int idx) { clk_t parent; const char *pname; int rv; rv = clk_get_by_ofw_index_prop(dev, 0, "assigned-clock-parents", idx, &parent); if (rv != 0) { device_printf(dev, "cannot get parent at idx %d\n", idx); return; } pname = clk_get_name(parent); rv = clk_set_parent_by_clk(clk, parent); if (rv != 0) device_printf(dev, "Cannot set parent %s for clock %s\n", pname, clk_get_name(clk)); else if (bootverbose) device_printf(dev, "Set %s as the parent of %s\n", pname, clk_get_name(clk)); clk_release(parent); } static void clk_set_assigned_rates(device_t dev, clk_t clk, uint32_t freq) { int rv; rv = clk_set_freq(clk, freq, CLK_SET_ROUND_DOWN | CLK_SET_ROUND_UP); if (rv != 0) { device_printf(dev, "Failed to set %s to a frequency of %u\n", clk_get_name(clk), freq); return; } if (bootverbose) device_printf(dev, "Set %s to %u\n", clk_get_name(clk), freq); } int clk_set_assigned(device_t dev, phandle_t node) { clk_t clk; uint32_t *rates; int rv, nclocks, nrates, nparents, i; rv = ofw_bus_parse_xref_list_get_length(node, "assigned-clocks", "#clock-cells", &nclocks); if (rv != 0) { if (rv != ENOENT) device_printf(dev, "cannot parse assigned-clock property\n"); return (rv); } nrates = OF_getencprop_alloc_multi(node, "assigned-clock-rates", sizeof(*rates), (void **)&rates); if (nrates <= 0) nrates = 0; if (ofw_bus_parse_xref_list_get_length(node, "assigned-clock-parents", "#clock-cells", &nparents) != 0) nparents = -1; for (i = 0; i < nclocks; i++) { /* First get the clock we are supposed to modify */ rv = clk_get_by_ofw_index_prop(dev, 0, "assigned-clocks", i, &clk); if (rv != 0) { if (bootverbose) device_printf(dev, "cannot get assigned clock at idx %d\n", i); continue; } /* First set it's parent if needed */ if (i < nparents) clk_set_assigned_parent(dev, clk, i); /* Then set a new frequency */ if (i < nrates && rates[i] != 0) clk_set_assigned_rates(dev, clk, rates[i]); clk_release(clk); } if (rates != NULL) OF_prop_free(rates); return (0); } int clk_get_by_ofw_index_prop(device_t dev, phandle_t cnode, const char *prop, int idx, clk_t *clk) { phandle_t parent, *cells; device_t clockdev; int ncells, rv; struct clkdom *clkdom; struct clknode *clknode; *clk = NULL; if (cnode <= 0) cnode = ofw_bus_get_node(dev); if (cnode <= 0) { device_printf(dev, "%s called on not ofw based device\n", __func__); return (ENXIO); } rv = ofw_bus_parse_xref_list_alloc(cnode, prop, "#clock-cells", idx, &parent, &ncells, &cells); if (rv != 0) { return (rv); } clockdev = OF_device_from_xref(parent); if (clockdev == NULL) { rv = ENODEV; goto done; } CLK_TOPO_SLOCK(); clkdom = clkdom_get_by_dev(clockdev); if (clkdom == NULL){ CLK_TOPO_UNLOCK(); rv = ENXIO; goto done; } rv = clkdom->ofw_mapper(clkdom, ncells, cells, &clknode); if (rv == 0) { *clk = clk_create(clknode, dev); } CLK_TOPO_UNLOCK(); done: if (cells != NULL) OF_prop_free(cells); return (rv); } int clk_get_by_ofw_index(device_t dev, phandle_t cnode, int idx, clk_t *clk) { return (clk_get_by_ofw_index_prop(dev, cnode, "clocks", idx, clk)); } int clk_get_by_ofw_name(device_t dev, phandle_t cnode, const char *name, clk_t *clk) { int rv, idx; if (cnode <= 0) cnode = ofw_bus_get_node(dev); if (cnode <= 0) { device_printf(dev, "%s called on not ofw based device\n", __func__); return (ENXIO); } rv = ofw_bus_find_string_index(cnode, "clock-names", name, &idx); if (rv != 0) return (rv); return (clk_get_by_ofw_index(dev, cnode, idx, clk)); } /* -------------------------------------------------------------------------- * * Support functions for parsing various clock related OFW things. */ /* * Get "clock-output-names" and (optional) "clock-indices" lists. - * Both lists are alocated using M_OFWPROP specifier. + * Both lists are allocated using M_OFWPROP specifier. * * Returns number of items or 0. */ int clk_parse_ofw_out_names(device_t dev, phandle_t node, const char ***out_names, uint32_t **indices) { int name_items, rv; *out_names = NULL; *indices = NULL; if (!OF_hasprop(node, "clock-output-names")) return (0); rv = ofw_bus_string_list_to_array(node, "clock-output-names", out_names); if (rv <= 0) return (0); name_items = rv; if (!OF_hasprop(node, "clock-indices")) return (name_items); rv = OF_getencprop_alloc_multi(node, "clock-indices", sizeof (uint32_t), (void **)indices); if (rv != name_items) { device_printf(dev, " Size of 'clock-output-names' and " "'clock-indices' differs\n"); OF_prop_free(*out_names); OF_prop_free(*indices); return (0); } return (name_items); } /* * Get output clock name for single output clock node. */ int clk_parse_ofw_clk_name(device_t dev, phandle_t node, const char **name) { const char **out_names; const char *tmp_name; int rv; *name = NULL; if (!OF_hasprop(node, "clock-output-names")) { tmp_name = ofw_bus_get_name(dev); if (tmp_name == NULL) return (ENXIO); *name = strdup(tmp_name, M_OFWPROP); return (0); } rv = ofw_bus_string_list_to_array(node, "clock-output-names", &out_names); if (rv != 1) { OF_prop_free(out_names); device_printf(dev, "Malformed 'clock-output-names' property\n"); return (ENXIO); } *name = strdup(out_names[0], M_OFWPROP); OF_prop_free(out_names); return (0); } #endif static int clkdom_sysctl(SYSCTL_HANDLER_ARGS) { struct clkdom *clkdom = arg1; struct clknode *clknode; struct sbuf *sb; int ret; sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); CLK_TOPO_SLOCK(); TAILQ_FOREACH(clknode, &clkdom->clknode_list, clkdom_link) { sbuf_printf(sb, "%s ", clknode->name); } CLK_TOPO_UNLOCK(); ret = sbuf_finish(sb); sbuf_delete(sb); return (ret); } static int clknode_sysctl(SYSCTL_HANDLER_ARGS) { struct clknode *clknode, *children; enum clknode_sysctl_type type = arg2; struct sbuf *sb; const char **parent_names; int ret, i; clknode = arg1; sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); if (sb == NULL) return (ENOMEM); CLK_TOPO_SLOCK(); switch (type) { case CLKNODE_SYSCTL_PARENT: if (clknode->parent) sbuf_printf(sb, "%s", clknode->parent->name); break; case CLKNODE_SYSCTL_PARENTS_LIST: parent_names = clknode_get_parent_names(clknode); for (i = 0; i < clknode->parent_cnt; i++) sbuf_printf(sb, "%s ", parent_names[i]); break; case CLKNODE_SYSCTL_CHILDREN_LIST: TAILQ_FOREACH(children, &(clknode->children), sibling_link) { sbuf_printf(sb, "%s ", children->name); } break; } CLK_TOPO_UNLOCK(); ret = sbuf_finish(sb); sbuf_delete(sb); return (ret); } diff --git a/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c b/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c index 9e6f4df107bd..8e7c410d1c55 100644 --- a/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c +++ b/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c @@ -1,1134 +1,1134 @@ /* * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ #include "opt_inet.h" #include #include #include #include #include #include #include #ifdef CONFIG_NET_RX_BUSY_POLL #include #endif #include "en.h" #if (MLX4_EN_MAX_RX_SEGS == 1) static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, int index) { struct mlx4_en_rx_desc *rx_desc = ((struct mlx4_en_rx_desc *)ring->buf) + index; int i; /* Set size and memtype fields */ rx_desc->data[0].byte_count = cpu_to_be32(priv->rx_mb_size - MLX4_NET_IP_ALIGN); rx_desc->data[0].lkey = cpu_to_be32(priv->mdev->mr.key); /* * If the number of used fragments does not fill up the ring * stride, remaining (unused) fragments must be padded with * null address/size and a special memory key: */ for (i = 1; i < MLX4_EN_MAX_RX_SEGS; i++) { rx_desc->data[i].byte_count = 0; rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); rx_desc->data[i].addr = 0; } } #endif static inline struct mbuf * mlx4_en_alloc_mbuf(struct mlx4_en_rx_ring *ring) { struct mbuf *mb; #if (MLX4_EN_MAX_RX_SEGS == 1) mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); if (likely(mb != NULL)) mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size; #else mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MLX4_EN_MAX_RX_BYTES); if (likely(mb != NULL)) { struct mbuf *mb_head = mb; int i; mb->m_len = MLX4_EN_MAX_RX_BYTES; mb->m_pkthdr.len = MLX4_EN_MAX_RX_BYTES; for (i = 1; i != MLX4_EN_MAX_RX_SEGS; i++) { if (mb_head->m_pkthdr.len >= ring->rx_mb_size) break; mb = (mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0, MLX4_EN_MAX_RX_BYTES)); if (unlikely(mb == NULL)) { m_freem(mb_head); return (NULL); } mb->m_len = MLX4_EN_MAX_RX_BYTES; mb_head->m_pkthdr.len += MLX4_EN_MAX_RX_BYTES; } /* rewind to first mbuf in chain */ mb = mb_head; } #endif return (mb); } static int mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_mbuf *mb_list) { bus_dma_segment_t segs[MLX4_EN_MAX_RX_SEGS]; bus_dmamap_t map; struct mbuf *mb; int nsegs; int err; #if (MLX4_EN_MAX_RX_SEGS != 1) int i; #endif /* try to allocate a new spare mbuf */ if (unlikely(ring->spare.mbuf == NULL)) { mb = mlx4_en_alloc_mbuf(ring); if (unlikely(mb == NULL)) return (-ENOMEM); /* make sure IP header gets aligned */ m_adj(mb, MLX4_NET_IP_ALIGN); /* load spare mbuf into BUSDMA */ err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, ring->spare.dma_map, mb, ring->spare.segs, &nsegs, BUS_DMA_NOWAIT); if (unlikely(err != 0)) { m_freem(mb); return (err); } /* store spare info */ ring->spare.mbuf = mb; #if (MLX4_EN_MAX_RX_SEGS != 1) /* zero remaining segs */ for (i = nsegs; i != MLX4_EN_MAX_RX_SEGS; i++) { ring->spare.segs[i].ds_addr = 0; ring->spare.segs[i].ds_len = 0; } #endif bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, BUS_DMASYNC_PREREAD); } /* synchronize and unload the current mbuf, if any */ if (likely(mb_list->mbuf != NULL)) { bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(ring->dma_tag, mb_list->dma_map); } mb = mlx4_en_alloc_mbuf(ring); if (unlikely(mb == NULL)) goto use_spare; /* make sure IP header gets aligned */ m_adj(mb, MLX4_NET_IP_ALIGN); err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, mb_list->dma_map, mb, segs, &nsegs, BUS_DMA_NOWAIT); if (unlikely(err != 0)) { m_freem(mb); goto use_spare; } #if (MLX4_EN_MAX_RX_SEGS == 1) rx_desc->data[0].addr = cpu_to_be64(segs[0].ds_addr); #else for (i = 0; i != nsegs; i++) { rx_desc->data[i].byte_count = cpu_to_be32(segs[i].ds_len); rx_desc->data[i].lkey = ring->rx_mr_key_be; rx_desc->data[i].addr = cpu_to_be64(segs[i].ds_addr); } for (; i != MLX4_EN_MAX_RX_SEGS; i++) { rx_desc->data[i].byte_count = 0; rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); rx_desc->data[i].addr = 0; } #endif mb_list->mbuf = mb; bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_PREREAD); return (0); use_spare: /* swap DMA maps */ map = mb_list->dma_map; mb_list->dma_map = ring->spare.dma_map; ring->spare.dma_map = map; /* swap MBUFs */ mb_list->mbuf = ring->spare.mbuf; ring->spare.mbuf = NULL; /* store physical address */ #if (MLX4_EN_MAX_RX_SEGS == 1) rx_desc->data[0].addr = cpu_to_be64(ring->spare.segs[0].ds_addr); #else for (i = 0; i != MLX4_EN_MAX_RX_SEGS; i++) { if (ring->spare.segs[i].ds_len != 0) { rx_desc->data[i].byte_count = cpu_to_be32(ring->spare.segs[i].ds_len); rx_desc->data[i].lkey = ring->rx_mr_key_be; rx_desc->data[i].addr = cpu_to_be64(ring->spare.segs[i].ds_addr); } else { rx_desc->data[i].byte_count = 0; rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); rx_desc->data[i].addr = 0; } } #endif return (0); } static void mlx4_en_free_buf(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_mbuf *mb_list) { bus_dmamap_t map = mb_list->dma_map; bus_dmamap_sync(ring->dma_tag, map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(ring->dma_tag, map); m_freem(mb_list->mbuf); mb_list->mbuf = NULL; /* safety clearing */ } static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, int index) { struct mlx4_en_rx_desc *rx_desc = ((struct mlx4_en_rx_desc *)ring->buf) + index; struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + index; mb_list->mbuf = NULL; if (mlx4_en_alloc_buf(ring, rx_desc, mb_list)) { priv->port_stats.rx_alloc_failed++; return (-ENOMEM); } return (0); } static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) { *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); } static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv) { struct mlx4_en_rx_ring *ring; int ring_ind; int buf_ind; int new_size; int err; for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) { for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { ring = priv->rx_ring[ring_ind]; err = mlx4_en_prepare_rx_desc(priv, ring, ring->actual_size); if (err) { if (ring->actual_size == 0) { en_err(priv, "Failed to allocate " "enough rx buffers\n"); return -ENOMEM; } else { new_size = rounddown_pow_of_two(ring->actual_size); en_warn(priv, "Only %d buffers allocated " "reducing ring size to %d\n", ring->actual_size, new_size); goto reduce_rings; } } ring->actual_size++; ring->prod++; } } return 0; reduce_rings: for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { ring = priv->rx_ring[ring_ind]; while (ring->actual_size > new_size) { ring->actual_size--; ring->prod--; mlx4_en_free_buf(ring, ring->mbuf + ring->actual_size); } } return 0; } static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring) { int index; en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n", ring->cons, ring->prod); /* Unmap and free Rx buffers */ BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size); while (ring->cons != ring->prod) { index = ring->cons & ring->size_mask; en_dbg(DRV, priv, "Processing descriptor:%d\n", index); mlx4_en_free_buf(ring, ring->mbuf + index); ++ring->cons; } } void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev) { int i; int num_of_eqs; int num_rx_rings; struct mlx4_dev *dev = mdev->dev; mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { num_of_eqs = max_t(int, MIN_RX_RINGS, min_t(int, mlx4_get_eqs_per_port(mdev->dev, i), DEF_RX_RINGS)); num_rx_rings = mlx4_low_memory_profile() ? MIN_RX_RINGS : num_of_eqs; mdev->profile.prof[i].rx_ring_num = rounddown_pow_of_two(num_rx_rings); } } void mlx4_en_calc_rx_buf(struct ifnet *dev) { struct mlx4_en_priv *priv = netdev_priv(dev); int eff_mtu = dev->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN + MLX4_NET_IP_ALIGN; if (eff_mtu > MJUM16BYTES) { en_err(priv, "MTU(%u) is too big\n", (unsigned)dev->if_mtu); eff_mtu = MJUM16BYTES; } else if (eff_mtu > MJUM9BYTES) { eff_mtu = MJUM16BYTES; } else if (eff_mtu > MJUMPAGESIZE) { eff_mtu = MJUM9BYTES; } else if (eff_mtu > MCLBYTES) { eff_mtu = MJUMPAGESIZE; } else { eff_mtu = MCLBYTES; } priv->rx_mb_size = eff_mtu; en_dbg(DRV, priv, "Effective RX MTU: %d bytes\n", eff_mtu); } int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring **pring, u32 size, int node) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_rx_ring *ring; int err; int tmp; uint32_t x; ring = kzalloc(sizeof(struct mlx4_en_rx_ring), GFP_KERNEL); if (!ring) { en_err(priv, "Failed to allocate RX ring structure\n"); return -ENOMEM; } /* Create DMA descriptor TAG */ if ((err = -bus_dma_tag_create( bus_get_dma_tag(mdev->pdev->dev.bsddev), 1, /* any alignment */ 0, /* no boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ MJUM16BYTES, /* maxsize */ MLX4_EN_MAX_RX_SEGS, /* nsegments */ MJUM16BYTES, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &ring->dma_tag))) { en_err(priv, "Failed to create DMA tag\n"); goto err_ring; } ring->prod = 0; ring->cons = 0; ring->size = size; ring->size_mask = size - 1; ring->log_stride = ilog2(sizeof(struct mlx4_en_rx_desc)); ring->buf_size = (ring->size * sizeof(struct mlx4_en_rx_desc)) + TXBB_SIZE; tmp = size * sizeof(struct mlx4_en_rx_mbuf); ring->mbuf = kzalloc(tmp, GFP_KERNEL); if (ring->mbuf == NULL) { err = -ENOMEM; goto err_dma_tag; } err = -bus_dmamap_create(ring->dma_tag, 0, &ring->spare.dma_map); if (err != 0) goto err_info; for (x = 0; x != size; x++) { err = -bus_dmamap_create(ring->dma_tag, 0, &ring->mbuf[x].dma_map); if (err != 0) { while (x--) bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); goto err_info; } } en_dbg(DRV, priv, "Allocated MBUF ring at addr:%p size:%d\n", ring->mbuf, tmp); err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size, 2 * PAGE_SIZE); if (err) goto err_dma_map; err = mlx4_en_map_buffer(&ring->wqres.buf); if (err) { en_err(priv, "Failed to map RX buffer\n"); goto err_hwq; } ring->buf = ring->wqres.buf.direct.buf; *pring = ring; return 0; err_hwq: mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); err_dma_map: for (x = 0; x != size; x++) { bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); } bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); err_info: vfree(ring->mbuf); err_dma_tag: bus_dma_tag_destroy(ring->dma_tag); err_ring: kfree(ring); return (err); } int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) { struct mlx4_en_rx_ring *ring; #if (MLX4_EN_MAX_RX_SEGS == 1) int i; #endif int ring_ind; int err; for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { ring = priv->rx_ring[ring_ind]; ring->prod = 0; ring->cons = 0; ring->actual_size = 0; ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn; ring->rx_mb_size = priv->rx_mb_size; if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE) { /* Stamp first unused send wqe */ __be32 *ptr = (__be32 *)ring->buf; __be32 stamp = cpu_to_be32(1 << STAMP_SHIFT); *ptr = stamp; /* Move pointer to start of rx section */ ring->buf += TXBB_SIZE; } ring->log_stride = ilog2(sizeof(struct mlx4_en_rx_desc)); ring->buf_size = ring->size * sizeof(struct mlx4_en_rx_desc); memset(ring->buf, 0, ring->buf_size); mlx4_en_update_rx_prod_db(ring); #if (MLX4_EN_MAX_RX_SEGS == 1) /* Initialize all descriptors */ for (i = 0; i < ring->size; i++) mlx4_en_init_rx_desc(priv, ring, i); #endif ring->rx_mr_key_be = cpu_to_be32(priv->mdev->mr.key); #ifdef INET /* Configure lro mngr */ if (priv->dev->if_capenable & IFCAP_LRO) { if (tcp_lro_init(&ring->lro)) priv->dev->if_capenable &= ~IFCAP_LRO; else ring->lro.ifp = priv->dev; } #endif } err = mlx4_en_fill_rx_buffers(priv); if (err) goto err_buffers; for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { ring = priv->rx_ring[ring_ind]; ring->size_mask = ring->actual_size - 1; mlx4_en_update_rx_prod_db(ring); } return 0; err_buffers: for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) mlx4_en_free_rx_buf(priv, priv->rx_ring[ring_ind]); ring_ind = priv->rx_ring_num - 1; while (ring_ind >= 0) { ring = priv->rx_ring[ring_ind]; if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE) ring->buf -= TXBB_SIZE; ring_ind--; } return err; } void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring **pring, u32 size) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_rx_ring *ring = *pring; uint32_t x; mlx4_en_unmap_buffer(&ring->wqres.buf); mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * sizeof(struct mlx4_en_rx_desc) + TXBB_SIZE); for (x = 0; x != size; x++) bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); /* free spare mbuf, if any */ if (ring->spare.mbuf != NULL) { bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(ring->dma_tag, ring->spare.dma_map); m_freem(ring->spare.mbuf); } bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); vfree(ring->mbuf); bus_dma_tag_destroy(ring->dma_tag); kfree(ring); *pring = NULL; #ifdef CONFIG_RFS_ACCEL mlx4_en_cleanup_filters(priv, ring); #endif } void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring) { #ifdef INET tcp_lro_free(&ring->lro); #endif mlx4_en_free_rx_buf(priv, ring); if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE) ring->buf -= TXBB_SIZE; } static void validate_loopback(struct mlx4_en_priv *priv, struct mbuf *mb) { int i; int offset = ETHER_HDR_LEN; for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) { if (*(mb->m_data + offset) != (unsigned char) (i & 0xff)) goto out_loopback; } /* Loopback found */ priv->loopback_ok = 1; out_loopback: m_freem(mb); } static inline int invalid_cqe(struct mlx4_en_priv *priv, struct mlx4_cqe *cqe) { /* Drop packet on bad receive or bad checksum */ if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_ERROR)) { en_err(priv, "CQE completed in error - vendor syndrom:%d syndrom:%d\n", ((struct mlx4_err_cqe *)cqe)->vendor_err_syndrome, ((struct mlx4_err_cqe *)cqe)->syndrome); return 1; } if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) { en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n"); return 1; } return 0; } static struct mbuf * mlx4_en_rx_mb(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_mbuf *mb_list, int length) { #if (MLX4_EN_MAX_RX_SEGS != 1) struct mbuf *mb_head; #endif struct mbuf *mb; /* optimise reception of small packets */ if (length <= (MHLEN - MLX4_NET_IP_ALIGN) && (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) { /* set packet length */ mb->m_pkthdr.len = mb->m_len = length; /* make sure IP header gets aligned */ mb->m_data += MLX4_NET_IP_ALIGN; bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_POSTREAD); bcopy(mtod(mb_list->mbuf, caddr_t), mtod(mb, caddr_t), length); return (mb); } /* get mbuf */ mb = mb_list->mbuf; /* collect used fragment while atomically replacing it */ if (mlx4_en_alloc_buf(ring, rx_desc, mb_list)) return (NULL); /* range check hardware computed value */ if (unlikely(length > mb->m_pkthdr.len)) length = mb->m_pkthdr.len; #if (MLX4_EN_MAX_RX_SEGS == 1) /* update total packet length in packet header */ mb->m_len = mb->m_pkthdr.len = length; #else mb->m_pkthdr.len = length; for (mb_head = mb; mb != NULL; mb = mb->m_next) { if (mb->m_len > length) mb->m_len = length; length -= mb->m_len; if (likely(length == 0)) { if (likely(mb->m_next != NULL)) { /* trim off empty mbufs */ m_freem(mb->m_next); mb->m_next = NULL; } break; } } /* rewind to first mbuf in chain */ mb = mb_head; #endif return (mb); } static __inline int mlx4_en_rss_hash(__be16 status, int udp_rss) { enum { status_all = cpu_to_be16( MLX4_CQE_STATUS_IPV4 | MLX4_CQE_STATUS_IPV4F | MLX4_CQE_STATUS_IPV6 | MLX4_CQE_STATUS_TCP | MLX4_CQE_STATUS_UDP), status_ipv4_tcp = cpu_to_be16( MLX4_CQE_STATUS_IPV4 | MLX4_CQE_STATUS_TCP), status_ipv6_tcp = cpu_to_be16( MLX4_CQE_STATUS_IPV6 | MLX4_CQE_STATUS_TCP), status_ipv4_udp = cpu_to_be16( MLX4_CQE_STATUS_IPV4 | MLX4_CQE_STATUS_UDP), status_ipv6_udp = cpu_to_be16( MLX4_CQE_STATUS_IPV6 | MLX4_CQE_STATUS_UDP), status_ipv4 = cpu_to_be16(MLX4_CQE_STATUS_IPV4), status_ipv6 = cpu_to_be16(MLX4_CQE_STATUS_IPV6) }; status &= status_all; switch (status) { case status_ipv4_tcp: return (M_HASHTYPE_RSS_TCP_IPV4); case status_ipv6_tcp: return (M_HASHTYPE_RSS_TCP_IPV6); case status_ipv4_udp: return (udp_rss ? M_HASHTYPE_RSS_UDP_IPV4 : M_HASHTYPE_RSS_IPV4); case status_ipv6_udp: return (udp_rss ? M_HASHTYPE_RSS_UDP_IPV6 : M_HASHTYPE_RSS_IPV6); default: if (status & status_ipv4) return (M_HASHTYPE_RSS_IPV4); if (status & status_ipv6) return (M_HASHTYPE_RSS_IPV6); return (M_HASHTYPE_OPAQUE_HASH); } } /* For cpu arch with cache line of 64B the performance is better when cqe size==64B * To enlarge cqe size from 32B to 64B --> 32B of garbage (i.e. 0xccccccc) * was added in the beginning of each cqe (the real data is in the corresponding 32B). * The following calc ensures that when factor==1, it means we are aligned to 64B * and we get the real cqe data*/ #define CQE_FACTOR_INDEX(index, factor) (((index) << (factor)) + (factor)) int mlx4_en_process_rx_cq(struct ifnet *dev, struct mlx4_en_cq *cq, int budget) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_cqe *cqe; struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring]; struct mlx4_en_rx_mbuf *mb_list; struct mlx4_en_rx_desc *rx_desc; struct mbuf *mb; struct mlx4_cq *mcq = &cq->mcq; struct mlx4_cqe *buf = cq->buf; int index; unsigned int length; int polled = 0; u32 cons_index = mcq->cons_index; u32 size_mask = ring->size_mask; int size = cq->size; int factor = priv->cqe_factor; const int udp_rss = priv->mdev->profile.udp_rss; if (!priv->port_up) return 0; /* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx * descriptor offset can be deducted from the CQE index instead of * reading 'cqe->index' */ index = cons_index & size_mask; cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; /* Process all completed CQEs */ while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, cons_index & size)) { mb_list = ring->mbuf + index; rx_desc = ((struct mlx4_en_rx_desc *)ring->buf) + index; /* * make sure we read the CQE after we read the ownership bit */ rmb(); if (invalid_cqe(priv, cqe)) { goto next; } /* * Packet is OK - process it. */ length = be32_to_cpu(cqe->byte_cnt); length -= ring->fcs_del; mb = mlx4_en_rx_mb(priv, ring, rx_desc, mb_list, length); if (unlikely(!mb)) { ring->errors++; goto next; } ring->bytes += length; ring->packets++; if (unlikely(priv->validate_loopback)) { validate_loopback(priv, mb); goto next; } /* forward Toeplitz compatible hash value */ mb->m_pkthdr.flowid = be32_to_cpu(cqe->immed_rss_invalid); M_HASHTYPE_SET(mb, mlx4_en_rss_hash(cqe->status, udp_rss)); mb->m_pkthdr.rcvif = dev; if (be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_CVLAN_PRESENT_MASK) { mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->sl_vid); mb->m_flags |= M_VLANTAG; } if (likely(dev->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) && (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) && (cqe->checksum == cpu_to_be16(0xffff))) { priv->port_stats.rx_chksum_good++; mb->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR; mb->m_pkthdr.csum_data = htons(0xffff); /* This packet is eligible for LRO if it is: * - DIX Ethernet (type interpretation) * - TCP/IP (v4) * - without IP options * - not an IP fragment */ #ifdef INET if (mlx4_en_can_lro(cqe->status) && (dev->if_capenable & IFCAP_LRO)) { if (ring->lro.lro_cnt != 0 && tcp_lro_rx(&ring->lro, mb, 0) == 0) goto next; } #endif /* LRO not possible, complete processing here */ INC_PERF_COUNTER(priv->pstats.lro_misses); } else { mb->m_pkthdr.csum_flags = 0; priv->port_stats.rx_chksum_none++; } /* Push it up the stack */ dev->if_input(dev, mb); next: ++cons_index; index = cons_index & size_mask; cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; if (++polled == budget) goto out; } /* Flush all pending IP reassembly sessions */ out: #ifdef INET tcp_lro_flush_all(&ring->lro); #endif AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled); mcq->cons_index = cons_index; mlx4_cq_set_ci(mcq); wmb(); /* ensure HW sees CQ consumer before we post new buffers */ ring->cons = mcq->cons_index; - ring->prod += polled; /* Polled descriptors were realocated in place */ + ring->prod += polled; /* Polled descriptors were reallocated in place */ mlx4_en_update_rx_prod_db(ring); return polled; } /* Rx CQ polling - called by NAPI */ static int mlx4_en_poll_rx_cq(struct mlx4_en_cq *cq, int budget) { struct ifnet *dev = cq->dev; struct epoch_tracker et; int done; NET_EPOCH_ENTER(et); done = mlx4_en_process_rx_cq(dev, cq, budget); NET_EPOCH_EXIT(et); cq->tot_rx += done; return done; } void mlx4_en_rx_irq(struct mlx4_cq *mcq) { struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); struct mlx4_en_priv *priv = netdev_priv(cq->dev); int done; // Shoot one within the irq context // Because there is no NAPI in freeBSD done = mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET); if (priv->port_up && (done == MLX4_EN_RX_BUDGET) ) { cq->curr_poll_rx_cpu_id = curcpu; taskqueue_enqueue(cq->tq, &cq->cq_task); } else { mlx4_en_arm_cq(priv, cq); } } void mlx4_en_rx_que(void *context, int pending) { struct epoch_tracker et; struct mlx4_en_cq *cq; struct thread *td; cq = context; td = curthread; thread_lock(td); sched_bind(td, cq->curr_poll_rx_cpu_id); thread_unlock(td); NET_EPOCH_ENTER(et); while (mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET) == MLX4_EN_RX_BUDGET); NET_EPOCH_EXIT(et); mlx4_en_arm_cq(cq->dev->if_softc, cq); } /* RSS related functions */ static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn, struct mlx4_en_rx_ring *ring, enum mlx4_qp_state *state, struct mlx4_qp *qp) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_qp_context *context; int err = 0; context = kmalloc(sizeof *context , GFP_KERNEL); if (!context) { en_err(priv, "Failed to allocate qp context\n"); return -ENOMEM; } err = mlx4_qp_alloc(mdev->dev, qpn, qp, GFP_KERNEL); if (err) { en_err(priv, "Failed to allocate qp #%x\n", qpn); goto out; } qp->event = mlx4_en_sqp_event; memset(context, 0, sizeof *context); mlx4_en_fill_qp_context(priv, ring->actual_size, sizeof(struct mlx4_en_rx_desc), 0, 0, qpn, ring->cqn, -1, context); context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma); /* Cancel FCS removal if FW allows */ if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) { context->param3 |= cpu_to_be32(1 << 29); ring->fcs_del = ETH_FCS_LEN; } else ring->fcs_del = 0; err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state); if (err) { mlx4_qp_remove(mdev->dev, qp); mlx4_qp_free(mdev->dev, qp); } mlx4_en_update_rx_prod_db(ring); out: kfree(context); return err; } int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv) { int err; u32 qpn; err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn, 0); if (err) { en_err(priv, "Failed reserving drop qpn\n"); return err; } err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp, GFP_KERNEL); if (err) { en_err(priv, "Failed allocating drop qp\n"); mlx4_qp_release_range(priv->mdev->dev, qpn, 1); return err; } return 0; } void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv) { u32 qpn; qpn = priv->drop_qp.qpn; mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp); mlx4_qp_free(priv->mdev->dev, &priv->drop_qp); mlx4_qp_release_range(priv->mdev->dev, qpn, 1); } const u32 * mlx4_en_get_rss_key(struct mlx4_en_priv *priv __unused, u16 *keylen) { static const u32 rsskey[10] = { cpu_to_be32(0xD181C62C), cpu_to_be32(0xF7F4DB5B), cpu_to_be32(0x1983A2FC), cpu_to_be32(0x943E1ADB), cpu_to_be32(0xD9389E6B), cpu_to_be32(0xD1039C2C), cpu_to_be32(0xA74499AD), cpu_to_be32(0x593D56D9), cpu_to_be32(0xF3253C06), cpu_to_be32(0x2ADC1FFC) }; if (keylen != NULL) *keylen = sizeof(rsskey); return (rsskey); } u8 mlx4_en_get_rss_mask(struct mlx4_en_priv *priv) { u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 | MLX4_RSS_TCP_IPV6); if (priv->mdev->profile.udp_rss) rss_mask |= MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6; return (rss_mask); } /* Allocate rx qp's and configure them according to rss map */ int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_rss_map *rss_map = &priv->rss_map; struct mlx4_qp_context context; struct mlx4_rss_context *rss_context; const u32 *key; int rss_rings; void *ptr; int i; int err = 0; int good_qps = 0; en_dbg(DRV, priv, "Configuring rss steering\n"); err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num, priv->rx_ring_num, &rss_map->base_qpn, 0); if (err) { en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num); return err; } for (i = 0; i < priv->rx_ring_num; i++) { priv->rx_ring[i]->qpn = rss_map->base_qpn + i; err = mlx4_en_config_rss_qp(priv, priv->rx_ring[i]->qpn, priv->rx_ring[i], &rss_map->state[i], &rss_map->qps[i]); if (err) goto rss_err; ++good_qps; } /* Configure RSS indirection qp */ err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp, GFP_KERNEL); if (err) { en_err(priv, "Failed to allocate RSS indirection QP\n"); goto rss_err; } rss_map->indir_qp.event = mlx4_en_sqp_event; mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn, priv->rx_ring[0]->cqn, -1, &context); if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num) rss_rings = priv->rx_ring_num; else rss_rings = priv->prof->rss_rings; ptr = ((u8 *)&context) + offsetof(struct mlx4_qp_context, pri_path) + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH; rss_context = ptr; rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 | (rss_map->base_qpn)); rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn); if (priv->mdev->profile.udp_rss) rss_context->base_qpn_udp = rss_context->default_qpn; rss_context->flags = mlx4_en_get_rss_mask(priv); rss_context->hash_fn = MLX4_RSS_HASH_TOP; key = mlx4_en_get_rss_key(priv, NULL); for (i = 0; i < 10; i++) rss_context->rss_key[i] = key[i]; err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context, &rss_map->indir_qp, &rss_map->indir_state); if (err) goto indir_err; return 0; indir_err: mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); mlx4_qp_free(mdev->dev, &rss_map->indir_qp); rss_err: for (i = 0; i < good_qps; i++) { mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); mlx4_qp_free(mdev->dev, &rss_map->qps[i]); } mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); return err; } void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_rss_map *rss_map = &priv->rss_map; int i; mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); mlx4_qp_free(mdev->dev, &rss_map->indir_qp); for (i = 0; i < priv->rx_ring_num; i++) { mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); mlx4_qp_free(mdev->dev, &rss_map->qps[i]); } mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); }