Index: sys/dev/mlx5/mlx5_core/mlx5_eq.c =================================================================== --- sys/dev/mlx5/mlx5_core/mlx5_eq.c +++ sys/dev/mlx5/mlx5_core/mlx5_eq.c @@ -31,6 +31,13 @@ #include #include "mlx5_core.h" +#include "opt_rss.h" + +#ifdef RSS +#include +#include +#endif + enum { MLX5_EQE_SIZE = sizeof(struct mlx5_eqe), MLX5_EQE_OWNER_INIT_VAL = 0x1, @@ -389,6 +396,18 @@ priv->irq_info[vecidx].name, eq); if (err) goto err_eq; +#ifdef RSS + if (vecidx >= MLX5_EQ_VEC_COMP_BASE) { + u8 bucket = vecidx - MLX5_EQ_VEC_COMP_BASE; + err = bind_irq_to_cpu(priv->msix_arr[vecidx].vector, + rss_getcpu(bucket % rss_getnumbuckets())); + if (err) + goto err_irq; + } +#else + if (0) + goto err_irq; +#endif /* EQs are created in ARMED state @@ -398,6 +417,8 @@ kvfree(in); return 0; +err_irq: + free_irq(priv->msix_arr[vecidx].vector, eq); err_eq: mlx5_cmd_destroy_eq(dev, eq->eqn); Index: sys/dev/mlx5/mlx5_en/en.h =================================================================== --- sys/dev/mlx5/mlx5_en/en.h +++ sys/dev/mlx5/mlx5_en/en.h @@ -50,6 +50,13 @@ #include #include +#include "opt_rss.h" + +#ifdef RSS +#include +#include +#endif + #include #ifdef HAVE_TURBO_LRO Index: sys/dev/mlx5/mlx5_en/mlx5_en_main.c =================================================================== --- sys/dev/mlx5/mlx5_en/mlx5_en_main.c +++ sys/dev/mlx5/mlx5_en/mlx5_en_main.c @@ -931,6 +931,10 @@ void *sqc = param->sqc; void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq); +#ifdef RSS + cpuset_t cpu_mask; + int cpu_id; +#endif int err; /* Create DMA descriptor TAG */ @@ -991,9 +995,15 @@ } TASK_INIT(&sq->sq_task, 0, mlx5e_tx_que, sq); - taskqueue_start_threads(&sq->sq_tq, 1, PI_NET, "%s tx sq", - c->ifp->if_xname); - +#ifdef RSS + cpu_id = rss_getcpu(c->ix % rss_getnumbuckets()); + CPU_SETOF(cpu_id, &cpu_mask); + taskqueue_start_threads_cpuset(&sq->sq_tq, 1, PI_NET, &cpu_mask, + "%s TX SQ%d.%d CPU%d", c->ifp->if_xname, c->ix, tc, cpu_id); +#else + taskqueue_start_threads(&sq->sq_tq, 1, PI_NET, + "%s TX SQ%d.%d", c->ifp->if_xname, c->ix, tc); +#endif snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc); mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM, @@ -1768,8 +1778,14 @@ MLX5_SET(rqtc, rqtc, rqt_max_size, sz); for (i = 0; i < sz; i++) { - int ix = i % priv->params.num_channels; - + int ix; +#ifdef RSS + ix = rss_get_indirection_to_bucket(i); +#else + ix = i; +#endif + /* ensure we don't overflow */ + ix %= priv->params.num_channels; MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix]->rq.rqn); } @@ -1834,6 +1850,8 @@ MLX5_CAP_ETH(priv->mdev, lro_timer_supported_periods[2])); } + + /* setup parameters for hashing TIR type, if any */ switch (tt) { case MLX5E_TT_ANY: MLX5_SET(tirc, tirc, disp_type, @@ -1848,8 +1866,16 @@ priv->rqtn); MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ); - MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); +#ifdef RSS + /* + * The FreeBSD RSS implementation does currently not + * support symmetric Toeplitz hashes: + */ + MLX5_SET(tirc, tirc, rx_hash_symmetric, 0); + rss_getkey((uint8_t *)hkey); +#else + MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); hkey[0] = cpu_to_be32(0xD181C62C); hkey[1] = cpu_to_be32(0xF7F4DB5B); hkey[2] = cpu_to_be32(0x1983A2FC); @@ -1860,6 +1886,7 @@ hkey[7] = cpu_to_be32(0x593D56D9); hkey[8] = cpu_to_be32(0xF3253C06); hkey[9] = cpu_to_be32(0x2ADC1FFC); +#endif break; } @@ -1869,6 +1896,12 @@ MLX5_L3_PROT_TYPE_IPV4); MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, MLX5_L4_PROT_TYPE_TCP); +#ifdef RSS + if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) { + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_IP); + } else +#endif MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_ALL); break; @@ -1878,6 +1911,12 @@ MLX5_L3_PROT_TYPE_IPV6); MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, MLX5_L4_PROT_TYPE_TCP); +#ifdef RSS + if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) { + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_IP); + } else +#endif MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_ALL); break; @@ -1887,6 +1926,12 @@ MLX5_L3_PROT_TYPE_IPV4); MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, MLX5_L4_PROT_TYPE_UDP); +#ifdef RSS + if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) { + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_IP); + } else +#endif MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_ALL); break; @@ -1896,6 +1941,12 @@ MLX5_L3_PROT_TYPE_IPV6); MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, MLX5_L4_PROT_TYPE_UDP); +#ifdef RSS + if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) { + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_IP); + } else +#endif MLX5_SET(rx_hash_field_select, hfso, selected_fields, MLX5_HASH_ALL); break; @@ -2052,6 +2103,13 @@ if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0) return (0); +#ifdef RSS + if (rss_getnumbuckets() > priv->params.num_channels) { + if_printf(ifp, "NOTE: There are more RSS buckets(%u) than " + "channels(%u) available\n", rss_getnumbuckets(), + priv->params.num_channels); + } +#endif err = mlx5e_open_tises(priv); if (err) { if_printf(ifp, "%s: mlx5e_open_tises failed, %d\n", Index: sys/dev/mlx5/mlx5_en/mlx5_en_rx.c =================================================================== --- sys/dev/mlx5/mlx5_en/mlx5_en_rx.c +++ sys/dev/mlx5/mlx5_en/mlx5_en_rx.c @@ -192,12 +192,43 @@ mb->m_pkthdr.len = mb->m_len = cqe_bcnt; /* check if a Toeplitz hash was computed */ - if (cqe->rss_hash_type != 0) + if (cqe->rss_hash_type != 0) { mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result); - else +#ifdef RSS + /* decode the RSS hash type */ + switch (cqe->rss_hash_type & + (CQE_RSS_DST_HTYPE_L4 | CQE_RSS_DST_HTYPE_IP)) { + /* IPv4 */ + case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV4): + M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV4); + break; + case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV4): + M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV4); + break; + case CQE_RSS_DST_HTYPE_IPV4: + M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV4); + break; + /* IPv6 */ + case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV6): + M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV6); + break; + case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV6): + M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV6); + break; + case CQE_RSS_DST_HTYPE_IPV6: + M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV6); + break; + default: /* Other */ + M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); + break; + } +#else + M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); +#endif + } else { mb->m_pkthdr.flowid = rq->ix; - - M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); + M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); + } mb->m_pkthdr.rcvif = ifp; if (likely(ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) && Index: sys/dev/mlx5/mlx5_en/mlx5_en_tx.c =================================================================== --- sys/dev/mlx5/mlx5_en/mlx5_en_tx.c +++ sys/dev/mlx5/mlx5_en/mlx5_en_tx.c @@ -85,7 +85,15 @@ /* check if flowid is set */ if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) { - ch = (mb->m_pkthdr.flowid % 128) % ch; +#ifdef RSS + u32 temp; + + if (rss_hash2bucket(mb->m_pkthdr.flowid, + M_HASHTYPE_GET(mb), &temp) == 0) + ch = temp % ch; + else +#endif + ch = (mb->m_pkthdr.flowid % 128) % ch; } else { #if (__FreeBSD_version >= 1100000) ch = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 |