diff --git a/sys/compat/linuxkpi/common/include/linux/netdevice.h b/sys/compat/linuxkpi/common/include/linux/netdevice.h index a904b7e70490..fdfd650a55e5 100644 --- a/sys/compat/linuxkpi/common/include/linux/netdevice.h +++ b/sys/compat/linuxkpi/common/include/linux/netdevice.h @@ -1,335 +1,335 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. * Copyright (c) 2013-2019 Mellanox Technologies, Ltd. * All rights reserved. * Copyright (c) 2020-2021 The FreeBSD Foundation * Copyright (c) 2020-2022 Bjoern A. Zeeb * * Portions of this software were developed by Björn Zeeb * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _LINUXKPI_LINUX_NETDEVICE_H #define _LINUXKPI_LINUX_NETDEVICE_H #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef VIMAGE #define init_net *vnet0 #else #define init_net *((struct vnet *)0) #endif struct sk_buff; struct net_device; struct wireless_dev; /* net/cfg80211.h */ #define MAX_ADDR_LEN 20 #define NET_NAME_UNKNOWN 0 enum netdev_tx { NETDEV_TX_OK = 0, }; typedef enum netdev_tx netdev_tx_t; struct netdev_hw_addr { struct list_head addr_list; uint8_t addr[MAX_ADDR_LEN]; }; struct netdev_hw_addr_list { struct list_head addr_list; int count; }; enum net_device_reg_state { NETREG_DUMMY = 1, NETREG_REGISTERED, }; struct net_device_ops { int (*ndo_open)(struct net_device *); int (*ndo_stop)(struct net_device *); int (*ndo_set_mac_address)(struct net_device *, void *); netdev_tx_t (*ndo_start_xmit)(struct sk_buff *, struct net_device *); void (*ndo_set_rx_mode)(struct net_device *); }; struct net_device { /* BSD specific for compat. */ struct ifnet bsdifp; /* net_device fields seen publicly. */ /* XXX can we later make some aliases to ifnet? */ char name[IFNAMSIZ]; struct wireless_dev *ieee80211_ptr; uint8_t dev_addr[ETH_ALEN]; struct netdev_hw_addr_list mc; netdev_features_t features; struct { unsigned long multicast; unsigned long rx_bytes; unsigned long rx_errors; unsigned long rx_packets; unsigned long tx_bytes; unsigned long tx_dropped; unsigned long tx_errors; unsigned long tx_packets; } stats; enum net_device_reg_state reg_state; const struct ethtool_ops *ethtool_ops; const struct net_device_ops *netdev_ops; bool needs_free_netdev; /* Not properly typed as-of now. */ int flags, type; int name_assign_type, needed_headroom; void (*priv_destructor)(struct net_device *); /* net_device internal. */ struct device dev; /* * In case we delete the net_device we need to be able to clear all * NAPI consumers. */ struct mtx napi_mtx; TAILQ_HEAD(, napi_struct) napi_head; struct taskqueue *napi_tq; /* Must stay last. */ uint8_t drv_priv[0] __aligned(CACHE_LINE_SIZE); }; #define SET_NETDEV_DEV(_ndev, _dev) (_ndev)->dev.parent = _dev; /* -------------------------------------------------------------------------- */ /* According to linux::ipoib_main.c. */ struct netdev_notifier_info { struct net_device *dev; struct ifnet *ifp; }; static inline struct net_device * netdev_notifier_info_to_dev(struct netdev_notifier_info *ni) { return (ni->dev); } static inline struct ifnet * netdev_notifier_info_to_ifp(struct netdev_notifier_info *ni) { return (ni->ifp); } int register_netdevice_notifier(struct notifier_block *); int register_inetaddr_notifier(struct notifier_block *); int unregister_netdevice_notifier(struct notifier_block *); int unregister_inetaddr_notifier(struct notifier_block *); /* -------------------------------------------------------------------------- */ #define NAPI_POLL_WEIGHT 64 /* budget */ /* * There are drivers directly testing napi state bits, so we need to publicly * expose them. If you ask me, those accesses should be hid behind an * inline function and the bit flags not be directly exposed. */ enum napi_state_bits { /* * Official Linux flags encountered. */ NAPI_STATE_SCHED = 1, /* * Our internal versions (for now). */ /* Do not schedule new things while we are waiting to clear things. */ LKPI_NAPI_FLAG_DISABLE_PENDING = 0, /* To synchronise that only one poll is ever running. */ LKPI_NAPI_FLAG_IS_SCHEDULED = 1, /* If trying to schedule while poll is running. Need to re-schedule. */ LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN = 2, /* When shutting down forcefully prevent anything from running task/poll. */ LKPI_NAPI_FLAG_SHUTDOWN = 3, }; struct napi_struct { TAILQ_ENTRY(napi_struct) entry; struct list_head rx_list; struct net_device *dev; int (*poll)(struct napi_struct *, int); int budget; int rx_count; /* * These flags mostly need to be checked/changed atomically * (multiple together in some cases). */ volatile unsigned long state; /* FreeBSD internal. */ /* Use task for now, so we can easily switch between direct and task. */ struct task napi_task; }; void linuxkpi_init_dummy_netdev(struct net_device *); void linuxkpi_netif_napi_add(struct net_device *, struct napi_struct *, - int(*napi_poll)(struct napi_struct *, int), int); + int(*napi_poll)(struct napi_struct *, int)); void linuxkpi_netif_napi_del(struct napi_struct *); bool linuxkpi_napi_schedule_prep(struct napi_struct *); void linuxkpi___napi_schedule(struct napi_struct *); void linuxkpi_napi_schedule(struct napi_struct *); void linuxkpi_napi_reschedule(struct napi_struct *); bool linuxkpi_napi_complete_done(struct napi_struct *, int); bool linuxkpi_napi_complete(struct napi_struct *); void linuxkpi_napi_disable(struct napi_struct *); void linuxkpi_napi_enable(struct napi_struct *); void linuxkpi_napi_synchronize(struct napi_struct *); #define init_dummy_netdev(_n) \ linuxkpi_init_dummy_netdev(_n) -#define netif_napi_add(_nd, _ns, _p, _b) \ - linuxkpi_netif_napi_add(_nd, _ns, _p, _b) +#define netif_napi_add(_nd, _ns, _p) \ + linuxkpi_netif_napi_add(_nd, _ns, _p) #define netif_napi_del(_n) \ linuxkpi_netif_napi_del(_n) #define napi_schedule_prep(_n) \ linuxkpi_napi_schedule_prep(_n) #define __napi_schedule(_n) \ linuxkpi___napi_schedule(_n) #define napi_schedule(_n) \ linuxkpi_napi_schedule(_n) #define napi_reschedule(_n) \ linuxkpi_napi_reschedule(_n) #define napi_complete_done(_n, _r) \ linuxkpi_napi_complete_done(_n, _r) #define napi_complete(_n) \ linuxkpi_napi_complete(_n) #define napi_disable(_n) \ linuxkpi_napi_disable(_n) #define napi_enable(_n) \ linuxkpi_napi_enable(_n) #define napi_synchronize(_n) \ linuxkpi_napi_synchronize(_n) /* -------------------------------------------------------------------------- */ static inline void netdev_rss_key_fill(uint32_t *buf, size_t len) { /* * Remembering from a previous life there was discussions on what is * a good RSS hash key. See end of rss_init() in net/rss_config.c. * iwlwifi is looking for a 10byte "secret" so stay with random for now. */ get_random_bytes(buf, len); } static inline int netdev_hw_addr_list_count(struct netdev_hw_addr_list *list) { return (list->count); } static inline int netdev_mc_count(struct net_device *ndev) { return (netdev_hw_addr_list_count(&ndev->mc)); } #define netdev_hw_addr_list_for_each(_addr, _list) \ list_for_each_entry((_addr), &(_list)->addr_list, addr_list) #define netdev_for_each_mc_addr(na, ndev) \ netdev_hw_addr_list_for_each(na, &(ndev)->mc) static __inline void synchronize_net(void) { /* We probably cannot do that unconditionally at some point anymore. */ synchronize_rcu(); } /* -------------------------------------------------------------------------- */ struct net_device *linuxkpi_alloc_netdev(size_t, const char *, uint32_t, void(*)(struct net_device *)); void linuxkpi_free_netdev(struct net_device *); #define alloc_netdev(_l, _n, _f, _func) \ linuxkpi_alloc_netdev(_l, _n, _f, _func) #define free_netdev(_n) \ linuxkpi_free_netdev(_n) static inline void * netdev_priv(const struct net_device *ndev) { return (__DECONST(void *, ndev->drv_priv)); } /* -------------------------------------------------------------------------- */ /* This is really rtnetlink and probably belongs elsewhere. */ #define rtnl_lock() do { } while(0) #define rtnl_unlock() do { } while(0) #endif /* _LINUXKPI_LINUX_NETDEVICE_H */ diff --git a/sys/compat/linuxkpi/common/src/linux_netdev.c b/sys/compat/linuxkpi/common/src/linux_netdev.c index 3055b9c46767..69adcff67930 100644 --- a/sys/compat/linuxkpi/common/src/linux_netdev.c +++ b/sys/compat/linuxkpi/common/src/linux_netdev.c @@ -1,434 +1,434 @@ /*- * Copyright (c) 2021 The FreeBSD Foundation * Copyright (c) 2022 Bjoern A. Zeeb * * This software was developed by Björn Zeeb under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include MALLOC_DEFINE(M_NETDEV, "lkpindev", "Linux KPI netdevice compat"); #define NAPI_LOCK_INIT(_ndev) \ mtx_init(&(_ndev)->napi_mtx, "napi_mtx", NULL, MTX_DEF) #define NAPI_LOCK_DESTROY(_ndev) mtx_destroy(&(_ndev)->napi_mtx) #define NAPI_LOCK_ASSERT(_ndev) mtx_assert(&(_ndev)->napi_mtx, MA_OWNED) #define NAPI_LOCK(_ndev) mtx_lock(&(_ndev)->napi_mtx) #define NAPI_UNLOCK(_ndev) mtx_unlock(&(_ndev)->napi_mtx) /* -------------------------------------------------------------------------- */ #define LKPI_NAPI_FLAGS \ "\20\1DISABLE_PENDING\2IS_SCHEDULED\3LOST_RACE_TRY_AGAIN" /* #define NAPI_DEBUG */ #ifdef NAPI_DEBUG static int debug_napi; SYSCTL_INT(_compat_linuxkpi, OID_AUTO, debug_napi, CTLFLAG_RWTUN, &debug_napi, 0, "NAPI debug level"); #define DNAPI_TODO 0x01 #define DNAPI_IMPROVE 0x02 #define DNAPI_TRACE 0x10 #define DNAPI_TRACE_TASK 0x20 #define DNAPI_DIRECT_DISPATCH 0x1000 #define NAPI_TRACE(_n) if (debug_napi & DNAPI_TRACE) \ printf("NAPI_TRACE %s:%d %u %p (%#jx %b)\n", __func__, __LINE__, \ (unsigned int)ticks, _n, (uintmax_t)(_n)->state, \ (int)(_n)->state, LKPI_NAPI_FLAGS) #define NAPI_TRACE2D(_n, _d) if (debug_napi & DNAPI_TRACE) \ printf("NAPI_TRACE %s:%d %u %p (%#jx %b) %d\n", __func__, __LINE__, \ (unsigned int)ticks, _n, (uintmax_t)(_n)->state, \ (int)(_n)->state, LKPI_NAPI_FLAGS, _d) #define NAPI_TRACE_TASK(_n, _p, _c) if (debug_napi & DNAPI_TRACE_TASK) \ printf("NAPI_TRACE %s:%d %u %p (%#jx %b) pending %d count %d " \ "rx_count %d\n", __func__, __LINE__, \ (unsigned int)ticks, _n, (uintmax_t)(_n)->state, \ (int)(_n)->state, LKPI_NAPI_FLAGS, _p, _c, (_n)->rx_count) #define NAPI_TODO() if (debug_napi & DNAPI_TODO) \ printf("NAPI_TODO %s:%d %d\n", __func__, __LINE__, ticks) #define NAPI_IMPROVE() if (debug_napi & DNAPI_IMPROVE) \ printf("NAPI_IMPROVE %s:%d %d\n", __func__, __LINE__, ticks) #define NAPI_DIRECT_DISPATCH() ((debug_napi & DNAPI_DIRECT_DISPATCH) != 0) #else #define NAPI_TRACE(_n) do { } while(0) #define NAPI_TRACE2D(_n, _d) do { } while(0) #define NAPI_TRACE_TASK(_n, _p, _c) do { } while(0) #define NAPI_TODO() do { } while(0) #define NAPI_IMPROVE() do { } while(0) #define NAPI_DIRECT_DISPATCH() (0) #endif /* -------------------------------------------------------------------------- */ /* * Check if a poll is running or can run and and if the latter * make us as running. That way we ensure that only one poll * can only ever run at the same time. Returns true if no poll * was scheduled yet. */ bool linuxkpi_napi_schedule_prep(struct napi_struct *napi) { unsigned long old, new; NAPI_TRACE(napi); /* Can can only update/return if all flags agree. */ do { old = READ_ONCE(napi->state); /* If we are stopping, cannot run again. */ if ((old & BIT(LKPI_NAPI_FLAG_DISABLE_PENDING)) != 0) { NAPI_TRACE(napi); return (false); } new = old; /* We were already scheduled. Need to try again? */ if ((old & BIT(LKPI_NAPI_FLAG_IS_SCHEDULED)) != 0) new |= BIT(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN); new |= BIT(LKPI_NAPI_FLAG_IS_SCHEDULED); } while (atomic_cmpset_acq_long(&napi->state, old, new) == 0); NAPI_TRACE(napi); return ((old & BIT(LKPI_NAPI_FLAG_IS_SCHEDULED)) == 0); } static void lkpi___napi_schedule_dd(struct napi_struct *napi) { unsigned long old, new; int rc; rc = 0; again: NAPI_TRACE2D(napi, rc); if (napi->poll != NULL) rc = napi->poll(napi, napi->budget); napi->rx_count += rc; /* Check if interrupts are still disabled, more work to do. */ /* Bandaid for now. */ if (rc >= napi->budget) goto again; /* Bandaid for now. */ if (test_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &napi->state)) goto again; do { new = old = READ_ONCE(napi->state); clear_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &new); clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &new); } while (atomic_cmpset_acq_long(&napi->state, old, new) == 0); NAPI_TRACE2D(napi, rc); } void linuxkpi___napi_schedule(struct napi_struct *napi) { int rc; NAPI_TRACE(napi); if (test_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->state)) { clear_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &napi->state); clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state); NAPI_TRACE(napi); return; } if (NAPI_DIRECT_DISPATCH()) { lkpi___napi_schedule_dd(napi); } else { rc = taskqueue_enqueue(napi->dev->napi_tq, &napi->napi_task); NAPI_TRACE2D(napi, rc); if (rc != 0) { /* Should we assert EPIPE? */ return; } } } void linuxkpi_napi_schedule(struct napi_struct *napi) { NAPI_TRACE(napi); /* * iwlwifi calls this sequence instead of napi_schedule() * to be able to test the prep result. */ if (napi_schedule_prep(napi)) __napi_schedule(napi); } void linuxkpi_napi_reschedule(struct napi_struct *napi) { NAPI_TRACE(napi); /* Not sure what is different to napi_schedule yet. */ if (napi_schedule_prep(napi)) __napi_schedule(napi); } bool linuxkpi_napi_complete_done(struct napi_struct *napi, int ret) { unsigned long old, new; NAPI_TRACE(napi); if (NAPI_DIRECT_DISPATCH()) return (true); do { new = old = READ_ONCE(napi->state); /* * If we lost a race before, we need to re-schedule. * Leave IS_SCHEDULED set essentially doing "_prep". */ if (!test_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &old)) clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &new); clear_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &new); } while (atomic_cmpset_acq_long(&napi->state, old, new) == 0); NAPI_TRACE(napi); /* Someone tried to schedule while poll was running. Re-sched. */ if (test_bit(LKPI_NAPI_FLAG_LOST_RACE_TRY_AGAIN, &old)) { __napi_schedule(napi); return (false); } return (true); } bool linuxkpi_napi_complete(struct napi_struct *napi) { NAPI_TRACE(napi); return (napi_complete_done(napi, 0)); } void linuxkpi_napi_disable(struct napi_struct *napi) { NAPI_TRACE(napi); set_bit(LKPI_NAPI_FLAG_DISABLE_PENDING, &napi->state); while (test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state)) pause_sbt("napidslp", SBT_1MS, 0, C_HARDCLOCK); clear_bit(LKPI_NAPI_FLAG_DISABLE_PENDING, &napi->state); } void linuxkpi_napi_enable(struct napi_struct *napi) { NAPI_TRACE(napi); KASSERT(!test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state), ("%s: enabling napi %p already scheduled\n", __func__, napi)); mb(); /* Let us be scheduled. */ clear_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state); } void linuxkpi_napi_synchronize(struct napi_struct *napi) { NAPI_TRACE(napi); #if defined(SMP) /* Check & sleep while a napi is scheduled. */ while (test_bit(LKPI_NAPI_FLAG_IS_SCHEDULED, &napi->state)) pause_sbt("napisslp", SBT_1MS, 0, C_HARDCLOCK); #else mb(); #endif } /* -------------------------------------------------------------------------- */ static void lkpi_napi_task(void *ctx, int pending) { struct napi_struct *napi; int count; KASSERT(ctx != NULL, ("%s: napi %p, pending %d\n", __func__, ctx, pending)); napi = ctx; KASSERT(napi->poll != NULL, ("%s: napi %p poll is NULL\n", __func__, napi)); NAPI_TRACE_TASK(napi, pending, napi->budget); count = napi->poll(napi, napi->budget); napi->rx_count += count; NAPI_TRACE_TASK(napi, pending, count); /* * We must not check against count < pending here. There are situations * when a driver may "poll" and we may not have any work to do and that * would make us re-schedule ourseless for ever. */ if (count >= napi->budget) { /* * Have to re-schedule ourselves. napi_complete() was not run * in this case which means we are still SCHEDULED. * In order to queue another task we have to directly call * __napi_schedule() without _prep() in the way. */ __napi_schedule(napi); } } /* -------------------------------------------------------------------------- */ void linuxkpi_netif_napi_add(struct net_device *ndev, struct napi_struct *napi, - int(*napi_poll)(struct napi_struct *, int), int budget) + int(*napi_poll)(struct napi_struct *, int)) { napi->dev = ndev; napi->poll = napi_poll; - napi->budget = budget; + napi->budget = NAPI_POLL_WEIGHT; INIT_LIST_HEAD(&napi->rx_list); napi->rx_count = 0; TASK_INIT(&napi->napi_task, 0, lkpi_napi_task, napi); NAPI_LOCK(ndev); TAILQ_INSERT_TAIL(&ndev->napi_head, napi, entry); NAPI_UNLOCK(ndev); /* Anything else to do on the ndev? */ clear_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->state); } static void lkpi_netif_napi_del_locked(struct napi_struct *napi) { struct net_device *ndev; ndev = napi->dev; NAPI_LOCK_ASSERT(ndev); set_bit(LKPI_NAPI_FLAG_SHUTDOWN, &napi->state); TAILQ_REMOVE(&ndev->napi_head, napi, entry); while (taskqueue_cancel(ndev->napi_tq, &napi->napi_task, NULL) != 0) taskqueue_drain(ndev->napi_tq, &napi->napi_task); } void linuxkpi_netif_napi_del(struct napi_struct *napi) { struct net_device *ndev; ndev = napi->dev; NAPI_LOCK(ndev); lkpi_netif_napi_del_locked(napi); NAPI_UNLOCK(ndev); } /* -------------------------------------------------------------------------- */ void linuxkpi_init_dummy_netdev(struct net_device *ndev) { memset(ndev, 0, sizeof(*ndev)); ndev->reg_state = NETREG_DUMMY; NAPI_LOCK_INIT(ndev); TAILQ_INIT(&ndev->napi_head); /* Anything else? */ ndev->napi_tq = taskqueue_create("tq_ndev_napi", M_WAITOK, taskqueue_thread_enqueue, &ndev->napi_tq); /* One thread for now. */ (void) taskqueue_start_threads(&ndev->napi_tq, 1, PWAIT, "ndev napi taskq"); } struct net_device * linuxkpi_alloc_netdev(size_t len, const char *name, uint32_t flags, void(*setup_func)(struct net_device *)) { struct net_device *ndev; ndev = malloc(sizeof(*ndev) + len, M_NETDEV, M_NOWAIT); if (ndev == NULL) return (ndev); /* Always first as it zeros! */ linuxkpi_init_dummy_netdev(ndev); strlcpy(ndev->name, name, sizeof(*ndev->name)); /* This needs extending as we support more. */ setup_func(ndev); return (ndev); } void linuxkpi_free_netdev(struct net_device *ndev) { struct napi_struct *napi, *temp; NAPI_LOCK(ndev); TAILQ_FOREACH_SAFE(napi, &ndev->napi_head, entry, temp) { lkpi_netif_napi_del_locked(napi); } NAPI_UNLOCK(ndev); taskqueue_free(ndev->napi_tq); ndev->napi_tq = NULL; NAPI_LOCK_DESTROY(ndev); /* This needs extending as we support more. */ free(ndev, M_NETDEV); } diff --git a/sys/contrib/dev/iwlwifi/pcie/rx.c b/sys/contrib/dev/iwlwifi/pcie/rx.c index e91121dda102..480cc9c765d8 100644 --- a/sys/contrib/dev/iwlwifi/pcie/rx.c +++ b/sys/contrib/dev/iwlwifi/pcie/rx.c @@ -1,2383 +1,2382 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* * Copyright (C) 2003-2014, 2018-2022 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ #include #include #include #include "iwl-prph.h" #include "iwl-io.h" #include "internal.h" #include "iwl-op-mode.h" #include "iwl-context-info-gen3.h" /****************************************************************************** * * RX path functions * ******************************************************************************/ /* * Rx theory of operation * * Driver allocates a circular buffer of Receive Buffer Descriptors (RBDs), * each of which point to Receive Buffers to be filled by the NIC. These get * used not only for Rx frames, but for any command response or notification * from the NIC. The driver and NIC manage the Rx buffers by means * of indexes into the circular buffer. * * Rx Queue Indexes * The host/firmware share two index registers for managing the Rx buffers. * * The READ index maps to the first position that the firmware may be writing * to -- the driver can read up to (but not including) this position and get * good data. * The READ index is managed by the firmware once the card is enabled. * * The WRITE index maps to the last position the driver has read from -- the * position preceding WRITE is the last slot the firmware can place a packet. * * The queue is empty (no good data) if WRITE = READ - 1, and is full if * WRITE = READ. * * During initialization, the host sets up the READ queue position to the first * INDEX position, and WRITE to the last (READ - 1 wrapped) * * When the firmware places a packet in a buffer, it will advance the READ index * and fire the RX interrupt. The driver can then query the READ index and * process as many packets as possible, moving the WRITE index forward as it * resets the Rx queue buffers with new memory. * * The management in the driver is as follows: * + A list of pre-allocated RBDs is stored in iwl->rxq->rx_free. * When the interrupt handler is called, the request is processed. * The page is either stolen - transferred to the upper layer * or reused - added immediately to the iwl->rxq->rx_free list. * + When the page is stolen - the driver updates the matching queue's used * count, detaches the RBD and transfers it to the queue used list. * When there are two used RBDs - they are transferred to the allocator empty * list. Work is then scheduled for the allocator to start allocating * eight buffers. * When there are another 6 used RBDs - they are transferred to the allocator * empty list and the driver tries to claim the pre-allocated buffers and * add them to iwl->rxq->rx_free. If it fails - it continues to claim them * until ready. * When there are 8+ buffers in the free list - either from allocation or from * 8 reused unstolen pages - restock is called to update the FW and indexes. * + In order to make sure the allocator always has RBDs to use for allocation * the allocator has initial pool in the size of num_queues*(8-2) - the * maximum missing RBDs per allocation request (request posted with 2 * empty RBDs, there is no guarantee when the other 6 RBDs are supplied). * The queues supplies the recycle of the rest of the RBDs. * + A received packet is processed and handed to the kernel network stack, * detached from the iwl->rxq. The driver 'processed' index is updated. * + If there are no allocated buffers in iwl->rxq->rx_free, * the READ INDEX is not incremented and iwl->status(RX_STALLED) is set. * If there were enough free buffers and RX_STALLED is set it is cleared. * * * Driver sequence: * * iwl_rxq_alloc() Allocates rx_free * iwl_pcie_rx_replenish() Replenishes rx_free list from rx_used, and calls * iwl_pcie_rxq_restock. * Used only during initialization. * iwl_pcie_rxq_restock() Moves available buffers from rx_free into Rx * queue, updates firmware pointers, and updates * the WRITE index. * iwl_pcie_rx_allocator() Background work for allocating pages. * * -- enable interrupts -- * ISR - iwl_rx() Detach iwl_rx_mem_buffers from pool up to the * READ INDEX, detaching the SKB from the pool. * Moves the packet buffer from queue to rx_used. * Posts and claims requests to the allocator. * Calls iwl_pcie_rxq_restock to refill any empty * slots. * * RBD life-cycle: * * Init: * rxq.pool -> rxq.rx_used -> rxq.rx_free -> rxq.queue * * Regular Receive interrupt: * Page Stolen: * rxq.queue -> rxq.rx_used -> allocator.rbd_empty -> * allocator.rbd_allocated -> rxq.rx_free -> rxq.queue * Page not Stolen: * rxq.queue -> rxq.rx_free -> rxq.queue * ... * */ /* * iwl_rxq_space - Return number of free slots available in queue. */ static int iwl_rxq_space(const struct iwl_rxq *rxq) { /* Make sure rx queue size is a power of 2 */ WARN_ON(rxq->queue_size & (rxq->queue_size - 1)); /* * There can be up to (RX_QUEUE_SIZE - 1) free slots, to avoid ambiguity * between empty and completely full queues. * The following is equivalent to modulo by RX_QUEUE_SIZE and is well * defined for negative dividends. */ return (rxq->read - rxq->write - 1) & (rxq->queue_size - 1); } /* * iwl_dma_addr2rbd_ptr - convert a DMA address to a uCode read buffer ptr */ static inline __le32 iwl_pcie_dma_addr2rbd_ptr(dma_addr_t dma_addr) { return cpu_to_le32((u32)(dma_addr >> 8)); } /* * iwl_pcie_rx_stop - stops the Rx DMA */ int iwl_pcie_rx_stop(struct iwl_trans *trans) { if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { /* TODO: remove this once fw does it */ iwl_write_umac_prph(trans, RFH_RXF_DMA_CFG_GEN3, 0); return iwl_poll_umac_prph_bit(trans, RFH_GEN_STATUS_GEN3, RXF_DMA_IDLE, RXF_DMA_IDLE, 1000); } else if (trans->trans_cfg->mq_rx_supported) { iwl_write_prph(trans, RFH_RXF_DMA_CFG, 0); return iwl_poll_prph_bit(trans, RFH_GEN_STATUS, RXF_DMA_IDLE, RXF_DMA_IDLE, 1000); } else { iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0); return iwl_poll_direct_bit(trans, FH_MEM_RSSR_RX_STATUS_REG, FH_RSSR_CHNL0_RX_STATUS_CHNL_IDLE, 1000); } } /* * iwl_pcie_rxq_inc_wr_ptr - Update the write pointer for the RX queue */ static void iwl_pcie_rxq_inc_wr_ptr(struct iwl_trans *trans, struct iwl_rxq *rxq) { u32 reg; lockdep_assert_held(&rxq->lock); /* * explicitly wake up the NIC if: * 1. shadow registers aren't enabled * 2. there is a chance that the NIC is asleep */ if (!trans->trans_cfg->base_params->shadow_reg_enable && test_bit(STATUS_TPOWER_PMI, &trans->status)) { reg = iwl_read32(trans, CSR_UCODE_DRV_GP1); if (reg & CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP) { IWL_DEBUG_INFO(trans, "Rx queue requesting wakeup, GP1 = 0x%x\n", reg); iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); rxq->need_update = true; return; } } rxq->write_actual = round_down(rxq->write, 8); if (!trans->trans_cfg->mq_rx_supported) iwl_write32(trans, FH_RSCSR_CHNL0_WPTR, rxq->write_actual); else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) iwl_write32(trans, HBUS_TARG_WRPTR, rxq->write_actual | HBUS_TARG_WRPTR_RX_Q(rxq->id)); else iwl_write32(trans, RFH_Q_FRBDCB_WIDX_TRG(rxq->id), rxq->write_actual); } static void iwl_pcie_rxq_check_wrptr(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); int i; for (i = 0; i < trans->num_rx_queues; i++) { struct iwl_rxq *rxq = &trans_pcie->rxq[i]; if (!rxq->need_update) continue; spin_lock_bh(&rxq->lock); iwl_pcie_rxq_inc_wr_ptr(trans, rxq); rxq->need_update = false; spin_unlock_bh(&rxq->lock); } } static void iwl_pcie_restock_bd(struct iwl_trans *trans, struct iwl_rxq *rxq, struct iwl_rx_mem_buffer *rxb) { if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { struct iwl_rx_transfer_desc *bd = rxq->bd; BUILD_BUG_ON(sizeof(*bd) != 2 * sizeof(u64)); bd[rxq->write].addr = cpu_to_le64(rxb->page_dma); bd[rxq->write].rbid = cpu_to_le16(rxb->vid); } else { __le64 *bd = rxq->bd; bd[rxq->write] = cpu_to_le64(rxb->page_dma | rxb->vid); } #if defined(__linux__) IWL_DEBUG_RX(trans, "Assigned virtual RB ID %u to queue %d index %d\n", #elif defined(__FreeBSD__) IWL_DEBUG_PCI_RW(trans, "Assigned virtual RB ID %u to queue %d index %d\n", (u32)rxb->vid, rxq->id, rxq->write); #endif } /* * iwl_pcie_rxmq_restock - restock implementation for multi-queue rx */ static void iwl_pcie_rxmq_restock(struct iwl_trans *trans, struct iwl_rxq *rxq) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rx_mem_buffer *rxb; /* * If the device isn't enabled - no need to try to add buffers... * This can happen when we stop the device and still have an interrupt * pending. We stop the APM before we sync the interrupts because we * have to (see comment there). On the other hand, since the APM is * stopped, we cannot access the HW (in particular not prph). * So don't try to restock if the APM has been already stopped. */ if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status)) return; spin_lock_bh(&rxq->lock); while (rxq->free_count) { /* Get next free Rx buffer, remove from free list */ rxb = list_first_entry(&rxq->rx_free, struct iwl_rx_mem_buffer, list); list_del(&rxb->list); rxb->invalid = false; /* some low bits are expected to be unset (depending on hw) */ WARN_ON(rxb->page_dma & trans_pcie->supported_dma_mask); /* Point to Rx buffer via next RBD in circular buffer */ iwl_pcie_restock_bd(trans, rxq, rxb); rxq->write = (rxq->write + 1) & (rxq->queue_size - 1); rxq->free_count--; } spin_unlock_bh(&rxq->lock); /* * If we've added more space for the firmware to place data, tell it. * Increment device's write pointer in multiples of 8. */ if (rxq->write_actual != (rxq->write & ~0x7)) { spin_lock_bh(&rxq->lock); iwl_pcie_rxq_inc_wr_ptr(trans, rxq); spin_unlock_bh(&rxq->lock); } } /* * iwl_pcie_rxsq_restock - restock implementation for single queue rx */ static void iwl_pcie_rxsq_restock(struct iwl_trans *trans, struct iwl_rxq *rxq) { struct iwl_rx_mem_buffer *rxb; /* * If the device isn't enabled - not need to try to add buffers... * This can happen when we stop the device and still have an interrupt * pending. We stop the APM before we sync the interrupts because we * have to (see comment there). On the other hand, since the APM is * stopped, we cannot access the HW (in particular not prph). * So don't try to restock if the APM has been already stopped. */ if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status)) return; spin_lock_bh(&rxq->lock); while ((iwl_rxq_space(rxq) > 0) && (rxq->free_count)) { __le32 *bd = (__le32 *)rxq->bd; /* The overwritten rxb must be a used one */ rxb = rxq->queue[rxq->write]; BUG_ON(rxb && rxb->page); /* Get next free Rx buffer, remove from free list */ rxb = list_first_entry(&rxq->rx_free, struct iwl_rx_mem_buffer, list); list_del(&rxb->list); rxb->invalid = false; /* Point to Rx buffer via next RBD in circular buffer */ bd[rxq->write] = iwl_pcie_dma_addr2rbd_ptr(rxb->page_dma); rxq->queue[rxq->write] = rxb; rxq->write = (rxq->write + 1) & RX_QUEUE_MASK; rxq->free_count--; } spin_unlock_bh(&rxq->lock); /* If we've added more space for the firmware to place data, tell it. * Increment device's write pointer in multiples of 8. */ if (rxq->write_actual != (rxq->write & ~0x7)) { spin_lock_bh(&rxq->lock); iwl_pcie_rxq_inc_wr_ptr(trans, rxq); spin_unlock_bh(&rxq->lock); } } /* * iwl_pcie_rxq_restock - refill RX queue from pre-allocated pool * * If there are slots in the RX queue that need to be restocked, * and we have free pre-allocated buffers, fill the ranks as much * as we can, pulling from rx_free. * * This moves the 'write' index forward to catch up with 'processed', and * also updates the memory address in the firmware to reference the new * target buffer. */ static void iwl_pcie_rxq_restock(struct iwl_trans *trans, struct iwl_rxq *rxq) { if (trans->trans_cfg->mq_rx_supported) iwl_pcie_rxmq_restock(trans, rxq); else iwl_pcie_rxsq_restock(trans, rxq); } /* * iwl_pcie_rx_alloc_page - allocates and returns a page. * */ static struct page *iwl_pcie_rx_alloc_page(struct iwl_trans *trans, u32 *offset, gfp_t priority) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); unsigned int rbsize = iwl_trans_get_rb_size(trans_pcie->rx_buf_size); unsigned int allocsize = PAGE_SIZE << trans_pcie->rx_page_order; struct page *page; gfp_t gfp_mask = priority; if (trans_pcie->rx_page_order > 0) gfp_mask |= __GFP_COMP; if (trans_pcie->alloc_page) { spin_lock_bh(&trans_pcie->alloc_page_lock); /* recheck */ if (trans_pcie->alloc_page) { *offset = trans_pcie->alloc_page_used; page = trans_pcie->alloc_page; trans_pcie->alloc_page_used += rbsize; if (trans_pcie->alloc_page_used >= allocsize) trans_pcie->alloc_page = NULL; else get_page(page); spin_unlock_bh(&trans_pcie->alloc_page_lock); return page; } spin_unlock_bh(&trans_pcie->alloc_page_lock); } /* Alloc a new receive buffer */ page = alloc_pages(gfp_mask, trans_pcie->rx_page_order); if (!page) { if (net_ratelimit()) IWL_DEBUG_INFO(trans, "alloc_pages failed, order: %d\n", trans_pcie->rx_page_order); /* * Issue an error if we don't have enough pre-allocated * buffers. */ if (!(gfp_mask & __GFP_NOWARN) && net_ratelimit()) IWL_CRIT(trans, "Failed to alloc_pages\n"); return NULL; } if (2 * rbsize <= allocsize) { spin_lock_bh(&trans_pcie->alloc_page_lock); if (!trans_pcie->alloc_page) { get_page(page); trans_pcie->alloc_page = page; trans_pcie->alloc_page_used = rbsize; } spin_unlock_bh(&trans_pcie->alloc_page_lock); } *offset = 0; return page; } /* * iwl_pcie_rxq_alloc_rbs - allocate a page for each used RBD * * A used RBD is an Rx buffer that has been given to the stack. To use it again * a page must be allocated and the RBD must point to the page. This function * doesn't change the HW pointer but handles the list of pages that is used by * iwl_pcie_rxq_restock. The latter function will update the HW to use the newly * allocated buffers. */ void iwl_pcie_rxq_alloc_rbs(struct iwl_trans *trans, gfp_t priority, struct iwl_rxq *rxq) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rx_mem_buffer *rxb; struct page *page; while (1) { unsigned int offset; spin_lock_bh(&rxq->lock); if (list_empty(&rxq->rx_used)) { spin_unlock_bh(&rxq->lock); return; } spin_unlock_bh(&rxq->lock); page = iwl_pcie_rx_alloc_page(trans, &offset, priority); if (!page) return; spin_lock_bh(&rxq->lock); if (list_empty(&rxq->rx_used)) { spin_unlock_bh(&rxq->lock); __free_pages(page, trans_pcie->rx_page_order); return; } rxb = list_first_entry(&rxq->rx_used, struct iwl_rx_mem_buffer, list); list_del(&rxb->list); spin_unlock_bh(&rxq->lock); BUG_ON(rxb->page); rxb->page = page; rxb->offset = offset; /* Get physical address of the RB */ rxb->page_dma = dma_map_page(trans->dev, page, rxb->offset, trans_pcie->rx_buf_bytes, DMA_FROM_DEVICE); if (dma_mapping_error(trans->dev, rxb->page_dma)) { rxb->page = NULL; spin_lock_bh(&rxq->lock); list_add(&rxb->list, &rxq->rx_used); spin_unlock_bh(&rxq->lock); __free_pages(page, trans_pcie->rx_page_order); return; } spin_lock_bh(&rxq->lock); list_add_tail(&rxb->list, &rxq->rx_free); rxq->free_count++; spin_unlock_bh(&rxq->lock); } } void iwl_pcie_free_rbs_pool(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); int i; if (!trans_pcie->rx_pool) return; for (i = 0; i < RX_POOL_SIZE(trans_pcie->num_rx_bufs); i++) { if (!trans_pcie->rx_pool[i].page) continue; dma_unmap_page(trans->dev, trans_pcie->rx_pool[i].page_dma, trans_pcie->rx_buf_bytes, DMA_FROM_DEVICE); __free_pages(trans_pcie->rx_pool[i].page, trans_pcie->rx_page_order); trans_pcie->rx_pool[i].page = NULL; } } /* * iwl_pcie_rx_allocator - Allocates pages in the background for RX queues * * Allocates for each received request 8 pages * Called as a scheduled work item. */ static void iwl_pcie_rx_allocator(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rb_allocator *rba = &trans_pcie->rba; struct list_head local_empty; int pending = atomic_read(&rba->req_pending); IWL_DEBUG_TPT(trans, "Pending allocation requests = %d\n", pending); /* If we were scheduled - there is at least one request */ spin_lock_bh(&rba->lock); /* swap out the rba->rbd_empty to a local list */ list_replace_init(&rba->rbd_empty, &local_empty); spin_unlock_bh(&rba->lock); while (pending) { int i; LIST_HEAD(local_allocated); gfp_t gfp_mask = GFP_KERNEL; /* Do not post a warning if there are only a few requests */ if (pending < RX_PENDING_WATERMARK) gfp_mask |= __GFP_NOWARN; for (i = 0; i < RX_CLAIM_REQ_ALLOC;) { struct iwl_rx_mem_buffer *rxb; struct page *page; /* List should never be empty - each reused RBD is * returned to the list, and initial pool covers any * possible gap between the time the page is allocated * to the time the RBD is added. */ BUG_ON(list_empty(&local_empty)); /* Get the first rxb from the rbd list */ rxb = list_first_entry(&local_empty, struct iwl_rx_mem_buffer, list); BUG_ON(rxb->page); /* Alloc a new receive buffer */ page = iwl_pcie_rx_alloc_page(trans, &rxb->offset, gfp_mask); if (!page) continue; rxb->page = page; /* Get physical address of the RB */ rxb->page_dma = dma_map_page(trans->dev, page, rxb->offset, trans_pcie->rx_buf_bytes, DMA_FROM_DEVICE); if (dma_mapping_error(trans->dev, rxb->page_dma)) { rxb->page = NULL; __free_pages(page, trans_pcie->rx_page_order); continue; } /* move the allocated entry to the out list */ list_move(&rxb->list, &local_allocated); i++; } atomic_dec(&rba->req_pending); pending--; if (!pending) { pending = atomic_read(&rba->req_pending); if (pending) IWL_DEBUG_TPT(trans, "Got more pending allocation requests = %d\n", pending); } spin_lock_bh(&rba->lock); /* add the allocated rbds to the allocator allocated list */ list_splice_tail(&local_allocated, &rba->rbd_allocated); /* get more empty RBDs for current pending requests */ list_splice_tail_init(&rba->rbd_empty, &local_empty); spin_unlock_bh(&rba->lock); atomic_inc(&rba->req_ready); } spin_lock_bh(&rba->lock); /* return unused rbds to the allocator empty list */ list_splice_tail(&local_empty, &rba->rbd_empty); spin_unlock_bh(&rba->lock); IWL_DEBUG_TPT(trans, "%s, exit.\n", __func__); } /* * iwl_pcie_rx_allocator_get - returns the pre-allocated pages .* .* Called by queue when the queue posted allocation request and * has freed 8 RBDs in order to restock itself. * This function directly moves the allocated RBs to the queue's ownership * and updates the relevant counters. */ static void iwl_pcie_rx_allocator_get(struct iwl_trans *trans, struct iwl_rxq *rxq) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rb_allocator *rba = &trans_pcie->rba; int i; lockdep_assert_held(&rxq->lock); /* * atomic_dec_if_positive returns req_ready - 1 for any scenario. * If req_ready is 0 atomic_dec_if_positive will return -1 and this * function will return early, as there are no ready requests. * atomic_dec_if_positive will perofrm the *actual* decrement only if * req_ready > 0, i.e. - there are ready requests and the function * hands one request to the caller. */ if (atomic_dec_if_positive(&rba->req_ready) < 0) return; spin_lock(&rba->lock); for (i = 0; i < RX_CLAIM_REQ_ALLOC; i++) { /* Get next free Rx buffer, remove it from free list */ struct iwl_rx_mem_buffer *rxb = list_first_entry(&rba->rbd_allocated, struct iwl_rx_mem_buffer, list); list_move(&rxb->list, &rxq->rx_free); } spin_unlock(&rba->lock); rxq->used_count -= RX_CLAIM_REQ_ALLOC; rxq->free_count += RX_CLAIM_REQ_ALLOC; } void iwl_pcie_rx_allocator_work(struct work_struct *data) { struct iwl_rb_allocator *rba_p = container_of(data, struct iwl_rb_allocator, rx_alloc); struct iwl_trans_pcie *trans_pcie = container_of(rba_p, struct iwl_trans_pcie, rba); iwl_pcie_rx_allocator(trans_pcie->trans); } static int iwl_pcie_free_bd_size(struct iwl_trans *trans) { if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) return sizeof(struct iwl_rx_transfer_desc); return trans->trans_cfg->mq_rx_supported ? sizeof(__le64) : sizeof(__le32); } static int iwl_pcie_used_bd_size(struct iwl_trans *trans) { if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) return sizeof(struct iwl_rx_completion_desc_bz); if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) return sizeof(struct iwl_rx_completion_desc); return sizeof(__le32); } static void iwl_pcie_free_rxq_dma(struct iwl_trans *trans, struct iwl_rxq *rxq) { int free_size = iwl_pcie_free_bd_size(trans); if (rxq->bd) dma_free_coherent(trans->dev, free_size * rxq->queue_size, rxq->bd, rxq->bd_dma); rxq->bd_dma = 0; rxq->bd = NULL; rxq->rb_stts_dma = 0; rxq->rb_stts = NULL; if (rxq->used_bd) dma_free_coherent(trans->dev, iwl_pcie_used_bd_size(trans) * rxq->queue_size, rxq->used_bd, rxq->used_bd_dma); rxq->used_bd_dma = 0; rxq->used_bd = NULL; } static int iwl_pcie_alloc_rxq_dma(struct iwl_trans *trans, struct iwl_rxq *rxq) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct device *dev = trans->dev; int i; int free_size; bool use_rx_td = (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210); size_t rb_stts_size = use_rx_td ? sizeof(__le16) : sizeof(struct iwl_rb_status); spin_lock_init(&rxq->lock); if (trans->trans_cfg->mq_rx_supported) rxq->queue_size = trans->cfg->num_rbds; else rxq->queue_size = RX_QUEUE_SIZE; free_size = iwl_pcie_free_bd_size(trans); /* * Allocate the circular buffer of Read Buffer Descriptors * (RBDs) */ rxq->bd = dma_alloc_coherent(dev, free_size * rxq->queue_size, &rxq->bd_dma, GFP_KERNEL); if (!rxq->bd) goto err; if (trans->trans_cfg->mq_rx_supported) { rxq->used_bd = dma_alloc_coherent(dev, iwl_pcie_used_bd_size(trans) * rxq->queue_size, &rxq->used_bd_dma, GFP_KERNEL); if (!rxq->used_bd) goto err; } rxq->rb_stts = (u8 *)trans_pcie->base_rb_stts + rxq->id * rb_stts_size; rxq->rb_stts_dma = trans_pcie->base_rb_stts_dma + rxq->id * rb_stts_size; return 0; err: for (i = 0; i < trans->num_rx_queues; i++) { struct iwl_rxq *rxq = &trans_pcie->rxq[i]; iwl_pcie_free_rxq_dma(trans, rxq); } return -ENOMEM; } static int iwl_pcie_rx_alloc(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rb_allocator *rba = &trans_pcie->rba; int i, ret; size_t rb_stts_size = trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210 ? sizeof(__le16) : sizeof(struct iwl_rb_status); if (WARN_ON(trans_pcie->rxq)) return -EINVAL; trans_pcie->rxq = kcalloc(trans->num_rx_queues, sizeof(struct iwl_rxq), GFP_KERNEL); trans_pcie->rx_pool = kcalloc(RX_POOL_SIZE(trans_pcie->num_rx_bufs), sizeof(trans_pcie->rx_pool[0]), GFP_KERNEL); trans_pcie->global_table = kcalloc(RX_POOL_SIZE(trans_pcie->num_rx_bufs), sizeof(trans_pcie->global_table[0]), GFP_KERNEL); if (!trans_pcie->rxq || !trans_pcie->rx_pool || !trans_pcie->global_table) { ret = -ENOMEM; goto err; } spin_lock_init(&rba->lock); /* * Allocate the driver's pointer to receive buffer status. * Allocate for all queues continuously (HW requirement). */ trans_pcie->base_rb_stts = dma_alloc_coherent(trans->dev, rb_stts_size * trans->num_rx_queues, &trans_pcie->base_rb_stts_dma, GFP_KERNEL); if (!trans_pcie->base_rb_stts) { ret = -ENOMEM; goto err; } for (i = 0; i < trans->num_rx_queues; i++) { struct iwl_rxq *rxq = &trans_pcie->rxq[i]; rxq->id = i; ret = iwl_pcie_alloc_rxq_dma(trans, rxq); if (ret) goto err; } return 0; err: if (trans_pcie->base_rb_stts) { dma_free_coherent(trans->dev, rb_stts_size * trans->num_rx_queues, trans_pcie->base_rb_stts, trans_pcie->base_rb_stts_dma); trans_pcie->base_rb_stts = NULL; trans_pcie->base_rb_stts_dma = 0; } kfree(trans_pcie->rx_pool); trans_pcie->rx_pool = NULL; kfree(trans_pcie->global_table); trans_pcie->global_table = NULL; kfree(trans_pcie->rxq); trans_pcie->rxq = NULL; return ret; } static void iwl_pcie_rx_hw_init(struct iwl_trans *trans, struct iwl_rxq *rxq) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); u32 rb_size; const u32 rfdnlog = RX_QUEUE_SIZE_LOG; /* 256 RBDs */ switch (trans_pcie->rx_buf_size) { case IWL_AMSDU_4K: rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K; break; case IWL_AMSDU_8K: rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_8K; break; case IWL_AMSDU_12K: rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_12K; break; default: WARN_ON(1); rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K; } if (!iwl_trans_grab_nic_access(trans)) return; /* Stop Rx DMA */ iwl_write32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0); /* reset and flush pointers */ iwl_write32(trans, FH_MEM_RCSR_CHNL0_RBDCB_WPTR, 0); iwl_write32(trans, FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ, 0); iwl_write32(trans, FH_RSCSR_CHNL0_RDPTR, 0); /* Reset driver's Rx queue write index */ iwl_write32(trans, FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0); /* Tell device where to find RBD circular buffer in DRAM */ iwl_write32(trans, FH_RSCSR_CHNL0_RBDCB_BASE_REG, (u32)(rxq->bd_dma >> 8)); /* Tell device where in DRAM to update its Rx status */ iwl_write32(trans, FH_RSCSR_CHNL0_STTS_WPTR_REG, rxq->rb_stts_dma >> 4); /* Enable Rx DMA * FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY is set because of HW bug in * the credit mechanism in 5000 HW RX FIFO * Direct rx interrupts to hosts * Rx buffer size 4 or 8k or 12k * RB timeout 0x10 * 256 RBDs */ iwl_write32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL | FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY | FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL | rb_size | (RX_RB_TIMEOUT << FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) | (rfdnlog << FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS)); iwl_trans_release_nic_access(trans); /* Set interrupt coalescing timer to default (2048 usecs) */ iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF); /* W/A for interrupt coalescing bug in 7260 and 3160 */ if (trans->cfg->host_interrupt_operation_mode) iwl_set_bit(trans, CSR_INT_COALESCING, IWL_HOST_INT_OPER_MODE); } static void iwl_pcie_rx_mq_hw_init(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); u32 rb_size, enabled = 0; int i; switch (trans_pcie->rx_buf_size) { case IWL_AMSDU_2K: rb_size = RFH_RXF_DMA_RB_SIZE_2K; break; case IWL_AMSDU_4K: rb_size = RFH_RXF_DMA_RB_SIZE_4K; break; case IWL_AMSDU_8K: rb_size = RFH_RXF_DMA_RB_SIZE_8K; break; case IWL_AMSDU_12K: rb_size = RFH_RXF_DMA_RB_SIZE_12K; break; default: WARN_ON(1); rb_size = RFH_RXF_DMA_RB_SIZE_4K; } if (!iwl_trans_grab_nic_access(trans)) return; /* Stop Rx DMA */ iwl_write_prph_no_grab(trans, RFH_RXF_DMA_CFG, 0); /* disable free amd used rx queue operation */ iwl_write_prph_no_grab(trans, RFH_RXF_RXQ_ACTIVE, 0); for (i = 0; i < trans->num_rx_queues; i++) { /* Tell device where to find RBD free table in DRAM */ iwl_write_prph64_no_grab(trans, RFH_Q_FRBDCB_BA_LSB(i), trans_pcie->rxq[i].bd_dma); /* Tell device where to find RBD used table in DRAM */ iwl_write_prph64_no_grab(trans, RFH_Q_URBDCB_BA_LSB(i), trans_pcie->rxq[i].used_bd_dma); /* Tell device where in DRAM to update its Rx status */ iwl_write_prph64_no_grab(trans, RFH_Q_URBD_STTS_WPTR_LSB(i), trans_pcie->rxq[i].rb_stts_dma); /* Reset device indice tables */ iwl_write_prph_no_grab(trans, RFH_Q_FRBDCB_WIDX(i), 0); iwl_write_prph_no_grab(trans, RFH_Q_FRBDCB_RIDX(i), 0); iwl_write_prph_no_grab(trans, RFH_Q_URBDCB_WIDX(i), 0); enabled |= BIT(i) | BIT(i + 16); } /* * Enable Rx DMA * Rx buffer size 4 or 8k or 12k * Min RB size 4 or 8 * Drop frames that exceed RB size * 512 RBDs */ iwl_write_prph_no_grab(trans, RFH_RXF_DMA_CFG, RFH_DMA_EN_ENABLE_VAL | rb_size | RFH_RXF_DMA_MIN_RB_4_8 | RFH_RXF_DMA_DROP_TOO_LARGE_MASK | RFH_RXF_DMA_RBDCB_SIZE_512); /* * Activate DMA snooping. * Set RX DMA chunk size to 64B for IOSF and 128B for PCIe * Default queue is 0 */ iwl_write_prph_no_grab(trans, RFH_GEN_CFG, RFH_GEN_CFG_RFH_DMA_SNOOP | RFH_GEN_CFG_VAL(DEFAULT_RXQ_NUM, 0) | RFH_GEN_CFG_SERVICE_DMA_SNOOP | RFH_GEN_CFG_VAL(RB_CHUNK_SIZE, trans->trans_cfg->integrated ? RFH_GEN_CFG_RB_CHUNK_SIZE_64 : RFH_GEN_CFG_RB_CHUNK_SIZE_128)); /* Enable the relevant rx queues */ iwl_write_prph_no_grab(trans, RFH_RXF_RXQ_ACTIVE, enabled); iwl_trans_release_nic_access(trans); /* Set interrupt coalescing timer to default (2048 usecs) */ iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF); } void iwl_pcie_rx_init_rxb_lists(struct iwl_rxq *rxq) { lockdep_assert_held(&rxq->lock); INIT_LIST_HEAD(&rxq->rx_free); INIT_LIST_HEAD(&rxq->rx_used); rxq->free_count = 0; rxq->used_count = 0; } static int iwl_pcie_rx_handle(struct iwl_trans *trans, int queue, int budget); static int iwl_pcie_napi_poll(struct napi_struct *napi, int budget) { struct iwl_rxq *rxq = container_of(napi, struct iwl_rxq, napi); struct iwl_trans_pcie *trans_pcie; struct iwl_trans *trans; int ret; trans_pcie = container_of(napi->dev, struct iwl_trans_pcie, napi_dev); trans = trans_pcie->trans; ret = iwl_pcie_rx_handle(trans, rxq->id, budget); IWL_DEBUG_ISR(trans, "[%d] handled %d, budget %d\n", rxq->id, ret, budget); if (ret < budget) { spin_lock(&trans_pcie->irq_lock); if (test_bit(STATUS_INT_ENABLED, &trans->status)) _iwl_enable_interrupts(trans); spin_unlock(&trans_pcie->irq_lock); napi_complete_done(&rxq->napi, ret); } return ret; } static int iwl_pcie_napi_poll_msix(struct napi_struct *napi, int budget) { struct iwl_rxq *rxq = container_of(napi, struct iwl_rxq, napi); struct iwl_trans_pcie *trans_pcie; struct iwl_trans *trans; int ret; trans_pcie = container_of(napi->dev, struct iwl_trans_pcie, napi_dev); trans = trans_pcie->trans; ret = iwl_pcie_rx_handle(trans, rxq->id, budget); IWL_DEBUG_ISR(trans, "[%d] handled %d, budget %d\n", rxq->id, ret, budget); if (ret < budget) { int irq_line = rxq->id; /* FIRST_RSS is shared with line 0 */ if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS && rxq->id == 1) irq_line = 0; spin_lock(&trans_pcie->irq_lock); iwl_pcie_clear_irq(trans, irq_line); spin_unlock(&trans_pcie->irq_lock); napi_complete_done(&rxq->napi, ret); } return ret; } static int _iwl_pcie_rx_init(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rxq *def_rxq; struct iwl_rb_allocator *rba = &trans_pcie->rba; int i, err, queue_size, allocator_pool_size, num_alloc; if (!trans_pcie->rxq) { err = iwl_pcie_rx_alloc(trans); if (err) return err; } def_rxq = trans_pcie->rxq; cancel_work_sync(&rba->rx_alloc); spin_lock_bh(&rba->lock); atomic_set(&rba->req_pending, 0); atomic_set(&rba->req_ready, 0); INIT_LIST_HEAD(&rba->rbd_allocated); INIT_LIST_HEAD(&rba->rbd_empty); spin_unlock_bh(&rba->lock); /* free all first - we overwrite everything here */ iwl_pcie_free_rbs_pool(trans); for (i = 0; i < RX_QUEUE_SIZE; i++) def_rxq->queue[i] = NULL; for (i = 0; i < trans->num_rx_queues; i++) { struct iwl_rxq *rxq = &trans_pcie->rxq[i]; spin_lock_bh(&rxq->lock); /* * Set read write pointer to reflect that we have processed * and used all buffers, but have not restocked the Rx queue * with fresh buffers */ rxq->read = 0; rxq->write = 0; rxq->write_actual = 0; memset(rxq->rb_stts, 0, (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) ? sizeof(__le16) : sizeof(struct iwl_rb_status)); iwl_pcie_rx_init_rxb_lists(rxq); spin_unlock_bh(&rxq->lock); if (!rxq->napi.poll) { int (*poll)(struct napi_struct *, int) = iwl_pcie_napi_poll; if (trans_pcie->msix_enabled) poll = iwl_pcie_napi_poll_msix; - netif_napi_add(&trans_pcie->napi_dev, &rxq->napi, - poll, NAPI_POLL_WEIGHT); + netif_napi_add(&trans_pcie->napi_dev, &rxq->napi, poll); napi_enable(&rxq->napi); } } /* move the pool to the default queue and allocator ownerships */ queue_size = trans->trans_cfg->mq_rx_supported ? trans_pcie->num_rx_bufs - 1 : RX_QUEUE_SIZE; allocator_pool_size = trans->num_rx_queues * (RX_CLAIM_REQ_ALLOC - RX_POST_REQ_ALLOC); num_alloc = queue_size + allocator_pool_size; for (i = 0; i < num_alloc; i++) { struct iwl_rx_mem_buffer *rxb = &trans_pcie->rx_pool[i]; if (i < allocator_pool_size) list_add(&rxb->list, &rba->rbd_empty); else list_add(&rxb->list, &def_rxq->rx_used); trans_pcie->global_table[i] = rxb; rxb->vid = (u16)(i + 1); rxb->invalid = true; } iwl_pcie_rxq_alloc_rbs(trans, GFP_KERNEL, def_rxq); return 0; } int iwl_pcie_rx_init(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); int ret = _iwl_pcie_rx_init(trans); if (ret) return ret; if (trans->trans_cfg->mq_rx_supported) iwl_pcie_rx_mq_hw_init(trans); else iwl_pcie_rx_hw_init(trans, trans_pcie->rxq); iwl_pcie_rxq_restock(trans, trans_pcie->rxq); spin_lock_bh(&trans_pcie->rxq->lock); iwl_pcie_rxq_inc_wr_ptr(trans, trans_pcie->rxq); spin_unlock_bh(&trans_pcie->rxq->lock); return 0; } int iwl_pcie_gen2_rx_init(struct iwl_trans *trans) { /* Set interrupt coalescing timer to default (2048 usecs) */ iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF); /* * We don't configure the RFH. * Restock will be done at alive, after firmware configured the RFH. */ return _iwl_pcie_rx_init(trans); } void iwl_pcie_rx_free(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rb_allocator *rba = &trans_pcie->rba; int i; size_t rb_stts_size = trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210 ? sizeof(__le16) : sizeof(struct iwl_rb_status); /* * if rxq is NULL, it means that nothing has been allocated, * exit now */ if (!trans_pcie->rxq) { IWL_DEBUG_INFO(trans, "Free NULL rx context\n"); return; } cancel_work_sync(&rba->rx_alloc); iwl_pcie_free_rbs_pool(trans); if (trans_pcie->base_rb_stts) { dma_free_coherent(trans->dev, rb_stts_size * trans->num_rx_queues, trans_pcie->base_rb_stts, trans_pcie->base_rb_stts_dma); trans_pcie->base_rb_stts = NULL; trans_pcie->base_rb_stts_dma = 0; } for (i = 0; i < trans->num_rx_queues; i++) { struct iwl_rxq *rxq = &trans_pcie->rxq[i]; iwl_pcie_free_rxq_dma(trans, rxq); if (rxq->napi.poll) { napi_disable(&rxq->napi); netif_napi_del(&rxq->napi); } } kfree(trans_pcie->rx_pool); kfree(trans_pcie->global_table); kfree(trans_pcie->rxq); if (trans_pcie->alloc_page) __free_pages(trans_pcie->alloc_page, trans_pcie->rx_page_order); } static void iwl_pcie_rx_move_to_allocator(struct iwl_rxq *rxq, struct iwl_rb_allocator *rba) { spin_lock(&rba->lock); list_splice_tail_init(&rxq->rx_used, &rba->rbd_empty); spin_unlock(&rba->lock); } /* * iwl_pcie_rx_reuse_rbd - Recycle used RBDs * * Called when a RBD can be reused. The RBD is transferred to the allocator. * When there are 2 empty RBDs - a request for allocation is posted */ static void iwl_pcie_rx_reuse_rbd(struct iwl_trans *trans, struct iwl_rx_mem_buffer *rxb, struct iwl_rxq *rxq, bool emergency) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rb_allocator *rba = &trans_pcie->rba; /* Move the RBD to the used list, will be moved to allocator in batches * before claiming or posting a request*/ list_add_tail(&rxb->list, &rxq->rx_used); if (unlikely(emergency)) return; /* Count the allocator owned RBDs */ rxq->used_count++; /* If we have RX_POST_REQ_ALLOC new released rx buffers - * issue a request for allocator. Modulo RX_CLAIM_REQ_ALLOC is * used for the case we failed to claim RX_CLAIM_REQ_ALLOC, * after but we still need to post another request. */ if ((rxq->used_count % RX_CLAIM_REQ_ALLOC) == RX_POST_REQ_ALLOC) { /* Move the 2 RBDs to the allocator ownership. Allocator has another 6 from pool for the request completion*/ iwl_pcie_rx_move_to_allocator(rxq, rba); atomic_inc(&rba->req_pending); queue_work(rba->alloc_wq, &rba->rx_alloc); } } static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans, struct iwl_rxq *rxq, struct iwl_rx_mem_buffer *rxb, bool emergency, int i) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id]; bool page_stolen = false; int max_len = trans_pcie->rx_buf_bytes; u32 offset = 0; if (WARN_ON(!rxb)) return; dma_unmap_page(trans->dev, rxb->page_dma, max_len, DMA_FROM_DEVICE); while (offset + sizeof(u32) + sizeof(struct iwl_cmd_header) < max_len) { struct iwl_rx_packet *pkt; bool reclaim; int len; struct iwl_rx_cmd_buffer rxcb = { ._offset = rxb->offset + offset, ._rx_page_order = trans_pcie->rx_page_order, ._page = rxb->page, ._page_stolen = false, .truesize = max_len, }; pkt = rxb_addr(&rxcb); if (pkt->len_n_flags == cpu_to_le32(FH_RSCSR_FRAME_INVALID)) { IWL_DEBUG_RX(trans, "Q %d: RB end marker at offset %d\n", rxq->id, offset); break; } WARN((le32_to_cpu(pkt->len_n_flags) & FH_RSCSR_RXQ_MASK) >> FH_RSCSR_RXQ_POS != rxq->id, "frame on invalid queue - is on %d and indicates %d\n", rxq->id, (le32_to_cpu(pkt->len_n_flags) & FH_RSCSR_RXQ_MASK) >> FH_RSCSR_RXQ_POS); IWL_DEBUG_RX(trans, "Q %d: cmd at offset %d: %s (%.2x.%2x, seq 0x%x)\n", rxq->id, offset, iwl_get_cmd_string(trans, WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd)), pkt->hdr.group_id, pkt->hdr.cmd, le16_to_cpu(pkt->hdr.sequence)); len = iwl_rx_packet_len(pkt); len += sizeof(u32); /* account for status word */ offset += ALIGN(len, FH_RSCSR_FRAME_ALIGN); /* check that what the device tells us made sense */ if (len < sizeof(*pkt) || offset > max_len) break; trace_iwlwifi_dev_rx(trans->dev, trans, pkt, len); trace_iwlwifi_dev_rx_data(trans->dev, trans, pkt, len); /* Reclaim a command buffer only if this packet is a response * to a (driver-originated) command. * If the packet (e.g. Rx frame) originated from uCode, * there is no command buffer to reclaim. * Ucode should set SEQ_RX_FRAME bit if ucode-originated, * but apparently a few don't get set; catch them here. */ reclaim = !(pkt->hdr.sequence & SEQ_RX_FRAME); if (reclaim && !pkt->hdr.group_id) { int i; for (i = 0; i < trans_pcie->n_no_reclaim_cmds; i++) { if (trans_pcie->no_reclaim_cmds[i] == pkt->hdr.cmd) { reclaim = false; break; } } } if (rxq->id == trans_pcie->def_rx_queue) iwl_op_mode_rx(trans->op_mode, &rxq->napi, &rxcb); else iwl_op_mode_rx_rss(trans->op_mode, &rxq->napi, &rxcb, rxq->id); /* * After here, we should always check rxcb._page_stolen, * if it is true then one of the handlers took the page. */ if (reclaim) { u16 sequence = le16_to_cpu(pkt->hdr.sequence); int index = SEQ_TO_INDEX(sequence); int cmd_index = iwl_txq_get_cmd_index(txq, index); kfree_sensitive(txq->entries[cmd_index].free_buf); txq->entries[cmd_index].free_buf = NULL; /* Invoke any callbacks, transfer the buffer to caller, * and fire off the (possibly) blocking * iwl_trans_send_cmd() * as we reclaim the driver command queue */ if (!rxcb._page_stolen) iwl_pcie_hcmd_complete(trans, &rxcb); else IWL_WARN(trans, "Claim null rxb?\n"); } page_stolen |= rxcb._page_stolen; if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) break; } /* page was stolen from us -- free our reference */ if (page_stolen) { __free_pages(rxb->page, trans_pcie->rx_page_order); rxb->page = NULL; } /* Reuse the page if possible. For notification packets and * SKBs that fail to Rx correctly, add them back into the * rx_free list for reuse later. */ if (rxb->page != NULL) { rxb->page_dma = dma_map_page(trans->dev, rxb->page, rxb->offset, trans_pcie->rx_buf_bytes, DMA_FROM_DEVICE); if (dma_mapping_error(trans->dev, rxb->page_dma)) { /* * free the page(s) as well to not break * the invariant that the items on the used * list have no page(s) */ __free_pages(rxb->page, trans_pcie->rx_page_order); rxb->page = NULL; iwl_pcie_rx_reuse_rbd(trans, rxb, rxq, emergency); } else { list_add_tail(&rxb->list, &rxq->rx_free); rxq->free_count++; } } else iwl_pcie_rx_reuse_rbd(trans, rxb, rxq, emergency); } static struct iwl_rx_mem_buffer *iwl_pcie_get_rxb(struct iwl_trans *trans, struct iwl_rxq *rxq, int i, bool *join) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rx_mem_buffer *rxb; u16 vid; BUILD_BUG_ON(sizeof(struct iwl_rx_completion_desc) != 32); BUILD_BUG_ON(sizeof(struct iwl_rx_completion_desc_bz) != 4); if (!trans->trans_cfg->mq_rx_supported) { rxb = rxq->queue[i]; rxq->queue[i] = NULL; return rxb; } if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) { struct iwl_rx_completion_desc_bz *cd = rxq->used_bd; vid = le16_to_cpu(cd[i].rbid); *join = cd[i].flags & IWL_RX_CD_FLAGS_FRAGMENTED; } else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { struct iwl_rx_completion_desc *cd = rxq->used_bd; vid = le16_to_cpu(cd[i].rbid); *join = cd[i].flags & IWL_RX_CD_FLAGS_FRAGMENTED; } else { __le32 *cd = rxq->used_bd; vid = le32_to_cpu(cd[i]) & 0x0FFF; /* 12-bit VID */ } if (!vid || vid > RX_POOL_SIZE(trans_pcie->num_rx_bufs)) goto out_err; rxb = trans_pcie->global_table[vid - 1]; if (rxb->invalid) goto out_err; IWL_DEBUG_RX(trans, "Got virtual RB ID %u\n", (u32)rxb->vid); rxb->invalid = true; return rxb; out_err: WARN(1, "Invalid rxb from HW %u\n", (u32)vid); iwl_force_nmi(trans); return NULL; } /* * iwl_pcie_rx_handle - Main entry function for receiving responses from fw */ static int iwl_pcie_rx_handle(struct iwl_trans *trans, int queue, int budget) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct iwl_rxq *rxq; u32 r, i, count = 0, handled = 0; bool emergency = false; if (WARN_ON_ONCE(!trans_pcie->rxq || !trans_pcie->rxq[queue].bd)) return budget; rxq = &trans_pcie->rxq[queue]; restart: spin_lock(&rxq->lock); /* uCode's read index (stored in shared DRAM) indicates the last Rx * buffer that the driver may process (last buffer filled by ucode). */ r = le16_to_cpu(iwl_get_closed_rb_stts(trans, rxq)) & 0x0FFF; i = rxq->read; /* W/A 9000 device step A0 wrap-around bug */ r &= (rxq->queue_size - 1); /* Rx interrupt, but nothing sent from uCode */ if (i == r) IWL_DEBUG_RX(trans, "Q %d: HW = SW = %d\n", rxq->id, r); while (i != r && ++handled < budget) { struct iwl_rb_allocator *rba = &trans_pcie->rba; struct iwl_rx_mem_buffer *rxb; /* number of RBDs still waiting for page allocation */ u32 rb_pending_alloc = atomic_read(&trans_pcie->rba.req_pending) * RX_CLAIM_REQ_ALLOC; bool join = false; if (unlikely(rb_pending_alloc >= rxq->queue_size / 2 && !emergency)) { iwl_pcie_rx_move_to_allocator(rxq, rba); emergency = true; IWL_DEBUG_TPT(trans, "RX path is in emergency. Pending allocations %d\n", rb_pending_alloc); } IWL_DEBUG_RX(trans, "Q %d: HW = %d, SW = %d\n", rxq->id, r, i); rxb = iwl_pcie_get_rxb(trans, rxq, i, &join); if (!rxb) goto out; if (unlikely(join || rxq->next_rb_is_fragment)) { rxq->next_rb_is_fragment = join; /* * We can only get a multi-RB in the following cases: * - firmware issue, sending a too big notification * - sniffer mode with a large A-MSDU * - large MTU frames (>2k) * since the multi-RB functionality is limited to newer * hardware that cannot put multiple entries into a * single RB. * * Right now, the higher layers aren't set up to deal * with that, so discard all of these. */ list_add_tail(&rxb->list, &rxq->rx_free); rxq->free_count++; } else { iwl_pcie_rx_handle_rb(trans, rxq, rxb, emergency, i); } i = (i + 1) & (rxq->queue_size - 1); /* * If we have RX_CLAIM_REQ_ALLOC released rx buffers - * try to claim the pre-allocated buffers from the allocator. * If not ready - will try to reclaim next time. * There is no need to reschedule work - allocator exits only * on success */ if (rxq->used_count >= RX_CLAIM_REQ_ALLOC) iwl_pcie_rx_allocator_get(trans, rxq); if (rxq->used_count % RX_CLAIM_REQ_ALLOC == 0 && !emergency) { /* Add the remaining empty RBDs for allocator use */ iwl_pcie_rx_move_to_allocator(rxq, rba); } else if (emergency) { count++; if (count == 8) { count = 0; if (rb_pending_alloc < rxq->queue_size / 3) { IWL_DEBUG_TPT(trans, "RX path exited emergency. Pending allocations %d\n", rb_pending_alloc); emergency = false; } rxq->read = i; spin_unlock(&rxq->lock); iwl_pcie_rxq_alloc_rbs(trans, GFP_ATOMIC, rxq); iwl_pcie_rxq_restock(trans, rxq); goto restart; } } } out: /* Backtrack one entry */ rxq->read = i; spin_unlock(&rxq->lock); /* * handle a case where in emergency there are some unallocated RBDs. * those RBDs are in the used list, but are not tracked by the queue's * used_count which counts allocator owned RBDs. * unallocated emergency RBDs must be allocated on exit, otherwise * when called again the function may not be in emergency mode and * they will be handed to the allocator with no tracking in the RBD * allocator counters, which will lead to them never being claimed back * by the queue. * by allocating them here, they are now in the queue free list, and * will be restocked by the next call of iwl_pcie_rxq_restock. */ if (unlikely(emergency && count)) iwl_pcie_rxq_alloc_rbs(trans, GFP_ATOMIC, rxq); iwl_pcie_rxq_restock(trans, rxq); return handled; } static struct iwl_trans_pcie *iwl_pcie_get_trans_pcie(struct msix_entry *entry) { u8 queue = entry->entry; struct msix_entry *entries = entry - queue; return container_of(entries, struct iwl_trans_pcie, msix_entries[0]); } /* * iwl_pcie_rx_msix_handle - Main entry function for receiving responses from fw * This interrupt handler should be used with RSS queue only. */ irqreturn_t iwl_pcie_irq_rx_msix_handler(int irq, void *dev_id) { struct msix_entry *entry = dev_id; struct iwl_trans_pcie *trans_pcie = iwl_pcie_get_trans_pcie(entry); struct iwl_trans *trans = trans_pcie->trans; struct iwl_rxq *rxq = &trans_pcie->rxq[entry->entry]; trace_iwlwifi_dev_irq_msix(trans->dev, entry, false, 0, 0); if (WARN_ON(entry->entry >= trans->num_rx_queues)) return IRQ_NONE; if (!rxq) { if (net_ratelimit()) IWL_ERR(trans, "[%d] Got MSI-X interrupt before we have Rx queues\n", entry->entry); return IRQ_NONE; } lock_map_acquire(&trans->sync_cmd_lockdep_map); IWL_DEBUG_ISR(trans, "[%d] Got interrupt\n", entry->entry); local_bh_disable(); if (napi_schedule_prep(&rxq->napi)) __napi_schedule(&rxq->napi); else iwl_pcie_clear_irq(trans, entry->entry); local_bh_enable(); lock_map_release(&trans->sync_cmd_lockdep_map); return IRQ_HANDLED; } /* * iwl_pcie_irq_handle_error - called for HW or SW error interrupt from card */ static void iwl_pcie_irq_handle_error(struct iwl_trans *trans) { int i; /* W/A for WiFi/WiMAX coex and WiMAX own the RF */ if (trans->cfg->internal_wimax_coex && !trans->cfg->apmg_not_supported && (!(iwl_read_prph(trans, APMG_CLK_CTRL_REG) & APMS_CLK_VAL_MRB_FUNC_MODE) || (iwl_read_prph(trans, APMG_PS_CTRL_REG) & APMG_PS_CTRL_VAL_RESET_REQ))) { clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); iwl_op_mode_wimax_active(trans->op_mode); wake_up(&trans->wait_command_queue); return; } for (i = 0; i < trans->trans_cfg->base_params->num_of_queues; i++) { if (!trans->txqs.txq[i]) continue; del_timer(&trans->txqs.txq[i]->stuck_timer); } /* The STATUS_FW_ERROR bit is set in this function. This must happen * before we wake up the command caller, to ensure a proper cleanup. */ iwl_trans_fw_error(trans, false); clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); wake_up(&trans->wait_command_queue); } static u32 iwl_pcie_int_cause_non_ict(struct iwl_trans *trans) { u32 inta; lockdep_assert_held(&IWL_TRANS_GET_PCIE_TRANS(trans)->irq_lock); trace_iwlwifi_dev_irq(trans->dev); /* Discover which interrupts are active/pending */ inta = iwl_read32(trans, CSR_INT); /* the thread will service interrupts and re-enable them */ return inta; } /* a device (PCI-E) page is 4096 bytes long */ #define ICT_SHIFT 12 #define ICT_SIZE (1 << ICT_SHIFT) #define ICT_COUNT (ICT_SIZE / sizeof(u32)) /* interrupt handler using ict table, with this interrupt driver will * stop using INTA register to get device's interrupt, reading this register * is expensive, device will write interrupts in ICT dram table, increment * index then will fire interrupt to driver, driver will OR all ICT table * entries from current index up to table entry with 0 value. the result is * the interrupt we need to service, driver will set the entries back to 0 and * set index. */ static u32 iwl_pcie_int_cause_ict(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); u32 inta; u32 val = 0; u32 read; trace_iwlwifi_dev_irq(trans->dev); /* Ignore interrupt if there's nothing in NIC to service. * This may be due to IRQ shared with another device, * or due to sporadic interrupts thrown from our NIC. */ read = le32_to_cpu(trans_pcie->ict_tbl[trans_pcie->ict_index]); trace_iwlwifi_dev_ict_read(trans->dev, trans_pcie->ict_index, read); if (!read) return 0; /* * Collect all entries up to the first 0, starting from ict_index; * note we already read at ict_index. */ do { val |= read; IWL_DEBUG_ISR(trans, "ICT index %d value 0x%08X\n", trans_pcie->ict_index, read); trans_pcie->ict_tbl[trans_pcie->ict_index] = 0; trans_pcie->ict_index = ((trans_pcie->ict_index + 1) & (ICT_COUNT - 1)); read = le32_to_cpu(trans_pcie->ict_tbl[trans_pcie->ict_index]); trace_iwlwifi_dev_ict_read(trans->dev, trans_pcie->ict_index, read); } while (read); /* We should not get this value, just ignore it. */ if (val == 0xffffffff) val = 0; /* * this is a w/a for a h/w bug. the h/w bug may cause the Rx bit * (bit 15 before shifting it to 31) to clear when using interrupt * coalescing. fortunately, bits 18 and 19 stay set when this happens * so we use them to decide on the real state of the Rx bit. * In order words, bit 15 is set if bit 18 or bit 19 are set. */ if (val & 0xC0000) val |= 0x8000; inta = (0xff & val) | ((0xff00 & val) << 16); return inta; } void iwl_pcie_handle_rfkill_irq(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct isr_statistics *isr_stats = &trans_pcie->isr_stats; bool hw_rfkill, prev, report; mutex_lock(&trans_pcie->mutex); prev = test_bit(STATUS_RFKILL_OPMODE, &trans->status); hw_rfkill = iwl_is_rfkill_set(trans); if (hw_rfkill) { set_bit(STATUS_RFKILL_OPMODE, &trans->status); set_bit(STATUS_RFKILL_HW, &trans->status); } if (trans_pcie->opmode_down) report = hw_rfkill; else report = test_bit(STATUS_RFKILL_OPMODE, &trans->status); IWL_WARN(trans, "RF_KILL bit toggled to %s.\n", hw_rfkill ? "disable radio" : "enable radio"); isr_stats->rfkill++; if (prev != report) iwl_trans_pcie_rf_kill(trans, report); mutex_unlock(&trans_pcie->mutex); if (hw_rfkill) { if (test_and_clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status)) IWL_DEBUG_RF_KILL(trans, "Rfkill while SYNC HCMD in flight\n"); wake_up(&trans->wait_command_queue); } else { clear_bit(STATUS_RFKILL_HW, &trans->status); if (trans_pcie->opmode_down) clear_bit(STATUS_RFKILL_OPMODE, &trans->status); } } irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id) { struct iwl_trans *trans = dev_id; struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); struct isr_statistics *isr_stats = &trans_pcie->isr_stats; u32 inta = 0; u32 handled = 0; bool polling = false; lock_map_acquire(&trans->sync_cmd_lockdep_map); spin_lock_bh(&trans_pcie->irq_lock); /* dram interrupt table not set yet, * use legacy interrupt. */ if (likely(trans_pcie->use_ict)) inta = iwl_pcie_int_cause_ict(trans); else inta = iwl_pcie_int_cause_non_ict(trans); #ifdef CONFIG_IWLWIFI_DEBUG if (iwl_have_debug_level(IWL_DL_ISR)) { IWL_DEBUG_ISR(trans, "ISR inta 0x%08x, enabled 0x%08x(sw), enabled(hw) 0x%08x, fh 0x%08x\n", inta, trans_pcie->inta_mask, iwl_read32(trans, CSR_INT_MASK), iwl_read32(trans, CSR_FH_INT_STATUS)); if (inta & (~trans_pcie->inta_mask)) IWL_DEBUG_ISR(trans, "We got a masked interrupt (0x%08x)\n", inta & (~trans_pcie->inta_mask)); } #endif inta &= trans_pcie->inta_mask; /* * Ignore interrupt if there's nothing in NIC to service. * This may be due to IRQ shared with another device, * or due to sporadic interrupts thrown from our NIC. */ if (unlikely(!inta)) { IWL_DEBUG_ISR(trans, "Ignore interrupt, inta == 0\n"); /* * Re-enable interrupts here since we don't * have anything to service */ if (test_bit(STATUS_INT_ENABLED, &trans->status)) _iwl_enable_interrupts(trans); spin_unlock_bh(&trans_pcie->irq_lock); lock_map_release(&trans->sync_cmd_lockdep_map); return IRQ_NONE; } if (unlikely(inta == 0xFFFFFFFF || (inta & 0xFFFFFFF0) == 0xa5a5a5a0)) { /* * Hardware disappeared. It might have * already raised an interrupt. */ IWL_WARN(trans, "HARDWARE GONE?? INTA == 0x%08x\n", inta); spin_unlock_bh(&trans_pcie->irq_lock); goto out; } /* Ack/clear/reset pending uCode interrupts. * Note: Some bits in CSR_INT are "OR" of bits in CSR_FH_INT_STATUS, */ /* There is a hardware bug in the interrupt mask function that some * interrupts (i.e. CSR_INT_BIT_SCD) can still be generated even if * they are disabled in the CSR_INT_MASK register. Furthermore the * ICT interrupt handling mechanism has another bug that might cause * these unmasked interrupts fail to be detected. We workaround the * hardware bugs here by ACKing all the possible interrupts so that * interrupt coalescing can still be achieved. */ iwl_write32(trans, CSR_INT, inta | ~trans_pcie->inta_mask); #ifdef CONFIG_IWLWIFI_DEBUG if (iwl_have_debug_level(IWL_DL_ISR)) IWL_DEBUG_ISR(trans, "inta 0x%08x, enabled 0x%08x\n", inta, iwl_read32(trans, CSR_INT_MASK)); #endif spin_unlock_bh(&trans_pcie->irq_lock); /* Now service all interrupt bits discovered above. */ if (inta & CSR_INT_BIT_HW_ERR) { IWL_ERR(trans, "Hardware error detected. Restarting.\n"); /* Tell the device to stop sending interrupts */ iwl_disable_interrupts(trans); isr_stats->hw++; iwl_pcie_irq_handle_error(trans); handled |= CSR_INT_BIT_HW_ERR; goto out; } /* NIC fires this, but we don't use it, redundant with WAKEUP */ if (inta & CSR_INT_BIT_SCD) { IWL_DEBUG_ISR(trans, "Scheduler finished to transmit the frame/frames.\n"); isr_stats->sch++; } /* Alive notification via Rx interrupt will do the real work */ if (inta & CSR_INT_BIT_ALIVE) { IWL_DEBUG_ISR(trans, "Alive interrupt\n"); isr_stats->alive++; if (trans->trans_cfg->gen2) { /* * We can restock, since firmware configured * the RFH */ iwl_pcie_rxmq_restock(trans, trans_pcie->rxq); } handled |= CSR_INT_BIT_ALIVE; } /* Safely ignore these bits for debug checks below */ inta &= ~(CSR_INT_BIT_SCD | CSR_INT_BIT_ALIVE); /* HW RF KILL switch toggled */ if (inta & CSR_INT_BIT_RF_KILL) { iwl_pcie_handle_rfkill_irq(trans); handled |= CSR_INT_BIT_RF_KILL; } /* Chip got too hot and stopped itself */ if (inta & CSR_INT_BIT_CT_KILL) { IWL_ERR(trans, "Microcode CT kill error detected.\n"); isr_stats->ctkill++; handled |= CSR_INT_BIT_CT_KILL; } /* Error detected by uCode */ if (inta & CSR_INT_BIT_SW_ERR) { IWL_ERR(trans, "Microcode SW error detected. " " Restarting 0x%X.\n", inta); isr_stats->sw++; iwl_pcie_irq_handle_error(trans); handled |= CSR_INT_BIT_SW_ERR; } /* uCode wakes up after power-down sleep */ if (inta & CSR_INT_BIT_WAKEUP) { IWL_DEBUG_ISR(trans, "Wakeup interrupt\n"); iwl_pcie_rxq_check_wrptr(trans); iwl_pcie_txq_check_wrptrs(trans); isr_stats->wakeup++; handled |= CSR_INT_BIT_WAKEUP; } /* All uCode command responses, including Tx command responses, * Rx "responses" (frame-received notification), and other * notifications from uCode come through here*/ if (inta & (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX | CSR_INT_BIT_RX_PERIODIC)) { IWL_DEBUG_ISR(trans, "Rx interrupt\n"); if (inta & (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX)) { handled |= (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX); iwl_write32(trans, CSR_FH_INT_STATUS, CSR_FH_INT_RX_MASK); } if (inta & CSR_INT_BIT_RX_PERIODIC) { handled |= CSR_INT_BIT_RX_PERIODIC; iwl_write32(trans, CSR_INT, CSR_INT_BIT_RX_PERIODIC); } /* Sending RX interrupt require many steps to be done in the * device: * 1- write interrupt to current index in ICT table. * 2- dma RX frame. * 3- update RX shared data to indicate last write index. * 4- send interrupt. * This could lead to RX race, driver could receive RX interrupt * but the shared data changes does not reflect this; * periodic interrupt will detect any dangling Rx activity. */ /* Disable periodic interrupt; we use it as just a one-shot. */ iwl_write8(trans, CSR_INT_PERIODIC_REG, CSR_INT_PERIODIC_DIS); /* * Enable periodic interrupt in 8 msec only if we received * real RX interrupt (instead of just periodic int), to catch * any dangling Rx interrupt. If it was just the periodic * interrupt, there was no dangling Rx activity, and no need * to extend the periodic interrupt; one-shot is enough. */ if (inta & (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX)) iwl_write8(trans, CSR_INT_PERIODIC_REG, CSR_INT_PERIODIC_ENA); isr_stats->rx++; local_bh_disable(); if (napi_schedule_prep(&trans_pcie->rxq[0].napi)) { polling = true; __napi_schedule(&trans_pcie->rxq[0].napi); } local_bh_enable(); } /* This "Tx" DMA channel is used only for loading uCode */ if (inta & CSR_INT_BIT_FH_TX) { iwl_write32(trans, CSR_FH_INT_STATUS, CSR_FH_INT_TX_MASK); IWL_DEBUG_ISR(trans, "uCode load interrupt\n"); isr_stats->tx++; handled |= CSR_INT_BIT_FH_TX; /* Wake up uCode load routine, now that load is complete */ trans_pcie->ucode_write_complete = true; wake_up(&trans_pcie->ucode_write_waitq); /* Wake up IMR write routine, now that write to SRAM is complete */ if (trans_pcie->imr_status == IMR_D2S_REQUESTED) { trans_pcie->imr_status = IMR_D2S_COMPLETED; wake_up(&trans_pcie->ucode_write_waitq); } } if (inta & ~handled) { IWL_ERR(trans, "Unhandled INTA bits 0x%08x\n", inta & ~handled); isr_stats->unhandled++; } if (inta & ~(trans_pcie->inta_mask)) { IWL_WARN(trans, "Disabled INTA bits 0x%08x were pending\n", inta & ~trans_pcie->inta_mask); } if (!polling) { spin_lock_bh(&trans_pcie->irq_lock); /* only Re-enable all interrupt if disabled by irq */ if (test_bit(STATUS_INT_ENABLED, &trans->status)) _iwl_enable_interrupts(trans); /* we are loading the firmware, enable FH_TX interrupt only */ else if (handled & CSR_INT_BIT_FH_TX) iwl_enable_fw_load_int(trans); /* Re-enable RF_KILL if it occurred */ else if (handled & CSR_INT_BIT_RF_KILL) iwl_enable_rfkill_int(trans); /* Re-enable the ALIVE / Rx interrupt if it occurred */ else if (handled & (CSR_INT_BIT_ALIVE | CSR_INT_BIT_FH_RX)) iwl_enable_fw_load_int_ctx_info(trans); spin_unlock_bh(&trans_pcie->irq_lock); } out: lock_map_release(&trans->sync_cmd_lockdep_map); return IRQ_HANDLED; } /****************************************************************************** * * ICT functions * ******************************************************************************/ /* Free dram table */ void iwl_pcie_free_ict(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); if (trans_pcie->ict_tbl) { dma_free_coherent(trans->dev, ICT_SIZE, trans_pcie->ict_tbl, trans_pcie->ict_tbl_dma); trans_pcie->ict_tbl = NULL; trans_pcie->ict_tbl_dma = 0; } } /* * allocate dram shared table, it is an aligned memory * block of ICT_SIZE. * also reset all data related to ICT table interrupt. */ int iwl_pcie_alloc_ict(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); trans_pcie->ict_tbl = dma_alloc_coherent(trans->dev, ICT_SIZE, &trans_pcie->ict_tbl_dma, GFP_KERNEL); if (!trans_pcie->ict_tbl) return -ENOMEM; /* just an API sanity check ... it is guaranteed to be aligned */ if (WARN_ON(trans_pcie->ict_tbl_dma & (ICT_SIZE - 1))) { iwl_pcie_free_ict(trans); return -EINVAL; } return 0; } /* Device is going up inform it about using ICT interrupt table, * also we need to tell the driver to start using ICT interrupt. */ void iwl_pcie_reset_ict(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); u32 val; if (!trans_pcie->ict_tbl) return; spin_lock_bh(&trans_pcie->irq_lock); _iwl_disable_interrupts(trans); memset(trans_pcie->ict_tbl, 0, ICT_SIZE); val = trans_pcie->ict_tbl_dma >> ICT_SHIFT; val |= CSR_DRAM_INT_TBL_ENABLE | CSR_DRAM_INIT_TBL_WRAP_CHECK | CSR_DRAM_INIT_TBL_WRITE_POINTER; IWL_DEBUG_ISR(trans, "CSR_DRAM_INT_TBL_REG =0x%x\n", val); iwl_write32(trans, CSR_DRAM_INT_TBL_REG, val); trans_pcie->use_ict = true; trans_pcie->ict_index = 0; iwl_write32(trans, CSR_INT, trans_pcie->inta_mask); _iwl_enable_interrupts(trans); spin_unlock_bh(&trans_pcie->irq_lock); } /* Device is going down disable ict interrupt usage */ void iwl_pcie_disable_ict(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); spin_lock_bh(&trans_pcie->irq_lock); trans_pcie->use_ict = false; spin_unlock_bh(&trans_pcie->irq_lock); } irqreturn_t iwl_pcie_isr(int irq, void *data) { struct iwl_trans *trans = data; if (!trans) return IRQ_NONE; /* Disable (but don't clear!) interrupts here to avoid * back-to-back ISRs and sporadic interrupts from our NIC. * If we have something to service, the tasklet will re-enable ints. * If we *don't* have something, we'll re-enable before leaving here. */ iwl_write32(trans, CSR_INT_MASK, 0x00000000); return IRQ_WAKE_THREAD; } irqreturn_t iwl_pcie_msix_isr(int irq, void *data) { return IRQ_WAKE_THREAD; } irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id) { struct msix_entry *entry = dev_id; struct iwl_trans_pcie *trans_pcie = iwl_pcie_get_trans_pcie(entry); struct iwl_trans *trans = trans_pcie->trans; struct isr_statistics *isr_stats = &trans_pcie->isr_stats; u32 inta_fh_msk = ~MSIX_FH_INT_CAUSES_DATA_QUEUE; u32 inta_fh, inta_hw; bool polling = false; bool sw_err; if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX) inta_fh_msk |= MSIX_FH_INT_CAUSES_Q0; if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS) inta_fh_msk |= MSIX_FH_INT_CAUSES_Q1; lock_map_acquire(&trans->sync_cmd_lockdep_map); spin_lock_bh(&trans_pcie->irq_lock); inta_fh = iwl_read32(trans, CSR_MSIX_FH_INT_CAUSES_AD); inta_hw = iwl_read32(trans, CSR_MSIX_HW_INT_CAUSES_AD); /* * Clear causes registers to avoid being handling the same cause. */ iwl_write32(trans, CSR_MSIX_FH_INT_CAUSES_AD, inta_fh & inta_fh_msk); iwl_write32(trans, CSR_MSIX_HW_INT_CAUSES_AD, inta_hw); spin_unlock_bh(&trans_pcie->irq_lock); trace_iwlwifi_dev_irq_msix(trans->dev, entry, true, inta_fh, inta_hw); if (unlikely(!(inta_fh | inta_hw))) { IWL_DEBUG_ISR(trans, "Ignore interrupt, inta == 0\n"); lock_map_release(&trans->sync_cmd_lockdep_map); return IRQ_NONE; } #ifdef CONFIG_IWLWIFI_DEBUG if (iwl_have_debug_level(IWL_DL_ISR)) { IWL_DEBUG_ISR(trans, "ISR[%d] inta_fh 0x%08x, enabled (sw) 0x%08x (hw) 0x%08x\n", entry->entry, inta_fh, trans_pcie->fh_mask, iwl_read32(trans, CSR_MSIX_FH_INT_MASK_AD)); if (inta_fh & ~trans_pcie->fh_mask) IWL_DEBUG_ISR(trans, "We got a masked interrupt (0x%08x)\n", inta_fh & ~trans_pcie->fh_mask); } #endif inta_fh &= trans_pcie->fh_mask; if ((trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX) && inta_fh & MSIX_FH_INT_CAUSES_Q0) { local_bh_disable(); if (napi_schedule_prep(&trans_pcie->rxq[0].napi)) { polling = true; __napi_schedule(&trans_pcie->rxq[0].napi); } local_bh_enable(); } if ((trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS) && inta_fh & MSIX_FH_INT_CAUSES_Q1) { local_bh_disable(); if (napi_schedule_prep(&trans_pcie->rxq[1].napi)) { polling = true; __napi_schedule(&trans_pcie->rxq[1].napi); } local_bh_enable(); } /* This "Tx" DMA channel is used only for loading uCode */ if (inta_fh & MSIX_FH_INT_CAUSES_D2S_CH0_NUM && trans_pcie->imr_status == IMR_D2S_REQUESTED) { IWL_DEBUG_ISR(trans, "IMR Complete interrupt\n"); isr_stats->tx++; /* Wake up IMR routine once write to SRAM is complete */ if (trans_pcie->imr_status == IMR_D2S_REQUESTED) { trans_pcie->imr_status = IMR_D2S_COMPLETED; wake_up(&trans_pcie->ucode_write_waitq); } } else if (inta_fh & MSIX_FH_INT_CAUSES_D2S_CH0_NUM) { IWL_DEBUG_ISR(trans, "uCode load interrupt\n"); isr_stats->tx++; /* * Wake up uCode load routine, * now that load is complete */ trans_pcie->ucode_write_complete = true; wake_up(&trans_pcie->ucode_write_waitq); /* Wake up IMR routine once write to SRAM is complete */ if (trans_pcie->imr_status == IMR_D2S_REQUESTED) { trans_pcie->imr_status = IMR_D2S_COMPLETED; wake_up(&trans_pcie->ucode_write_waitq); } } if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) sw_err = inta_hw & MSIX_HW_INT_CAUSES_REG_SW_ERR_BZ; else sw_err = inta_hw & MSIX_HW_INT_CAUSES_REG_SW_ERR; /* Error detected by uCode */ if ((inta_fh & MSIX_FH_INT_CAUSES_FH_ERR) || sw_err) { IWL_ERR(trans, "Microcode SW error detected. Restarting 0x%X.\n", inta_fh); isr_stats->sw++; /* during FW reset flow report errors from there */ if (trans_pcie->imr_status == IMR_D2S_REQUESTED) { trans_pcie->imr_status = IMR_D2S_ERROR; wake_up(&trans_pcie->imr_waitq); } else if (trans_pcie->fw_reset_state == FW_RESET_REQUESTED) { trans_pcie->fw_reset_state = FW_RESET_ERROR; wake_up(&trans_pcie->fw_reset_waitq); } else { iwl_pcie_irq_handle_error(trans); } } /* After checking FH register check HW register */ #ifdef CONFIG_IWLWIFI_DEBUG if (iwl_have_debug_level(IWL_DL_ISR)) { IWL_DEBUG_ISR(trans, "ISR[%d] inta_hw 0x%08x, enabled (sw) 0x%08x (hw) 0x%08x\n", entry->entry, inta_hw, trans_pcie->hw_mask, iwl_read32(trans, CSR_MSIX_HW_INT_MASK_AD)); if (inta_hw & ~trans_pcie->hw_mask) IWL_DEBUG_ISR(trans, "We got a masked interrupt 0x%08x\n", inta_hw & ~trans_pcie->hw_mask); } #endif inta_hw &= trans_pcie->hw_mask; /* Alive notification via Rx interrupt will do the real work */ if (inta_hw & MSIX_HW_INT_CAUSES_REG_ALIVE) { IWL_DEBUG_ISR(trans, "Alive interrupt\n"); isr_stats->alive++; if (trans->trans_cfg->gen2) { /* We can restock, since firmware configured the RFH */ iwl_pcie_rxmq_restock(trans, trans_pcie->rxq); } } /* * In some rare cases when the HW is in a bad state, we may * get this interrupt too early, when prph_info is still NULL. * So make sure that it's not NULL to prevent crashing. */ if (inta_hw & MSIX_HW_INT_CAUSES_REG_WAKEUP && trans_pcie->prph_info) { u32 sleep_notif = le32_to_cpu(trans_pcie->prph_info->sleep_notif); if (sleep_notif == IWL_D3_SLEEP_STATUS_SUSPEND || sleep_notif == IWL_D3_SLEEP_STATUS_RESUME) { IWL_DEBUG_ISR(trans, "Sx interrupt: sleep notification = 0x%x\n", sleep_notif); trans_pcie->sx_complete = true; wake_up(&trans_pcie->sx_waitq); } else { /* uCode wakes up after power-down sleep */ IWL_DEBUG_ISR(trans, "Wakeup interrupt\n"); iwl_pcie_rxq_check_wrptr(trans); iwl_pcie_txq_check_wrptrs(trans); isr_stats->wakeup++; } } /* Chip got too hot and stopped itself */ if (inta_hw & MSIX_HW_INT_CAUSES_REG_CT_KILL) { IWL_ERR(trans, "Microcode CT kill error detected.\n"); isr_stats->ctkill++; } /* HW RF KILL switch toggled */ if (inta_hw & MSIX_HW_INT_CAUSES_REG_RF_KILL) iwl_pcie_handle_rfkill_irq(trans); if (inta_hw & MSIX_HW_INT_CAUSES_REG_HW_ERR) { IWL_ERR(trans, "Hardware error detected. Restarting.\n"); isr_stats->hw++; trans->dbg.hw_error = true; iwl_pcie_irq_handle_error(trans); } if (inta_hw & MSIX_HW_INT_CAUSES_REG_RESET_DONE) { IWL_DEBUG_ISR(trans, "Reset flow completed\n"); trans_pcie->fw_reset_state = FW_RESET_OK; wake_up(&trans_pcie->fw_reset_waitq); } if (!polling) iwl_pcie_clear_irq(trans, entry->entry); lock_map_release(&trans->sync_cmd_lockdep_map); return IRQ_HANDLED; } diff --git a/sys/contrib/dev/rtw88/pci.c b/sys/contrib/dev/rtw88/pci.c index bad6dd5b5b98..151400f6f892 100644 --- a/sys/contrib/dev/rtw88/pci.c +++ b/sys/contrib/dev/rtw88/pci.c @@ -1,1960 +1,1959 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* Copyright(c) 2018-2019 Realtek Corporation */ #if defined(__FreeBSD__) #define LINUXKPI_PARAM_PREFIX rtw88_pci_ #endif #include #include #include "main.h" #include "pci.h" #include "reg.h" #include "tx.h" #include "rx.h" #include "fw.h" #include "ps.h" #include "debug.h" #if defined(__FreeBSD__) #include #endif static bool rtw_disable_msi; static bool rtw_pci_disable_aspm; module_param_named(disable_msi, rtw_disable_msi, bool, 0644); module_param_named(disable_aspm, rtw_pci_disable_aspm, bool, 0644); MODULE_PARM_DESC(disable_msi, "Set Y to disable MSI interrupt support"); MODULE_PARM_DESC(disable_aspm, "Set Y to disable PCI ASPM support"); static u32 rtw_pci_tx_queue_idx_addr[] = { [RTW_TX_QUEUE_BK] = RTK_PCI_TXBD_IDX_BKQ, [RTW_TX_QUEUE_BE] = RTK_PCI_TXBD_IDX_BEQ, [RTW_TX_QUEUE_VI] = RTK_PCI_TXBD_IDX_VIQ, [RTW_TX_QUEUE_VO] = RTK_PCI_TXBD_IDX_VOQ, [RTW_TX_QUEUE_MGMT] = RTK_PCI_TXBD_IDX_MGMTQ, [RTW_TX_QUEUE_HI0] = RTK_PCI_TXBD_IDX_HI0Q, [RTW_TX_QUEUE_H2C] = RTK_PCI_TXBD_IDX_H2CQ, }; static u8 rtw_pci_get_tx_qsel(struct sk_buff *skb, u8 queue) { switch (queue) { case RTW_TX_QUEUE_BCN: return TX_DESC_QSEL_BEACON; case RTW_TX_QUEUE_H2C: return TX_DESC_QSEL_H2C; case RTW_TX_QUEUE_MGMT: return TX_DESC_QSEL_MGMT; case RTW_TX_QUEUE_HI0: return TX_DESC_QSEL_HIGH; default: return skb->priority; } }; static u8 rtw_pci_read8(struct rtw_dev *rtwdev, u32 addr) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; #if defined(__linux__) return readb(rtwpci->mmap + addr); #elif defined(__FreeBSD__) u8 val; val = bus_read_1((struct resource *)rtwpci->mmap, addr); rtw_dbg(rtwdev, RTW_DBG_IO_RW, "R08 (%#010x) -> %#04x\n", addr, val); return (val); #endif } static u16 rtw_pci_read16(struct rtw_dev *rtwdev, u32 addr) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; #if defined(__linux__) return readw(rtwpci->mmap + addr); #elif defined(__FreeBSD__) u16 val; val = bus_read_2((struct resource *)rtwpci->mmap, addr); rtw_dbg(rtwdev, RTW_DBG_IO_RW, "R16 (%#010x) -> %#06x\n", addr, val); return (val); #endif } static u32 rtw_pci_read32(struct rtw_dev *rtwdev, u32 addr) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; #if defined(__linux__) return readl(rtwpci->mmap + addr); #elif defined(__FreeBSD__) u32 val; val = bus_read_4((struct resource *)rtwpci->mmap, addr); rtw_dbg(rtwdev, RTW_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val); return (val); #endif } static void rtw_pci_write8(struct rtw_dev *rtwdev, u32 addr, u8 val) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; #if defined(__linux__) writeb(val, rtwpci->mmap + addr); #elif defined(__FreeBSD__) rtw_dbg(rtwdev, RTW_DBG_IO_RW, "W08 (%#010x) <- %#04x\n", addr, val); return (bus_write_1((struct resource *)rtwpci->mmap, addr, val)); #endif } static void rtw_pci_write16(struct rtw_dev *rtwdev, u32 addr, u16 val) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; #if defined(__linux__) writew(val, rtwpci->mmap + addr); #elif defined(__FreeBSD__) rtw_dbg(rtwdev, RTW_DBG_IO_RW, "W16 (%#010x) <- %#06x\n", addr, val); return (bus_write_2((struct resource *)rtwpci->mmap, addr, val)); #endif } static void rtw_pci_write32(struct rtw_dev *rtwdev, u32 addr, u32 val) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; #if defined(__linux__) writel(val, rtwpci->mmap + addr); #elif defined(__FreeBSD__) rtw_dbg(rtwdev, RTW_DBG_IO_RW, "W32 (%#010x) <- %#010x\n", addr, val); return (bus_write_4((struct resource *)rtwpci->mmap, addr, val)); #endif } #if defined(__linux__) && 0 static inline void *rtw_pci_get_tx_desc(struct rtw_pci_tx_ring *tx_ring, u8 idx) { int offset = tx_ring->r.desc_size * idx; return tx_ring->r.head + offset; } #endif static void rtw_pci_free_tx_ring_skbs(struct rtw_dev *rtwdev, struct rtw_pci_tx_ring *tx_ring) { struct pci_dev *pdev = to_pci_dev(rtwdev->dev); struct rtw_pci_tx_data *tx_data; struct sk_buff *skb, *tmp; dma_addr_t dma; /* free every skb remained in tx list */ skb_queue_walk_safe(&tx_ring->queue, skb, tmp) { __skb_unlink(skb, &tx_ring->queue); tx_data = rtw_pci_get_tx_data(skb); dma = tx_data->dma; dma_unmap_single(&pdev->dev, dma, skb->len, DMA_TO_DEVICE); dev_kfree_skb_any(skb); } } static void rtw_pci_free_tx_ring(struct rtw_dev *rtwdev, struct rtw_pci_tx_ring *tx_ring) { struct pci_dev *pdev = to_pci_dev(rtwdev->dev); u8 *head = tx_ring->r.head; u32 len = tx_ring->r.len; int ring_sz = len * tx_ring->r.desc_size; rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring); /* free the ring itself */ dma_free_coherent(&pdev->dev, ring_sz, head, tx_ring->r.dma); tx_ring->r.head = NULL; } static void rtw_pci_free_rx_ring_skbs(struct rtw_dev *rtwdev, struct rtw_pci_rx_ring *rx_ring) { struct pci_dev *pdev = to_pci_dev(rtwdev->dev); struct sk_buff *skb; int buf_sz = RTK_PCI_RX_BUF_SIZE; dma_addr_t dma; int i; for (i = 0; i < rx_ring->r.len; i++) { skb = rx_ring->buf[i]; if (!skb) continue; dma = *((dma_addr_t *)skb->cb); dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); dev_kfree_skb(skb); rx_ring->buf[i] = NULL; } } static void rtw_pci_free_rx_ring(struct rtw_dev *rtwdev, struct rtw_pci_rx_ring *rx_ring) { struct pci_dev *pdev = to_pci_dev(rtwdev->dev); u8 *head = rx_ring->r.head; int ring_sz = rx_ring->r.desc_size * rx_ring->r.len; rtw_pci_free_rx_ring_skbs(rtwdev, rx_ring); dma_free_coherent(&pdev->dev, ring_sz, head, rx_ring->r.dma); } static void rtw_pci_free_trx_ring(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct rtw_pci_tx_ring *tx_ring; struct rtw_pci_rx_ring *rx_ring; int i; for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) { tx_ring = &rtwpci->tx_rings[i]; rtw_pci_free_tx_ring(rtwdev, tx_ring); } for (i = 0; i < RTK_MAX_RX_QUEUE_NUM; i++) { rx_ring = &rtwpci->rx_rings[i]; rtw_pci_free_rx_ring(rtwdev, rx_ring); } } static int rtw_pci_init_tx_ring(struct rtw_dev *rtwdev, struct rtw_pci_tx_ring *tx_ring, u8 desc_size, u32 len) { struct pci_dev *pdev = to_pci_dev(rtwdev->dev); int ring_sz = desc_size * len; dma_addr_t dma; u8 *head; if (len > TRX_BD_IDX_MASK) { rtw_err(rtwdev, "len %d exceeds maximum TX entries\n", len); return -EINVAL; } head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); if (!head) { rtw_err(rtwdev, "failed to allocate tx ring\n"); return -ENOMEM; } skb_queue_head_init(&tx_ring->queue); tx_ring->r.head = head; tx_ring->r.dma = dma; tx_ring->r.len = len; tx_ring->r.desc_size = desc_size; tx_ring->r.wp = 0; tx_ring->r.rp = 0; return 0; } static int rtw_pci_reset_rx_desc(struct rtw_dev *rtwdev, struct sk_buff *skb, struct rtw_pci_rx_ring *rx_ring, u32 idx, u32 desc_sz) { struct pci_dev *pdev = to_pci_dev(rtwdev->dev); struct rtw_pci_rx_buffer_desc *buf_desc; int buf_sz = RTK_PCI_RX_BUF_SIZE; dma_addr_t dma; if (!skb) return -EINVAL; dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE); if (dma_mapping_error(&pdev->dev, dma)) return -EBUSY; *((dma_addr_t *)skb->cb) = dma; buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + idx * desc_sz); memset(buf_desc, 0, sizeof(*buf_desc)); buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE); buf_desc->dma = cpu_to_le32(dma); return 0; } static void rtw_pci_sync_rx_desc_device(struct rtw_dev *rtwdev, dma_addr_t dma, struct rtw_pci_rx_ring *rx_ring, u32 idx, u32 desc_sz) { struct device *dev = rtwdev->dev; struct rtw_pci_rx_buffer_desc *buf_desc; int buf_sz = RTK_PCI_RX_BUF_SIZE; dma_sync_single_for_device(dev, dma, buf_sz, DMA_FROM_DEVICE); buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + idx * desc_sz); memset(buf_desc, 0, sizeof(*buf_desc)); buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE); buf_desc->dma = cpu_to_le32(dma); } static int rtw_pci_init_rx_ring(struct rtw_dev *rtwdev, struct rtw_pci_rx_ring *rx_ring, u8 desc_size, u32 len) { struct pci_dev *pdev = to_pci_dev(rtwdev->dev); struct sk_buff *skb = NULL; dma_addr_t dma; u8 *head; int ring_sz = desc_size * len; int buf_sz = RTK_PCI_RX_BUF_SIZE; int i, allocated; int ret = 0; head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); if (!head) { rtw_err(rtwdev, "failed to allocate rx ring\n"); return -ENOMEM; } rx_ring->r.head = head; for (i = 0; i < len; i++) { skb = dev_alloc_skb(buf_sz); if (!skb) { allocated = i; ret = -ENOMEM; goto err_out; } memset(skb->data, 0, buf_sz); rx_ring->buf[i] = skb; ret = rtw_pci_reset_rx_desc(rtwdev, skb, rx_ring, i, desc_size); if (ret) { allocated = i; dev_kfree_skb_any(skb); goto err_out; } } rx_ring->r.dma = dma; rx_ring->r.len = len; rx_ring->r.desc_size = desc_size; rx_ring->r.wp = 0; rx_ring->r.rp = 0; return 0; err_out: for (i = 0; i < allocated; i++) { skb = rx_ring->buf[i]; if (!skb) continue; dma = *((dma_addr_t *)skb->cb); dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); dev_kfree_skb_any(skb); rx_ring->buf[i] = NULL; } dma_free_coherent(&pdev->dev, ring_sz, head, dma); rtw_err(rtwdev, "failed to init rx buffer\n"); return ret; } static int rtw_pci_init_trx_ring(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct rtw_pci_tx_ring *tx_ring; struct rtw_pci_rx_ring *rx_ring; struct rtw_chip_info *chip = rtwdev->chip; int i = 0, j = 0, tx_alloced = 0, rx_alloced = 0; int tx_desc_size, rx_desc_size; u32 len; int ret; tx_desc_size = chip->tx_buf_desc_sz; for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) { tx_ring = &rtwpci->tx_rings[i]; len = max_num_of_tx_queue(i); ret = rtw_pci_init_tx_ring(rtwdev, tx_ring, tx_desc_size, len); if (ret) goto out; } rx_desc_size = chip->rx_buf_desc_sz; for (j = 0; j < RTK_MAX_RX_QUEUE_NUM; j++) { rx_ring = &rtwpci->rx_rings[j]; ret = rtw_pci_init_rx_ring(rtwdev, rx_ring, rx_desc_size, RTK_MAX_RX_DESC_NUM); if (ret) goto out; } return 0; out: tx_alloced = i; for (i = 0; i < tx_alloced; i++) { tx_ring = &rtwpci->tx_rings[i]; rtw_pci_free_tx_ring(rtwdev, tx_ring); } rx_alloced = j; for (j = 0; j < rx_alloced; j++) { rx_ring = &rtwpci->rx_rings[j]; rtw_pci_free_rx_ring(rtwdev, rx_ring); } return ret; } static void rtw_pci_deinit(struct rtw_dev *rtwdev) { rtw_pci_free_trx_ring(rtwdev); } static int rtw_pci_init(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; int ret = 0; rtwpci->irq_mask[0] = IMR_HIGHDOK | IMR_MGNTDOK | IMR_BKDOK | IMR_BEDOK | IMR_VIDOK | IMR_VODOK | IMR_ROK | IMR_BCNDMAINT_E | IMR_C2HCMD | 0; rtwpci->irq_mask[1] = IMR_TXFOVW | 0; rtwpci->irq_mask[3] = IMR_H2CDOK | 0; spin_lock_init(&rtwpci->irq_lock); spin_lock_init(&rtwpci->hwirq_lock); ret = rtw_pci_init_trx_ring(rtwdev); return ret; } static void rtw_pci_reset_buf_desc(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; u32 len; u8 tmp; dma_addr_t dma; tmp = rtw_read8(rtwdev, RTK_PCI_CTRL + 3); rtw_write8(rtwdev, RTK_PCI_CTRL + 3, tmp | 0xf7); dma = rtwpci->tx_rings[RTW_TX_QUEUE_BCN].r.dma; rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BCNQ, dma); if (!rtw_chip_wcpu_11n(rtwdev)) { len = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.len; dma = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.dma; rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.rp = 0; rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.wp = 0; rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_H2CQ, len & TRX_BD_IDX_MASK); rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_H2CQ, dma); } len = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.len; dma = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.dma; rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.rp = 0; rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.wp = 0; rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BKQ, len & TRX_BD_IDX_MASK); rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BKQ, dma); len = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.len; dma = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.dma; rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.rp = 0; rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.wp = 0; rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BEQ, len & TRX_BD_IDX_MASK); rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BEQ, dma); len = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.len; dma = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.dma; rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.rp = 0; rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.wp = 0; rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VOQ, len & TRX_BD_IDX_MASK); rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VOQ, dma); len = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.len; dma = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.dma; rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.rp = 0; rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.wp = 0; rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VIQ, len & TRX_BD_IDX_MASK); rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VIQ, dma); len = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.len; dma = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.dma; rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.rp = 0; rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.wp = 0; rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_MGMTQ, len & TRX_BD_IDX_MASK); rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_MGMTQ, dma); len = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.len; dma = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.dma; rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.rp = 0; rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.wp = 0; rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_HI0Q, len & TRX_BD_IDX_MASK); rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_HI0Q, dma); len = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.len; dma = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.dma; rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.rp = 0; rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.wp = 0; rtw_write16(rtwdev, RTK_PCI_RXBD_NUM_MPDUQ, len & TRX_BD_IDX_MASK); rtw_write32(rtwdev, RTK_PCI_RXBD_DESA_MPDUQ, dma); /* reset read/write point */ rtw_write32(rtwdev, RTK_PCI_TXBD_RWPTR_CLR, 0xffffffff); /* reset H2C Queue index in a single write */ if (rtw_chip_wcpu_11ac(rtwdev)) rtw_write32_set(rtwdev, RTK_PCI_TXBD_H2CQ_CSR, BIT_CLR_H2CQ_HOST_IDX | BIT_CLR_H2CQ_HW_IDX); } static void rtw_pci_reset_trx_ring(struct rtw_dev *rtwdev) { rtw_pci_reset_buf_desc(rtwdev); } static void rtw_pci_enable_interrupt(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, bool exclude_rx) { unsigned long flags; u32 imr0_unmask = exclude_rx ? IMR_ROK : 0; spin_lock_irqsave(&rtwpci->hwirq_lock, flags); rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask); rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]); if (rtw_chip_wcpu_11ac(rtwdev)) rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]); rtwpci->irq_enabled = true; spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); } static void rtw_pci_disable_interrupt(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci) { unsigned long flags; spin_lock_irqsave(&rtwpci->hwirq_lock, flags); if (!rtwpci->irq_enabled) goto out; rtw_write32(rtwdev, RTK_PCI_HIMR0, 0); rtw_write32(rtwdev, RTK_PCI_HIMR1, 0); if (rtw_chip_wcpu_11ac(rtwdev)) rtw_write32(rtwdev, RTK_PCI_HIMR3, 0); rtwpci->irq_enabled = false; out: spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); } static void rtw_pci_dma_reset(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci) { /* reset dma and rx tag */ rtw_write32_set(rtwdev, RTK_PCI_CTRL, BIT_RST_TRXDMA_INTF | BIT_RX_TAG_EN); rtwpci->rx_tag = 0; } static int rtw_pci_setup(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; rtw_pci_reset_trx_ring(rtwdev); rtw_pci_dma_reset(rtwdev, rtwpci); return 0; } static void rtw_pci_dma_release(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci) { struct rtw_pci_tx_ring *tx_ring; u8 queue; rtw_pci_reset_trx_ring(rtwdev); for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) { tx_ring = &rtwpci->tx_rings[queue]; rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring); } } static void rtw_pci_napi_start(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; if (test_and_set_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags)) return; napi_enable(&rtwpci->napi); } static void rtw_pci_napi_stop(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; if (!test_and_clear_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags)) return; napi_synchronize(&rtwpci->napi); napi_disable(&rtwpci->napi); } static int rtw_pci_start(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; rtw_pci_napi_start(rtwdev); spin_lock_bh(&rtwpci->irq_lock); rtwpci->running = true; rtw_pci_enable_interrupt(rtwdev, rtwpci, false); spin_unlock_bh(&rtwpci->irq_lock); return 0; } static void rtw_pci_stop(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct pci_dev *pdev = rtwpci->pdev; spin_lock_bh(&rtwpci->irq_lock); rtwpci->running = false; rtw_pci_disable_interrupt(rtwdev, rtwpci); spin_unlock_bh(&rtwpci->irq_lock); synchronize_irq(pdev->irq); rtw_pci_napi_stop(rtwdev); spin_lock_bh(&rtwpci->irq_lock); rtw_pci_dma_release(rtwdev, rtwpci); spin_unlock_bh(&rtwpci->irq_lock); } static void rtw_pci_deep_ps_enter(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct rtw_pci_tx_ring *tx_ring; bool tx_empty = true; u8 queue; if (rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE)) goto enter_deep_ps; lockdep_assert_held(&rtwpci->irq_lock); /* Deep PS state is not allowed to TX-DMA */ for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) { /* BCN queue is rsvd page, does not have DMA interrupt * H2C queue is managed by firmware */ if (queue == RTW_TX_QUEUE_BCN || queue == RTW_TX_QUEUE_H2C) continue; tx_ring = &rtwpci->tx_rings[queue]; /* check if there is any skb DMAing */ if (skb_queue_len(&tx_ring->queue)) { tx_empty = false; break; } } if (!tx_empty) { rtw_dbg(rtwdev, RTW_DBG_PS, "TX path not empty, cannot enter deep power save state\n"); return; } enter_deep_ps: set_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags); rtw_power_mode_change(rtwdev, true); } static void rtw_pci_deep_ps_leave(struct rtw_dev *rtwdev) { #if defined(__linux__) struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; lockdep_assert_held(&rtwpci->irq_lock); #elif defined(__FreeBSD__) lockdep_assert_held(&((struct rtw_pci *)rtwdev->priv)->irq_lock); #endif if (test_and_clear_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) rtw_power_mode_change(rtwdev, false); } static void rtw_pci_deep_ps(struct rtw_dev *rtwdev, bool enter) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; spin_lock_bh(&rtwpci->irq_lock); if (enter && !test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) rtw_pci_deep_ps_enter(rtwdev); if (!enter && test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) rtw_pci_deep_ps_leave(rtwdev); spin_unlock_bh(&rtwpci->irq_lock); } static u8 ac_to_hwq[] = { [IEEE80211_AC_VO] = RTW_TX_QUEUE_VO, [IEEE80211_AC_VI] = RTW_TX_QUEUE_VI, [IEEE80211_AC_BE] = RTW_TX_QUEUE_BE, [IEEE80211_AC_BK] = RTW_TX_QUEUE_BK, }; #if defined(__linux__) static_assert(ARRAY_SIZE(ac_to_hwq) == IEEE80211_NUM_ACS); #elif defined(__FreeBSD__) rtw88_static_assert(ARRAY_SIZE(ac_to_hwq) == IEEE80211_NUM_ACS); #endif static u8 rtw_hw_queue_mapping(struct sk_buff *skb) { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; __le16 fc = hdr->frame_control; u8 q_mapping = skb_get_queue_mapping(skb); u8 queue; if (unlikely(ieee80211_is_beacon(fc))) queue = RTW_TX_QUEUE_BCN; else if (unlikely(ieee80211_is_mgmt(fc) || ieee80211_is_ctl(fc))) queue = RTW_TX_QUEUE_MGMT; else if (is_broadcast_ether_addr(hdr->addr1) || is_multicast_ether_addr(hdr->addr1)) queue = RTW_TX_QUEUE_HI0; else if (WARN_ON_ONCE(q_mapping >= ARRAY_SIZE(ac_to_hwq))) queue = ac_to_hwq[IEEE80211_AC_BE]; else queue = ac_to_hwq[q_mapping]; return queue; } static void rtw_pci_release_rsvd_page(struct rtw_pci *rtwpci, struct rtw_pci_tx_ring *ring) { struct sk_buff *prev = skb_dequeue(&ring->queue); struct rtw_pci_tx_data *tx_data; dma_addr_t dma; if (!prev) return; tx_data = rtw_pci_get_tx_data(prev); dma = tx_data->dma; dma_unmap_single(&rtwpci->pdev->dev, dma, prev->len, DMA_TO_DEVICE); dev_kfree_skb_any(prev); } static void rtw_pci_dma_check(struct rtw_dev *rtwdev, struct rtw_pci_rx_ring *rx_ring, u32 idx) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct rtw_chip_info *chip = rtwdev->chip; struct rtw_pci_rx_buffer_desc *buf_desc; u32 desc_sz = chip->rx_buf_desc_sz; u16 total_pkt_size; buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + idx * desc_sz); total_pkt_size = le16_to_cpu(buf_desc->total_pkt_size); /* rx tag mismatch, throw a warning */ if (total_pkt_size != rtwpci->rx_tag) rtw_warn(rtwdev, "pci bus timeout, check dma status\n"); rtwpci->rx_tag = (rtwpci->rx_tag + 1) % RX_TAG_MAX; } static u32 __pci_get_hw_tx_ring_rp(struct rtw_dev *rtwdev, u8 pci_q) { u32 bd_idx_addr = rtw_pci_tx_queue_idx_addr[pci_q]; u32 bd_idx = rtw_read16(rtwdev, bd_idx_addr + 2); return FIELD_GET(TRX_BD_IDX_MASK, bd_idx); } static void __pci_flush_queue(struct rtw_dev *rtwdev, u8 pci_q, bool drop) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct rtw_pci_tx_ring *ring = &rtwpci->tx_rings[pci_q]; u32 cur_rp; u8 i; /* Because the time taked by the I/O in __pci_get_hw_tx_ring_rp is a * bit dynamic, it's hard to define a reasonable fixed total timeout to * use read_poll_timeout* helper. Instead, we can ensure a reasonable * polling times, so we just use for loop with udelay here. */ for (i = 0; i < 30; i++) { cur_rp = __pci_get_hw_tx_ring_rp(rtwdev, pci_q); if (cur_rp == ring->r.wp) return; udelay(1); } if (!drop) rtw_warn(rtwdev, "timed out to flush pci tx ring[%d]\n", pci_q); } static void __rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 pci_queues, bool drop) { u8 q; for (q = 0; q < RTK_MAX_TX_QUEUE_NUM; q++) { /* It may be not necessary to flush BCN and H2C tx queues. */ if (q == RTW_TX_QUEUE_BCN || q == RTW_TX_QUEUE_H2C) continue; if (pci_queues & BIT(q)) __pci_flush_queue(rtwdev, q, drop); } } static void rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 queues, bool drop) { u32 pci_queues = 0; u8 i; /* If all of the hardware queues are requested to flush, * flush all of the pci queues. */ if (queues == BIT(rtwdev->hw->queues) - 1) { pci_queues = BIT(RTK_MAX_TX_QUEUE_NUM) - 1; } else { for (i = 0; i < rtwdev->hw->queues; i++) if (queues & BIT(i)) pci_queues |= BIT(ac_to_hwq[i]); } __rtw_pci_flush_queues(rtwdev, pci_queues, drop); } static void rtw_pci_tx_kick_off_queue(struct rtw_dev *rtwdev, u8 queue) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct rtw_pci_tx_ring *ring; u32 bd_idx; ring = &rtwpci->tx_rings[queue]; bd_idx = rtw_pci_tx_queue_idx_addr[queue]; spin_lock_bh(&rtwpci->irq_lock); if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE)) rtw_pci_deep_ps_leave(rtwdev); rtw_write16(rtwdev, bd_idx, ring->r.wp & TRX_BD_IDX_MASK); spin_unlock_bh(&rtwpci->irq_lock); } static void rtw_pci_tx_kick_off(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; u8 queue; for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) if (test_and_clear_bit(queue, rtwpci->tx_queued)) rtw_pci_tx_kick_off_queue(rtwdev, queue); } static int rtw_pci_tx_write_data(struct rtw_dev *rtwdev, struct rtw_tx_pkt_info *pkt_info, struct sk_buff *skb, u8 queue) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct rtw_chip_info *chip = rtwdev->chip; struct rtw_pci_tx_ring *ring; struct rtw_pci_tx_data *tx_data; dma_addr_t dma; u32 tx_pkt_desc_sz = chip->tx_pkt_desc_sz; u32 tx_buf_desc_sz = chip->tx_buf_desc_sz; u32 size; u32 psb_len; u8 *pkt_desc; struct rtw_pci_tx_buffer_desc *buf_desc; ring = &rtwpci->tx_rings[queue]; size = skb->len; if (queue == RTW_TX_QUEUE_BCN) rtw_pci_release_rsvd_page(rtwpci, ring); else if (!avail_desc(ring->r.wp, ring->r.rp, ring->r.len)) return -ENOSPC; pkt_desc = skb_push(skb, chip->tx_pkt_desc_sz); memset(pkt_desc, 0, tx_pkt_desc_sz); pkt_info->qsel = rtw_pci_get_tx_qsel(skb, queue); rtw_tx_fill_tx_desc(pkt_info, skb); dma = dma_map_single(&rtwpci->pdev->dev, skb->data, skb->len, DMA_TO_DEVICE); if (dma_mapping_error(&rtwpci->pdev->dev, dma)) return -EBUSY; /* after this we got dma mapped, there is no way back */ buf_desc = get_tx_buffer_desc(ring, tx_buf_desc_sz); memset(buf_desc, 0, tx_buf_desc_sz); psb_len = (skb->len - 1) / 128 + 1; if (queue == RTW_TX_QUEUE_BCN) psb_len |= 1 << RTK_PCI_TXBD_OWN_OFFSET; buf_desc[0].psb_len = cpu_to_le16(psb_len); buf_desc[0].buf_size = cpu_to_le16(tx_pkt_desc_sz); buf_desc[0].dma = cpu_to_le32(dma); buf_desc[1].buf_size = cpu_to_le16(size); buf_desc[1].dma = cpu_to_le32(dma + tx_pkt_desc_sz); tx_data = rtw_pci_get_tx_data(skb); tx_data->dma = dma; tx_data->sn = pkt_info->sn; spin_lock_bh(&rtwpci->irq_lock); skb_queue_tail(&ring->queue, skb); if (queue == RTW_TX_QUEUE_BCN) goto out_unlock; /* update write-index, and kick it off later */ set_bit(queue, rtwpci->tx_queued); if (++ring->r.wp >= ring->r.len) ring->r.wp = 0; out_unlock: spin_unlock_bh(&rtwpci->irq_lock); return 0; } static int rtw_pci_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf, u32 size) { struct sk_buff *skb; struct rtw_tx_pkt_info pkt_info = {0}; u8 reg_bcn_work; int ret; skb = rtw_tx_write_data_rsvd_page_get(rtwdev, &pkt_info, buf, size); if (!skb) return -ENOMEM; ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_BCN); if (ret) { #if defined(__FreeBSD__) dev_kfree_skb_any(skb); #endif rtw_err(rtwdev, "failed to write rsvd page data\n"); return ret; } /* reserved pages go through beacon queue */ reg_bcn_work = rtw_read8(rtwdev, RTK_PCI_TXBD_BCN_WORK); reg_bcn_work |= BIT_PCI_BCNQ_FLAG; rtw_write8(rtwdev, RTK_PCI_TXBD_BCN_WORK, reg_bcn_work); return 0; } static int rtw_pci_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size) { struct sk_buff *skb; struct rtw_tx_pkt_info pkt_info = {0}; int ret; skb = rtw_tx_write_data_h2c_get(rtwdev, &pkt_info, buf, size); if (!skb) return -ENOMEM; ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_H2C); if (ret) { #if defined(__FreeBSD__) dev_kfree_skb_any(skb); #endif rtw_err(rtwdev, "failed to write h2c data\n"); return ret; } rtw_pci_tx_kick_off_queue(rtwdev, RTW_TX_QUEUE_H2C); return 0; } static int rtw_pci_tx_write(struct rtw_dev *rtwdev, struct rtw_tx_pkt_info *pkt_info, struct sk_buff *skb) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct rtw_pci_tx_ring *ring; u8 queue = rtw_hw_queue_mapping(skb); int ret; ret = rtw_pci_tx_write_data(rtwdev, pkt_info, skb, queue); if (ret) return ret; ring = &rtwpci->tx_rings[queue]; spin_lock_bh(&rtwpci->irq_lock); if (avail_desc(ring->r.wp, ring->r.rp, ring->r.len) < 2) { ieee80211_stop_queue(rtwdev->hw, skb_get_queue_mapping(skb)); ring->queue_stopped = true; } spin_unlock_bh(&rtwpci->irq_lock); return 0; } static void rtw_pci_tx_isr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, u8 hw_queue) { struct ieee80211_hw *hw = rtwdev->hw; struct ieee80211_tx_info *info; struct rtw_pci_tx_ring *ring; struct rtw_pci_tx_data *tx_data; struct sk_buff *skb; u32 count; u32 bd_idx_addr; u32 bd_idx, cur_rp, rp_idx; u16 q_map; ring = &rtwpci->tx_rings[hw_queue]; bd_idx_addr = rtw_pci_tx_queue_idx_addr[hw_queue]; bd_idx = rtw_read32(rtwdev, bd_idx_addr); cur_rp = bd_idx >> 16; cur_rp &= TRX_BD_IDX_MASK; rp_idx = ring->r.rp; if (cur_rp >= ring->r.rp) count = cur_rp - ring->r.rp; else count = ring->r.len - (ring->r.rp - cur_rp); while (count--) { skb = skb_dequeue(&ring->queue); if (!skb) { rtw_err(rtwdev, "failed to dequeue %d skb TX queue %d, BD=0x%08x, rp %d -> %d\n", count, hw_queue, bd_idx, ring->r.rp, cur_rp); break; } tx_data = rtw_pci_get_tx_data(skb); dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len, DMA_TO_DEVICE); /* just free command packets from host to card */ if (hw_queue == RTW_TX_QUEUE_H2C) { dev_kfree_skb_irq(skb); continue; } if (ring->queue_stopped && avail_desc(ring->r.wp, rp_idx, ring->r.len) > 4) { q_map = skb_get_queue_mapping(skb); ieee80211_wake_queue(hw, q_map); ring->queue_stopped = false; } if (++rp_idx >= ring->r.len) rp_idx = 0; skb_pull(skb, rtwdev->chip->tx_pkt_desc_sz); info = IEEE80211_SKB_CB(skb); /* enqueue to wait for tx report */ if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) { rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn); continue; } /* always ACK for others, then they won't be marked as drop */ if (info->flags & IEEE80211_TX_CTL_NO_ACK) info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; else info->flags |= IEEE80211_TX_STAT_ACK; ieee80211_tx_info_clear_status(info); ieee80211_tx_status_irqsafe(hw, skb); } ring->r.rp = cur_rp; } static void rtw_pci_rx_isr(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct napi_struct *napi = &rtwpci->napi; napi_schedule(napi); } static int rtw_pci_get_hw_rx_ring_nr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci) { struct rtw_pci_rx_ring *ring; int count = 0; u32 tmp, cur_wp; ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU]; tmp = rtw_read32(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ); cur_wp = u32_get_bits(tmp, TRX_BD_HW_IDX_MASK); if (cur_wp >= ring->r.wp) count = cur_wp - ring->r.wp; else count = ring->r.len - (ring->r.wp - cur_wp); return count; } static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, u8 hw_queue, u32 limit) { struct rtw_chip_info *chip = rtwdev->chip; struct napi_struct *napi = &rtwpci->napi; struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU]; struct rtw_rx_pkt_stat pkt_stat; struct ieee80211_rx_status rx_status; struct sk_buff *skb, *new; u32 cur_rp = ring->r.rp; u32 count, rx_done = 0; u32 pkt_offset; u32 pkt_desc_sz = chip->rx_pkt_desc_sz; u32 buf_desc_sz = chip->rx_buf_desc_sz; u32 new_len; u8 *rx_desc; dma_addr_t dma; count = rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci); count = min(count, limit); while (count--) { rtw_pci_dma_check(rtwdev, ring, cur_rp); skb = ring->buf[cur_rp]; dma = *((dma_addr_t *)skb->cb); dma_sync_single_for_cpu(rtwdev->dev, dma, RTK_PCI_RX_BUF_SIZE, DMA_FROM_DEVICE); rx_desc = skb->data; chip->ops->query_rx_desc(rtwdev, rx_desc, &pkt_stat, &rx_status); /* offset from rx_desc to payload */ pkt_offset = pkt_desc_sz + pkt_stat.drv_info_sz + pkt_stat.shift; /* allocate a new skb for this frame, * discard the frame if none available */ new_len = pkt_stat.pkt_len + pkt_offset; new = dev_alloc_skb(new_len); if (WARN_ONCE(!new, "rx routine starvation\n")) goto next_rp; /* put the DMA data including rx_desc from phy to new skb */ skb_put_data(new, skb->data, new_len); if (pkt_stat.is_c2h) { rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, new); } else { /* remove rx_desc */ skb_pull(new, pkt_offset); rtw_rx_stats(rtwdev, pkt_stat.vif, new); memcpy(new->cb, &rx_status, sizeof(rx_status)); ieee80211_rx_napi(rtwdev->hw, NULL, new, napi); rx_done++; } next_rp: /* new skb delivered to mac80211, re-enable original skb DMA */ rtw_pci_sync_rx_desc_device(rtwdev, dma, ring, cur_rp, buf_desc_sz); /* host read next element in ring */ if (++cur_rp >= ring->r.len) cur_rp = 0; } ring->r.rp = cur_rp; /* 'rp', the last position we have read, is seen as previous posistion * of 'wp' that is used to calculate 'count' next time. */ ring->r.wp = cur_rp; rtw_write16(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ, ring->r.rp); return rx_done; } static void rtw_pci_irq_recognized(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, u32 *irq_status) { unsigned long flags; spin_lock_irqsave(&rtwpci->hwirq_lock, flags); irq_status[0] = rtw_read32(rtwdev, RTK_PCI_HISR0); irq_status[1] = rtw_read32(rtwdev, RTK_PCI_HISR1); if (rtw_chip_wcpu_11ac(rtwdev)) irq_status[3] = rtw_read32(rtwdev, RTK_PCI_HISR3); else irq_status[3] = 0; irq_status[0] &= rtwpci->irq_mask[0]; irq_status[1] &= rtwpci->irq_mask[1]; irq_status[3] &= rtwpci->irq_mask[3]; rtw_write32(rtwdev, RTK_PCI_HISR0, irq_status[0]); rtw_write32(rtwdev, RTK_PCI_HISR1, irq_status[1]); if (rtw_chip_wcpu_11ac(rtwdev)) rtw_write32(rtwdev, RTK_PCI_HISR3, irq_status[3]); spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); } static irqreturn_t rtw_pci_interrupt_handler(int irq, void *dev) { struct rtw_dev *rtwdev = dev; struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; /* disable RTW PCI interrupt to avoid more interrupts before the end of * thread function * * disable HIMR here to also avoid new HISR flag being raised before * the HISRs have been Write-1-cleared for MSI. If not all of the HISRs * are cleared, the edge-triggered interrupt will not be generated when * a new HISR flag is set. */ rtw_pci_disable_interrupt(rtwdev, rtwpci); return IRQ_WAKE_THREAD; } static irqreturn_t rtw_pci_interrupt_threadfn(int irq, void *dev) { struct rtw_dev *rtwdev = dev; struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; u32 irq_status[4]; bool rx = false; spin_lock_bh(&rtwpci->irq_lock); rtw_pci_irq_recognized(rtwdev, rtwpci, irq_status); if (irq_status[0] & IMR_MGNTDOK) rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_MGMT); if (irq_status[0] & IMR_HIGHDOK) rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_HI0); if (irq_status[0] & IMR_BEDOK) rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BE); if (irq_status[0] & IMR_BKDOK) rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BK); if (irq_status[0] & IMR_VODOK) rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VO); if (irq_status[0] & IMR_VIDOK) rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VI); if (irq_status[3] & IMR_H2CDOK) rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_H2C); if (irq_status[0] & IMR_ROK) { rtw_pci_rx_isr(rtwdev); rx = true; } if (unlikely(irq_status[0] & IMR_C2HCMD)) rtw_fw_c2h_cmd_isr(rtwdev); /* all of the jobs for this interrupt have been done */ if (rtwpci->running) rtw_pci_enable_interrupt(rtwdev, rtwpci, rx); spin_unlock_bh(&rtwpci->irq_lock); return IRQ_HANDLED; } static int rtw_pci_io_mapping(struct rtw_dev *rtwdev, struct pci_dev *pdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; unsigned long len; u8 bar_id = 2; int ret; ret = pci_request_regions(pdev, KBUILD_MODNAME); if (ret) { rtw_err(rtwdev, "failed to request pci regions\n"); return ret; } #if defined(__FreeBSD__) ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); if (ret) { rtw_err(rtwdev, "failed to set dma mask to 32-bit\n"); goto err_release_regions; } ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); if (ret) { rtw_err(rtwdev, "failed to set consistent dma mask to 32-bit\n"); goto err_release_regions; } #endif len = pci_resource_len(pdev, bar_id); #if defined(__FreeBSD__) linuxkpi_pcim_want_to_use_bus_functions(pdev); #endif rtwpci->mmap = pci_iomap(pdev, bar_id, len); if (!rtwpci->mmap) { pci_release_regions(pdev); rtw_err(rtwdev, "failed to map pci memory\n"); return -ENOMEM; } return 0; #if defined(__FreeBSD__) err_release_regions: pci_release_regions(pdev); return ret; #endif } static void rtw_pci_io_unmapping(struct rtw_dev *rtwdev, struct pci_dev *pdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; if (rtwpci->mmap) { pci_iounmap(pdev, rtwpci->mmap); pci_release_regions(pdev); } } static void rtw_dbi_write8(struct rtw_dev *rtwdev, u16 addr, u8 data) { u16 write_addr; u16 remainder = addr & ~(BITS_DBI_WREN | BITS_DBI_ADDR_MASK); u8 flag; u8 cnt; write_addr = addr & BITS_DBI_ADDR_MASK; write_addr |= u16_encode_bits(BIT(remainder), BITS_DBI_WREN); rtw_write8(rtwdev, REG_DBI_WDATA_V1 + remainder, data); rtw_write16(rtwdev, REG_DBI_FLAG_V1, write_addr); rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_WFLAG >> 16); for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2); if (flag == 0) return; udelay(10); } WARN(flag, "failed to write to DBI register, addr=0x%04x\n", addr); } static int rtw_dbi_read8(struct rtw_dev *rtwdev, u16 addr, u8 *value) { u16 read_addr = addr & BITS_DBI_ADDR_MASK; u8 flag; u8 cnt; rtw_write16(rtwdev, REG_DBI_FLAG_V1, read_addr); rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_RFLAG >> 16); for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2); if (flag == 0) { read_addr = REG_DBI_RDATA_V1 + (addr & 3); *value = rtw_read8(rtwdev, read_addr); return 0; } udelay(10); } WARN(1, "failed to read DBI register, addr=0x%04x\n", addr); return -EIO; } static void rtw_mdio_write(struct rtw_dev *rtwdev, u8 addr, u16 data, bool g1) { u8 page; u8 wflag; u8 cnt; rtw_write16(rtwdev, REG_MDIO_V1, data); page = addr < RTW_PCI_MDIO_PG_SZ ? 0 : 1; page += g1 ? RTW_PCI_MDIO_PG_OFFS_G1 : RTW_PCI_MDIO_PG_OFFS_G2; rtw_write8(rtwdev, REG_PCIE_MIX_CFG, addr & BITS_MDIO_ADDR_MASK); rtw_write8(rtwdev, REG_PCIE_MIX_CFG + 3, page); rtw_write32_mask(rtwdev, REG_PCIE_MIX_CFG, BIT_MDIO_WFLAG_V1, 1); for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { wflag = rtw_read32_mask(rtwdev, REG_PCIE_MIX_CFG, BIT_MDIO_WFLAG_V1); if (wflag == 0) return; udelay(10); } WARN(wflag, "failed to write to MDIO register, addr=0x%02x\n", addr); } static void rtw_pci_clkreq_set(struct rtw_dev *rtwdev, bool enable) { u8 value; int ret; if (rtw_pci_disable_aspm) return; ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); if (ret) { rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret); return; } if (enable) value |= BIT_CLKREQ_SW_EN; else value &= ~BIT_CLKREQ_SW_EN; rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); } static void rtw_pci_clkreq_pad_low(struct rtw_dev *rtwdev, bool enable) { u8 value; int ret; ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); if (ret) { rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret); return; } if (enable) value &= ~BIT_CLKREQ_N_PAD; else value |= BIT_CLKREQ_N_PAD; rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); } static void rtw_pci_aspm_set(struct rtw_dev *rtwdev, bool enable) { u8 value; int ret; if (rtw_pci_disable_aspm) return; ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); if (ret) { rtw_err(rtwdev, "failed to read ASPM, ret=%d", ret); return; } if (enable) value |= BIT_L1_SW_EN; else value &= ~BIT_L1_SW_EN; rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); } static void rtw_pci_link_ps(struct rtw_dev *rtwdev, bool enter) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; /* Like CLKREQ, ASPM is also implemented by two HW modules, and can * only be enabled when host supports it. * * And ASPM mechanism should be enabled when driver/firmware enters * power save mode, without having heavy traffic. Because we've * experienced some inter-operability issues that the link tends * to enter L1 state on the fly even when driver is having high * throughput. This is probably because the ASPM behavior slightly * varies from different SOC. */ if (!(rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1)) return; if ((enter && atomic_dec_if_positive(&rtwpci->link_usage) == 0) || (!enter && atomic_inc_return(&rtwpci->link_usage) == 1)) rtw_pci_aspm_set(rtwdev, enter); } static void rtw_pci_link_cfg(struct rtw_dev *rtwdev) { struct rtw_chip_info *chip = rtwdev->chip; struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct pci_dev *pdev = rtwpci->pdev; u16 link_ctrl; int ret; /* RTL8822CE has enabled REFCLK auto calibration, it does not need * to add clock delay to cover the REFCLK timing gap. */ if (chip->id == RTW_CHIP_TYPE_8822C) rtw_dbi_write8(rtwdev, RTK_PCIE_CLKDLY_CTRL, 0); /* Though there is standard PCIE configuration space to set the * link control register, but by Realtek's design, driver should * check if host supports CLKREQ/ASPM to enable the HW module. * * These functions are implemented by two HW modules associated, * one is responsible to access PCIE configuration space to * follow the host settings, and another is in charge of doing * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes * the host does not support it, and due to some reasons or wrong * settings (ex. CLKREQ# not Bi-Direction), it could lead to device * loss if HW misbehaves on the link. * * Hence it's designed that driver should first check the PCIE * configuration space is sync'ed and enabled, then driver can turn * on the other module that is actually working on the mechanism. */ ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl); if (ret) { rtw_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret); return; } if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN) rtw_pci_clkreq_set(rtwdev, true); rtwpci->link_ctrl = link_ctrl; } static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev) { struct rtw_chip_info *chip = rtwdev->chip; switch (chip->id) { case RTW_CHIP_TYPE_8822C: if (rtwdev->hal.cut_version >= RTW_CHIP_VER_CUT_D) rtw_write32_mask(rtwdev, REG_HCI_MIX_CFG, BIT_PCIE_EMAC_PDN_AUX_TO_FAST_CLK, 1); break; default: break; } } static void rtw_pci_phy_cfg(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct rtw_chip_info *chip = rtwdev->chip; struct pci_dev *pdev = rtwpci->pdev; const struct rtw_intf_phy_para *para; u16 cut; u16 value; u16 offset; int i; int ret; cut = BIT(0) << rtwdev->hal.cut_version; for (i = 0; i < chip->intf_table->n_gen1_para; i++) { para = &chip->intf_table->gen1_para[i]; if (!(para->cut_mask & cut)) continue; if (para->offset == 0xffff) break; offset = para->offset; value = para->value; if (para->ip_sel == RTW_IP_SEL_PHY) rtw_mdio_write(rtwdev, offset, value, true); else rtw_dbi_write8(rtwdev, offset, value); } for (i = 0; i < chip->intf_table->n_gen2_para; i++) { para = &chip->intf_table->gen2_para[i]; if (!(para->cut_mask & cut)) continue; if (para->offset == 0xffff) break; offset = para->offset; value = para->value; if (para->ip_sel == RTW_IP_SEL_PHY) rtw_mdio_write(rtwdev, offset, value, false); else rtw_dbi_write8(rtwdev, offset, value); } rtw_pci_link_cfg(rtwdev); /* Disable 8821ce completion timeout by default */ if (chip->id == RTW_CHIP_TYPE_8821C) { ret = pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2, PCI_EXP_DEVCTL2_COMP_TMOUT_DIS); if (ret) rtw_err(rtwdev, "failed to set PCI cap, ret = %d\n", ret); } } static int __maybe_unused rtw_pci_suspend(struct device *dev) { struct ieee80211_hw *hw = dev_get_drvdata(dev); struct rtw_dev *rtwdev = hw->priv; struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6) rtw_pci_clkreq_pad_low(rtwdev, true); return 0; } static int __maybe_unused rtw_pci_resume(struct device *dev) { struct ieee80211_hw *hw = dev_get_drvdata(dev); struct rtw_dev *rtwdev = hw->priv; struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6) rtw_pci_clkreq_pad_low(rtwdev, false); return 0; } SIMPLE_DEV_PM_OPS(rtw_pm_ops, rtw_pci_suspend, rtw_pci_resume); EXPORT_SYMBOL(rtw_pm_ops); static int rtw_pci_claim(struct rtw_dev *rtwdev, struct pci_dev *pdev) { int ret; ret = pci_enable_device(pdev); if (ret) { rtw_err(rtwdev, "failed to enable pci device\n"); return ret; } pci_set_master(pdev); pci_set_drvdata(pdev, rtwdev->hw); SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev); return 0; } static void rtw_pci_declaim(struct rtw_dev *rtwdev, struct pci_dev *pdev) { pci_clear_master(pdev); pci_disable_device(pdev); } static int rtw_pci_setup_resource(struct rtw_dev *rtwdev, struct pci_dev *pdev) { struct rtw_pci *rtwpci; int ret; rtwpci = (struct rtw_pci *)rtwdev->priv; rtwpci->pdev = pdev; /* after this driver can access to hw registers */ ret = rtw_pci_io_mapping(rtwdev, pdev); if (ret) { rtw_err(rtwdev, "failed to request pci io region\n"); goto err_out; } ret = rtw_pci_init(rtwdev); if (ret) { rtw_err(rtwdev, "failed to allocate pci resources\n"); goto err_io_unmap; } return 0; err_io_unmap: rtw_pci_io_unmapping(rtwdev, pdev); err_out: return ret; } static void rtw_pci_destroy(struct rtw_dev *rtwdev, struct pci_dev *pdev) { rtw_pci_deinit(rtwdev); rtw_pci_io_unmapping(rtwdev, pdev); } static struct rtw_hci_ops rtw_pci_ops = { .tx_write = rtw_pci_tx_write, .tx_kick_off = rtw_pci_tx_kick_off, .flush_queues = rtw_pci_flush_queues, .setup = rtw_pci_setup, .start = rtw_pci_start, .stop = rtw_pci_stop, .deep_ps = rtw_pci_deep_ps, .link_ps = rtw_pci_link_ps, .interface_cfg = rtw_pci_interface_cfg, .read8 = rtw_pci_read8, .read16 = rtw_pci_read16, .read32 = rtw_pci_read32, .write8 = rtw_pci_write8, .write16 = rtw_pci_write16, .write32 = rtw_pci_write32, .write_data_rsvd_page = rtw_pci_write_data_rsvd_page, .write_data_h2c = rtw_pci_write_data_h2c, }; static int rtw_pci_request_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev) { unsigned int flags = PCI_IRQ_LEGACY; int ret; if (!rtw_disable_msi) flags |= PCI_IRQ_MSI; ret = pci_alloc_irq_vectors(pdev, 1, 1, flags); if (ret < 0) { rtw_err(rtwdev, "failed to alloc PCI irq vectors\n"); return ret; } ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq, rtw_pci_interrupt_handler, rtw_pci_interrupt_threadfn, IRQF_SHARED, KBUILD_MODNAME, rtwdev); if (ret) { rtw_err(rtwdev, "failed to request irq %d\n", ret); pci_free_irq_vectors(pdev); } return ret; } static void rtw_pci_free_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev) { devm_free_irq(rtwdev->dev, pdev->irq, rtwdev); pci_free_irq_vectors(pdev); } static int rtw_pci_napi_poll(struct napi_struct *napi, int budget) { struct rtw_pci *rtwpci = container_of(napi, struct rtw_pci, napi); struct rtw_dev *rtwdev = container_of((void *)rtwpci, struct rtw_dev, priv); int work_done = 0; if (rtwpci->rx_no_aspm) rtw_pci_link_ps(rtwdev, false); while (work_done < budget) { u32 work_done_once; work_done_once = rtw_pci_rx_napi(rtwdev, rtwpci, RTW_RX_QUEUE_MPDU, budget - work_done); if (work_done_once == 0) break; work_done += work_done_once; } if (work_done < budget) { napi_complete_done(napi, work_done); spin_lock_bh(&rtwpci->irq_lock); if (rtwpci->running) rtw_pci_enable_interrupt(rtwdev, rtwpci, false); spin_unlock_bh(&rtwpci->irq_lock); /* When ISR happens during polling and before napi_complete * while no further data is received. Data on the dma_ring will * not be processed immediately. Check whether dma ring is * empty and perform napi_schedule accordingly. */ if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci)) napi_schedule(napi); } if (rtwpci->rx_no_aspm) rtw_pci_link_ps(rtwdev, true); return work_done; } static void rtw_pci_napi_init(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; init_dummy_netdev(&rtwpci->netdev); - netif_napi_add(&rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(&rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll); } static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; rtw_pci_napi_stop(rtwdev); netif_napi_del(&rtwpci->napi); } int rtw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct pci_dev *bridge = pci_upstream_bridge(pdev); struct ieee80211_hw *hw; struct rtw_dev *rtwdev; struct rtw_pci *rtwpci; int drv_data_size; int ret; drv_data_size = sizeof(struct rtw_dev) + sizeof(struct rtw_pci); hw = ieee80211_alloc_hw(drv_data_size, &rtw_ops); if (!hw) { dev_err(&pdev->dev, "failed to allocate hw\n"); return -ENOMEM; } rtwdev = hw->priv; rtwdev->hw = hw; rtwdev->dev = &pdev->dev; rtwdev->chip = (struct rtw_chip_info *)id->driver_data; rtwdev->hci.ops = &rtw_pci_ops; rtwdev->hci.type = RTW_HCI_TYPE_PCIE; rtwpci = (struct rtw_pci *)rtwdev->priv; atomic_set(&rtwpci->link_usage, 1); ret = rtw_core_init(rtwdev); if (ret) goto err_release_hw; rtw_dbg(rtwdev, RTW_DBG_PCI, "rtw88 pci probe: vendor=0x%4.04X device=0x%4.04X rev=%d\n", pdev->vendor, pdev->device, pdev->revision); ret = rtw_pci_claim(rtwdev, pdev); if (ret) { rtw_err(rtwdev, "failed to claim pci device\n"); goto err_deinit_core; } ret = rtw_pci_setup_resource(rtwdev, pdev); if (ret) { rtw_err(rtwdev, "failed to setup pci resources\n"); goto err_pci_declaim; } rtw_pci_napi_init(rtwdev); ret = rtw_chip_info_setup(rtwdev); if (ret) { rtw_err(rtwdev, "failed to setup chip information\n"); goto err_destroy_pci; } /* Disable PCIe ASPM L1 while doing NAPI poll for 8821CE */ if (rtwdev->chip->id == RTW_CHIP_TYPE_8821C && bridge->vendor == PCI_VENDOR_ID_INTEL) rtwpci->rx_no_aspm = true; rtw_pci_phy_cfg(rtwdev); ret = rtw_register_hw(rtwdev, hw); if (ret) { rtw_err(rtwdev, "failed to register hw\n"); goto err_destroy_pci; } ret = rtw_pci_request_irq(rtwdev, pdev); if (ret) { ieee80211_unregister_hw(hw); goto err_destroy_pci; } return 0; err_destroy_pci: rtw_pci_napi_deinit(rtwdev); rtw_pci_destroy(rtwdev, pdev); err_pci_declaim: rtw_pci_declaim(rtwdev, pdev); err_deinit_core: rtw_core_deinit(rtwdev); err_release_hw: ieee80211_free_hw(hw); return ret; } EXPORT_SYMBOL(rtw_pci_probe); void rtw_pci_remove(struct pci_dev *pdev) { struct ieee80211_hw *hw = pci_get_drvdata(pdev); struct rtw_dev *rtwdev; struct rtw_pci *rtwpci; if (!hw) return; rtwdev = hw->priv; rtwpci = (struct rtw_pci *)rtwdev->priv; rtw_unregister_hw(rtwdev, hw); rtw_pci_disable_interrupt(rtwdev, rtwpci); rtw_pci_napi_deinit(rtwdev); rtw_pci_destroy(rtwdev, pdev); rtw_pci_declaim(rtwdev, pdev); rtw_pci_free_irq(rtwdev, pdev); rtw_core_deinit(rtwdev); ieee80211_free_hw(hw); } EXPORT_SYMBOL(rtw_pci_remove); void rtw_pci_shutdown(struct pci_dev *pdev) { struct ieee80211_hw *hw = pci_get_drvdata(pdev); struct rtw_dev *rtwdev; struct rtw_chip_info *chip; if (!hw) return; rtwdev = hw->priv; chip = rtwdev->chip; if (chip->ops->shutdown) chip->ops->shutdown(rtwdev); pci_set_power_state(pdev, PCI_D3hot); } EXPORT_SYMBOL(rtw_pci_shutdown); MODULE_AUTHOR("Realtek Corporation"); MODULE_DESCRIPTION("Realtek 802.11ac wireless PCI driver"); MODULE_LICENSE("Dual BSD/GPL"); #if defined(__FreeBSD__) MODULE_VERSION(rtw_pci, 1); MODULE_DEPEND(rtw_pci, linuxkpi, 1, 1, 1); MODULE_DEPEND(rtw_pci, linuxkpi_wlan, 1, 1, 1); #ifdef CONFIG_RTW88_DEBUGFS MODULE_DEPEND(rtw_pci, lindebugfs, 1, 1, 1); #endif #endif diff --git a/sys/contrib/dev/rtw89/core.c b/sys/contrib/dev/rtw89/core.c index 98f47c4daeac..25940c141f84 100644 --- a/sys/contrib/dev/rtw89/core.c +++ b/sys/contrib/dev/rtw89/core.c @@ -1,3318 +1,3318 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* Copyright(c) 2019-2020 Realtek Corporation */ #if defined(__FreeBSD__) #define LINUXKPI_PARAM_PREFIX rtw89_ #endif #include #include #include "cam.h" #include "chan.h" #include "coex.h" #include "core.h" #include "efuse.h" #include "fw.h" #include "mac.h" #include "phy.h" #include "ps.h" #include "reg.h" #include "sar.h" #include "ser.h" #include "txrx.h" #include "util.h" static bool rtw89_disable_ps_mode; module_param_named(disable_ps_mode, rtw89_disable_ps_mode, bool, 0644); MODULE_PARM_DESC(disable_ps_mode, "Set Y to disable low power mode"); #define RTW89_DEF_CHAN(_freq, _hw_val, _flags, _band) \ { .center_freq = _freq, .hw_value = _hw_val, .flags = _flags, .band = _band, } #define RTW89_DEF_CHAN_2G(_freq, _hw_val) \ RTW89_DEF_CHAN(_freq, _hw_val, 0, NL80211_BAND_2GHZ) #define RTW89_DEF_CHAN_5G(_freq, _hw_val) \ RTW89_DEF_CHAN(_freq, _hw_val, 0, NL80211_BAND_5GHZ) #define RTW89_DEF_CHAN_5G_NO_HT40MINUS(_freq, _hw_val) \ RTW89_DEF_CHAN(_freq, _hw_val, IEEE80211_CHAN_NO_HT40MINUS, NL80211_BAND_5GHZ) #define RTW89_DEF_CHAN_6G(_freq, _hw_val) \ RTW89_DEF_CHAN(_freq, _hw_val, 0, NL80211_BAND_6GHZ) static struct ieee80211_channel rtw89_channels_2ghz[] = { RTW89_DEF_CHAN_2G(2412, 1), RTW89_DEF_CHAN_2G(2417, 2), RTW89_DEF_CHAN_2G(2422, 3), RTW89_DEF_CHAN_2G(2427, 4), RTW89_DEF_CHAN_2G(2432, 5), RTW89_DEF_CHAN_2G(2437, 6), RTW89_DEF_CHAN_2G(2442, 7), RTW89_DEF_CHAN_2G(2447, 8), RTW89_DEF_CHAN_2G(2452, 9), RTW89_DEF_CHAN_2G(2457, 10), RTW89_DEF_CHAN_2G(2462, 11), RTW89_DEF_CHAN_2G(2467, 12), RTW89_DEF_CHAN_2G(2472, 13), RTW89_DEF_CHAN_2G(2484, 14), }; static struct ieee80211_channel rtw89_channels_5ghz[] = { RTW89_DEF_CHAN_5G(5180, 36), RTW89_DEF_CHAN_5G(5200, 40), RTW89_DEF_CHAN_5G(5220, 44), RTW89_DEF_CHAN_5G(5240, 48), RTW89_DEF_CHAN_5G(5260, 52), RTW89_DEF_CHAN_5G(5280, 56), RTW89_DEF_CHAN_5G(5300, 60), RTW89_DEF_CHAN_5G(5320, 64), RTW89_DEF_CHAN_5G(5500, 100), RTW89_DEF_CHAN_5G(5520, 104), RTW89_DEF_CHAN_5G(5540, 108), RTW89_DEF_CHAN_5G(5560, 112), RTW89_DEF_CHAN_5G(5580, 116), RTW89_DEF_CHAN_5G(5600, 120), RTW89_DEF_CHAN_5G(5620, 124), RTW89_DEF_CHAN_5G(5640, 128), RTW89_DEF_CHAN_5G(5660, 132), RTW89_DEF_CHAN_5G(5680, 136), RTW89_DEF_CHAN_5G(5700, 140), RTW89_DEF_CHAN_5G(5720, 144), RTW89_DEF_CHAN_5G(5745, 149), RTW89_DEF_CHAN_5G(5765, 153), RTW89_DEF_CHAN_5G(5785, 157), RTW89_DEF_CHAN_5G(5805, 161), RTW89_DEF_CHAN_5G_NO_HT40MINUS(5825, 165), }; static struct ieee80211_channel rtw89_channels_6ghz[] = { RTW89_DEF_CHAN_6G(5955, 1), RTW89_DEF_CHAN_6G(5975, 5), RTW89_DEF_CHAN_6G(5995, 9), RTW89_DEF_CHAN_6G(6015, 13), RTW89_DEF_CHAN_6G(6035, 17), RTW89_DEF_CHAN_6G(6055, 21), RTW89_DEF_CHAN_6G(6075, 25), RTW89_DEF_CHAN_6G(6095, 29), RTW89_DEF_CHAN_6G(6115, 33), RTW89_DEF_CHAN_6G(6135, 37), RTW89_DEF_CHAN_6G(6155, 41), RTW89_DEF_CHAN_6G(6175, 45), RTW89_DEF_CHAN_6G(6195, 49), RTW89_DEF_CHAN_6G(6215, 53), RTW89_DEF_CHAN_6G(6235, 57), RTW89_DEF_CHAN_6G(6255, 61), RTW89_DEF_CHAN_6G(6275, 65), RTW89_DEF_CHAN_6G(6295, 69), RTW89_DEF_CHAN_6G(6315, 73), RTW89_DEF_CHAN_6G(6335, 77), RTW89_DEF_CHAN_6G(6355, 81), RTW89_DEF_CHAN_6G(6375, 85), RTW89_DEF_CHAN_6G(6395, 89), RTW89_DEF_CHAN_6G(6415, 93), RTW89_DEF_CHAN_6G(6435, 97), RTW89_DEF_CHAN_6G(6455, 101), RTW89_DEF_CHAN_6G(6475, 105), RTW89_DEF_CHAN_6G(6495, 109), RTW89_DEF_CHAN_6G(6515, 113), RTW89_DEF_CHAN_6G(6535, 117), RTW89_DEF_CHAN_6G(6555, 121), RTW89_DEF_CHAN_6G(6575, 125), RTW89_DEF_CHAN_6G(6595, 129), RTW89_DEF_CHAN_6G(6615, 133), RTW89_DEF_CHAN_6G(6635, 137), RTW89_DEF_CHAN_6G(6655, 141), RTW89_DEF_CHAN_6G(6675, 145), RTW89_DEF_CHAN_6G(6695, 149), RTW89_DEF_CHAN_6G(6715, 153), RTW89_DEF_CHAN_6G(6735, 157), RTW89_DEF_CHAN_6G(6755, 161), RTW89_DEF_CHAN_6G(6775, 165), RTW89_DEF_CHAN_6G(6795, 169), RTW89_DEF_CHAN_6G(6815, 173), RTW89_DEF_CHAN_6G(6835, 177), RTW89_DEF_CHAN_6G(6855, 181), RTW89_DEF_CHAN_6G(6875, 185), RTW89_DEF_CHAN_6G(6895, 189), RTW89_DEF_CHAN_6G(6915, 193), RTW89_DEF_CHAN_6G(6935, 197), RTW89_DEF_CHAN_6G(6955, 201), RTW89_DEF_CHAN_6G(6975, 205), RTW89_DEF_CHAN_6G(6995, 209), RTW89_DEF_CHAN_6G(7015, 213), RTW89_DEF_CHAN_6G(7035, 217), RTW89_DEF_CHAN_6G(7055, 221), RTW89_DEF_CHAN_6G(7075, 225), RTW89_DEF_CHAN_6G(7095, 229), RTW89_DEF_CHAN_6G(7115, 233), }; static struct ieee80211_rate rtw89_bitrates[] = { { .bitrate = 10, .hw_value = 0x00, }, { .bitrate = 20, .hw_value = 0x01, }, { .bitrate = 55, .hw_value = 0x02, }, { .bitrate = 110, .hw_value = 0x03, }, { .bitrate = 60, .hw_value = 0x04, }, { .bitrate = 90, .hw_value = 0x05, }, { .bitrate = 120, .hw_value = 0x06, }, { .bitrate = 180, .hw_value = 0x07, }, { .bitrate = 240, .hw_value = 0x08, }, { .bitrate = 360, .hw_value = 0x09, }, { .bitrate = 480, .hw_value = 0x0a, }, { .bitrate = 540, .hw_value = 0x0b, }, }; bool rtw89_ra_report_to_bitrate(struct rtw89_dev *rtwdev, u8 rpt_rate, u16 *bitrate) { struct ieee80211_rate rate; if (unlikely(rpt_rate >= ARRAY_SIZE(rtw89_bitrates))) { rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "invalid rpt rate %d\n", rpt_rate); return false; } rate = rtw89_bitrates[rpt_rate]; *bitrate = rate.bitrate; return true; } static struct ieee80211_supported_band rtw89_sband_2ghz = { .band = NL80211_BAND_2GHZ, .channels = rtw89_channels_2ghz, .n_channels = ARRAY_SIZE(rtw89_channels_2ghz), .bitrates = rtw89_bitrates, .n_bitrates = ARRAY_SIZE(rtw89_bitrates), .ht_cap = {0}, .vht_cap = {0}, }; static struct ieee80211_supported_band rtw89_sband_5ghz = { .band = NL80211_BAND_5GHZ, .channels = rtw89_channels_5ghz, .n_channels = ARRAY_SIZE(rtw89_channels_5ghz), /* 5G has no CCK rates, 1M/2M/5.5M/11M */ .bitrates = rtw89_bitrates + 4, .n_bitrates = ARRAY_SIZE(rtw89_bitrates) - 4, .ht_cap = {0}, .vht_cap = {0}, }; static struct ieee80211_supported_band rtw89_sband_6ghz = { .band = NL80211_BAND_6GHZ, .channels = rtw89_channels_6ghz, .n_channels = ARRAY_SIZE(rtw89_channels_6ghz), /* 6G has no CCK rates, 1M/2M/5.5M/11M */ .bitrates = rtw89_bitrates + 4, .n_bitrates = ARRAY_SIZE(rtw89_bitrates) - 4, }; static void rtw89_traffic_stats_accu(struct rtw89_dev *rtwdev, struct rtw89_traffic_stats *stats, struct sk_buff *skb, bool tx) { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; if (!ieee80211_is_data(hdr->frame_control)) return; if (is_broadcast_ether_addr(hdr->addr1) || is_multicast_ether_addr(hdr->addr1)) return; if (tx) { stats->tx_cnt++; stats->tx_unicast += skb->len; } else { stats->rx_cnt++; stats->rx_unicast += skb->len; } } void rtw89_get_default_chandef(struct cfg80211_chan_def *chandef) { cfg80211_chandef_create(chandef, &rtw89_channels_2ghz[0], NL80211_CHAN_NO_HT); } static void rtw89_get_channel_params(const struct cfg80211_chan_def *chandef, struct rtw89_chan *chan) { struct ieee80211_channel *channel = chandef->chan; enum nl80211_chan_width width = chandef->width; u32 primary_freq, center_freq; u8 center_chan; u8 bandwidth = RTW89_CHANNEL_WIDTH_20; u32 offset; u8 band; center_chan = channel->hw_value; primary_freq = channel->center_freq; center_freq = chandef->center_freq1; switch (width) { case NL80211_CHAN_WIDTH_20_NOHT: case NL80211_CHAN_WIDTH_20: bandwidth = RTW89_CHANNEL_WIDTH_20; break; case NL80211_CHAN_WIDTH_40: bandwidth = RTW89_CHANNEL_WIDTH_40; if (primary_freq > center_freq) { center_chan -= 2; } else { center_chan += 2; } break; case NL80211_CHAN_WIDTH_80: case NL80211_CHAN_WIDTH_160: bandwidth = nl_to_rtw89_bandwidth(width); if (primary_freq > center_freq) { offset = (primary_freq - center_freq - 10) / 20; center_chan -= 2 + offset * 4; } else { offset = (center_freq - primary_freq - 10) / 20; center_chan += 2 + offset * 4; } break; default: center_chan = 0; break; } switch (channel->band) { default: case NL80211_BAND_2GHZ: band = RTW89_BAND_2G; break; case NL80211_BAND_5GHZ: band = RTW89_BAND_5G; break; case NL80211_BAND_6GHZ: band = RTW89_BAND_6G; break; } rtw89_chan_create(chan, center_chan, channel->hw_value, band, bandwidth); } void rtw89_core_set_chip_txpwr(struct rtw89_dev *rtwdev) { const struct rtw89_chip_info *chip = rtwdev->chip; const struct rtw89_chan *chan; enum rtw89_sub_entity_idx sub_entity_idx; enum rtw89_phy_idx phy_idx; enum rtw89_entity_mode mode; bool entity_active; entity_active = rtw89_get_entity_state(rtwdev); if (!entity_active) return; mode = rtw89_get_entity_mode(rtwdev); if (WARN(mode != RTW89_ENTITY_MODE_SCC, "Invalid ent mode: %d\n", mode)) return; sub_entity_idx = RTW89_SUB_ENTITY_0; phy_idx = RTW89_PHY_0; chan = rtw89_chan_get(rtwdev, sub_entity_idx); if (chip->ops->set_txpwr) chip->ops->set_txpwr(rtwdev, chan, phy_idx); } void rtw89_set_channel(struct rtw89_dev *rtwdev) { const struct rtw89_chip_info *chip = rtwdev->chip; const struct cfg80211_chan_def *chandef; enum rtw89_sub_entity_idx sub_entity_idx; enum rtw89_mac_idx mac_idx; enum rtw89_phy_idx phy_idx; struct rtw89_chan chan; struct rtw89_channel_help_params bak; enum rtw89_entity_mode mode; bool band_changed; bool entity_active; entity_active = rtw89_get_entity_state(rtwdev); mode = rtw89_entity_recalc(rtwdev); if (WARN(mode != RTW89_ENTITY_MODE_SCC, "Invalid ent mode: %d\n", mode)) return; sub_entity_idx = RTW89_SUB_ENTITY_0; mac_idx = RTW89_MAC_0; phy_idx = RTW89_PHY_0; chandef = rtw89_chandef_get(rtwdev, sub_entity_idx); rtw89_get_channel_params(chandef, &chan); if (WARN(chan.channel == 0, "Invalid channel\n")) return; band_changed = rtw89_assign_entity_chan(rtwdev, sub_entity_idx, &chan); rtw89_chip_set_channel_prepare(rtwdev, &bak, &chan, mac_idx, phy_idx); chip->ops->set_channel(rtwdev, &chan, mac_idx, phy_idx); rtw89_core_set_chip_txpwr(rtwdev); rtw89_chip_set_channel_done(rtwdev, &bak, &chan, mac_idx, phy_idx); if (!entity_active || band_changed) { rtw89_btc_ntfy_switch_band(rtwdev, phy_idx, chan.band_type); rtw89_chip_rfk_band_changed(rtwdev, phy_idx); } rtw89_set_entity_state(rtwdev, true); } static enum rtw89_core_tx_type rtw89_core_get_tx_type(struct rtw89_dev *rtwdev, struct sk_buff *skb) { struct ieee80211_hdr *hdr = (void *)skb->data; __le16 fc = hdr->frame_control; if (ieee80211_is_mgmt(fc) || ieee80211_is_nullfunc(fc)) return RTW89_CORE_TX_TYPE_MGMT; return RTW89_CORE_TX_TYPE_DATA; } static void rtw89_core_tx_update_ampdu_info(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req, enum btc_pkt_type pkt_type) { struct ieee80211_sta *sta = tx_req->sta; struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; struct sk_buff *skb = tx_req->skb; struct rtw89_sta *rtwsta; u8 ampdu_num; u8 tid; if (pkt_type == PACKET_EAPOL) { desc_info->bk = true; return; } if (!(IEEE80211_SKB_CB(skb)->flags & IEEE80211_TX_CTL_AMPDU)) return; if (!sta) { rtw89_warn(rtwdev, "cannot set ampdu info without sta\n"); return; } tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK; rtwsta = (struct rtw89_sta *)sta->drv_priv; ampdu_num = (u8)((rtwsta->ampdu_params[tid].agg_num ? rtwsta->ampdu_params[tid].agg_num : 4 << sta->deflink.ht_cap.ampdu_factor) - 1); desc_info->agg_en = true; desc_info->ampdu_density = sta->deflink.ht_cap.ampdu_density; desc_info->ampdu_num = ampdu_num; } static void rtw89_core_tx_update_sec_key(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req) { const struct rtw89_chip_info *chip = rtwdev->chip; struct ieee80211_vif *vif = tx_req->vif; struct ieee80211_sta *sta = tx_req->sta; struct ieee80211_tx_info *info; struct ieee80211_key_conf *key; struct rtw89_vif *rtwvif; struct rtw89_sta *rtwsta = sta_to_rtwsta_safe(sta); struct rtw89_addr_cam_entry *addr_cam; struct rtw89_sec_cam_entry *sec_cam; struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; struct sk_buff *skb = tx_req->skb; u8 sec_type = RTW89_SEC_KEY_TYPE_NONE; u64 pn64; if (!vif) { rtw89_warn(rtwdev, "cannot set sec key without vif\n"); return; } rtwvif = (struct rtw89_vif *)vif->drv_priv; addr_cam = rtw89_get_addr_cam_of(rtwvif, rtwsta); info = IEEE80211_SKB_CB(skb); key = info->control.hw_key; sec_cam = addr_cam->sec_entries[key->hw_key_idx]; if (!sec_cam) { rtw89_warn(rtwdev, "sec cam entry is empty\n"); return; } switch (key->cipher) { case WLAN_CIPHER_SUITE_WEP40: sec_type = RTW89_SEC_KEY_TYPE_WEP40; break; case WLAN_CIPHER_SUITE_WEP104: sec_type = RTW89_SEC_KEY_TYPE_WEP104; break; case WLAN_CIPHER_SUITE_TKIP: sec_type = RTW89_SEC_KEY_TYPE_TKIP; break; case WLAN_CIPHER_SUITE_CCMP: sec_type = RTW89_SEC_KEY_TYPE_CCMP128; break; case WLAN_CIPHER_SUITE_CCMP_256: sec_type = RTW89_SEC_KEY_TYPE_CCMP256; break; case WLAN_CIPHER_SUITE_GCMP: sec_type = RTW89_SEC_KEY_TYPE_GCMP128; break; case WLAN_CIPHER_SUITE_GCMP_256: sec_type = RTW89_SEC_KEY_TYPE_GCMP256; break; default: rtw89_warn(rtwdev, "key cipher not supported %d\n", key->cipher); return; } desc_info->sec_en = true; desc_info->sec_keyid = key->keyidx; desc_info->sec_type = sec_type; desc_info->sec_cam_idx = sec_cam->sec_cam_idx; if (!chip->hw_sec_hdr) return; pn64 = atomic64_inc_return(&key->tx_pn); desc_info->sec_seq[0] = pn64; desc_info->sec_seq[1] = pn64 >> 8; desc_info->sec_seq[2] = pn64 >> 16; desc_info->sec_seq[3] = pn64 >> 24; desc_info->sec_seq[4] = pn64 >> 32; desc_info->sec_seq[5] = pn64 >> 40; desc_info->wp_offset = 1; /* in unit of 8 bytes for security header */ } static u16 rtw89_core_get_mgmt_rate(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req) { struct sk_buff *skb = tx_req->skb; struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(skb); struct ieee80211_vif *vif = tx_info->control.vif; const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); u16 lowest_rate = chan->band_type == RTW89_BAND_2G ? RTW89_HW_RATE_CCK1 : RTW89_HW_RATE_OFDM6; if (!vif || !vif->bss_conf.basic_rates || !tx_req->sta) return lowest_rate; return __ffs(vif->bss_conf.basic_rates) + lowest_rate; } static void rtw89_core_tx_update_mgmt_info(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req) { struct ieee80211_vif *vif = tx_req->vif; struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); u8 qsel, ch_dma; qsel = desc_info->hiq ? RTW89_TX_QSEL_B0_HI : RTW89_TX_QSEL_B0_MGMT; ch_dma = rtw89_core_get_ch_dma(rtwdev, qsel); desc_info->qsel = qsel; desc_info->ch_dma = ch_dma; desc_info->port = desc_info->hiq ? rtwvif->port : 0; desc_info->hw_ssn_sel = RTW89_MGMT_HW_SSN_SEL; desc_info->hw_seq_mode = RTW89_MGMT_HW_SEQ_MODE; /* fixed data rate for mgmt frames */ desc_info->en_wd_info = true; desc_info->use_rate = true; desc_info->dis_data_fb = true; desc_info->data_rate = rtw89_core_get_mgmt_rate(rtwdev, tx_req); rtw89_debug(rtwdev, RTW89_DBG_TXRX, "tx mgmt frame with rate 0x%x on channel %d (band %d, bw %d)\n", desc_info->data_rate, chan->channel, chan->band_type, chan->band_width); } static void rtw89_core_tx_update_h2c_info(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req) { struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; desc_info->is_bmc = false; desc_info->wd_page = false; desc_info->ch_dma = RTW89_DMA_H2C; } static void rtw89_core_get_no_ul_ofdma_htc(struct rtw89_dev *rtwdev, __le32 *htc) { static const u8 rtw89_bandwidth_to_om[] = { [RTW89_CHANNEL_WIDTH_20] = HTC_OM_CHANNEL_WIDTH_20, [RTW89_CHANNEL_WIDTH_40] = HTC_OM_CHANNEL_WIDTH_40, [RTW89_CHANNEL_WIDTH_80] = HTC_OM_CHANNEL_WIDTH_80, [RTW89_CHANNEL_WIDTH_160] = HTC_OM_CHANNEL_WIDTH_160_OR_80_80, [RTW89_CHANNEL_WIDTH_80_80] = HTC_OM_CHANNEL_WIDTH_160_OR_80_80, }; const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_hal *hal = &rtwdev->hal; const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); u8 om_bandwidth; if (!chip->dis_2g_40m_ul_ofdma || chan->band_type != RTW89_BAND_2G || chan->band_width != RTW89_CHANNEL_WIDTH_40) return; om_bandwidth = chan->band_width < ARRAY_SIZE(rtw89_bandwidth_to_om) ? rtw89_bandwidth_to_om[chan->band_width] : 0; *htc = le32_encode_bits(RTW89_HTC_VARIANT_HE, RTW89_HTC_MASK_VARIANT) | le32_encode_bits(RTW89_HTC_VARIANT_HE_CID_OM, RTW89_HTC_MASK_CTL_ID) | le32_encode_bits(hal->rx_nss - 1, RTW89_HTC_MASK_HTC_OM_RX_NSS) | le32_encode_bits(om_bandwidth, RTW89_HTC_MASK_HTC_OM_CH_WIDTH) | le32_encode_bits(1, RTW89_HTC_MASK_HTC_OM_UL_MU_DIS) | le32_encode_bits(hal->tx_nss - 1, RTW89_HTC_MASK_HTC_OM_TX_NSTS) | le32_encode_bits(0, RTW89_HTC_MASK_HTC_OM_ER_SU_DIS) | le32_encode_bits(0, RTW89_HTC_MASK_HTC_OM_DL_MU_MIMO_RR) | le32_encode_bits(0, RTW89_HTC_MASK_HTC_OM_UL_MU_DATA_DIS); } static bool __rtw89_core_tx_check_he_qos_htc(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req, enum btc_pkt_type pkt_type) { struct ieee80211_sta *sta = tx_req->sta; struct rtw89_sta *rtwsta = sta_to_rtwsta_safe(sta); struct sk_buff *skb = tx_req->skb; struct ieee80211_hdr *hdr = (void *)skb->data; __le16 fc = hdr->frame_control; /* AP IOT issue with EAPoL, ARP and DHCP */ if (pkt_type < PACKET_MAX) return false; if (!sta || !sta->deflink.he_cap.has_he) return false; if (!ieee80211_is_data_qos(fc)) return false; if (skb_headroom(skb) < IEEE80211_HT_CTL_LEN) return false; if (rtwsta && rtwsta->ra_report.might_fallback_legacy) return false; return true; } static void __rtw89_core_tx_adjust_he_qos_htc(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req) { struct ieee80211_sta *sta = tx_req->sta; struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; struct sk_buff *skb = tx_req->skb; struct ieee80211_hdr *hdr = (void *)skb->data; __le16 fc = hdr->frame_control; void *data; __le32 *htc; u8 *qc; int hdr_len; hdr_len = ieee80211_has_a4(fc) ? 32 : 26; data = skb_push(skb, IEEE80211_HT_CTL_LEN); #if defined(__linux__) memmove(data, data + IEEE80211_HT_CTL_LEN, hdr_len); #elif defined(__FreeBSD__) memmove(data, (u8 *)data + IEEE80211_HT_CTL_LEN, hdr_len); #endif hdr = data; #if defined(__linux__) htc = data + hdr_len; #elif defined(__FreeBSD__) htc = (__le32 *)((u8 *)data + hdr_len); #endif hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_ORDER); *htc = rtwsta->htc_template ? rtwsta->htc_template : le32_encode_bits(RTW89_HTC_VARIANT_HE, RTW89_HTC_MASK_VARIANT) | le32_encode_bits(RTW89_HTC_VARIANT_HE_CID_CAS, RTW89_HTC_MASK_CTL_ID); #if defined(__linux__) qc = data + hdr_len - IEEE80211_QOS_CTL_LEN; #elif defined(__FreeBSD__) qc = (u8 *)data + hdr_len - IEEE80211_QOS_CTL_LEN; #endif qc[0] |= IEEE80211_QOS_CTL_EOSP; } static void rtw89_core_tx_update_he_qos_htc(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req, enum btc_pkt_type pkt_type) { struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; struct ieee80211_vif *vif = tx_req->vif; struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; if (!__rtw89_core_tx_check_he_qos_htc(rtwdev, tx_req, pkt_type)) goto desc_bk; __rtw89_core_tx_adjust_he_qos_htc(rtwdev, tx_req); desc_info->pkt_size += IEEE80211_HT_CTL_LEN; desc_info->a_ctrl_bsr = true; desc_bk: if (!rtwvif || rtwvif->last_a_ctrl == desc_info->a_ctrl_bsr) return; rtwvif->last_a_ctrl = desc_info->a_ctrl_bsr; desc_info->bk = true; } static u8 rtw89_core_tx_get_mac_id(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req) { struct ieee80211_vif *vif = tx_req->vif; struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; struct ieee80211_sta *sta = tx_req->sta; struct rtw89_sta *rtwsta; if (!sta) return rtwvif->mac_id; rtwsta = (struct rtw89_sta *)sta->drv_priv; return rtwsta->mac_id; } static void rtw89_core_tx_update_data_info(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req) { struct ieee80211_vif *vif = tx_req->vif; struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; struct rtw89_phy_rate_pattern *rate_pattern = &rtwvif->rate_pattern; const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; struct sk_buff *skb = tx_req->skb; u8 tid, tid_indicate; u8 qsel, ch_dma; tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK; tid_indicate = rtw89_core_get_tid_indicate(rtwdev, tid); qsel = desc_info->hiq ? RTW89_TX_QSEL_B0_HI : rtw89_core_get_qsel(rtwdev, tid); ch_dma = rtw89_core_get_ch_dma(rtwdev, qsel); desc_info->ch_dma = ch_dma; desc_info->tid_indicate = tid_indicate; desc_info->qsel = qsel; desc_info->mac_id = rtw89_core_tx_get_mac_id(rtwdev, tx_req); desc_info->port = desc_info->hiq ? rtwvif->port : 0; /* enable wd_info for AMPDU */ desc_info->en_wd_info = true; if (IEEE80211_SKB_CB(skb)->control.hw_key) rtw89_core_tx_update_sec_key(rtwdev, tx_req); if (rate_pattern->enable) desc_info->data_retry_lowest_rate = rate_pattern->rate; else if (chan->band_type == RTW89_BAND_2G) desc_info->data_retry_lowest_rate = RTW89_HW_RATE_CCK1; else desc_info->data_retry_lowest_rate = RTW89_HW_RATE_OFDM6; } static enum btc_pkt_type rtw89_core_tx_btc_spec_pkt_notify(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req) { struct sk_buff *skb = tx_req->skb; struct udphdr *udphdr; if (IEEE80211_SKB_CB(skb)->control.flags & IEEE80211_TX_CTRL_PORT_CTRL_PROTO) { ieee80211_queue_work(rtwdev->hw, &rtwdev->btc.eapol_notify_work); return PACKET_EAPOL; } if (skb->protocol == htons(ETH_P_ARP)) { ieee80211_queue_work(rtwdev->hw, &rtwdev->btc.arp_notify_work); return PACKET_ARP; } if (skb->protocol == htons(ETH_P_IP) && ip_hdr(skb)->protocol == IPPROTO_UDP) { udphdr = udp_hdr(skb); if (((udphdr->source == htons(67) && udphdr->dest == htons(68)) || (udphdr->source == htons(68) && udphdr->dest == htons(67))) && skb->len > 282) { ieee80211_queue_work(rtwdev->hw, &rtwdev->btc.dhcp_notify_work); return PACKET_DHCP; } } if (skb->protocol == htons(ETH_P_IP) && ip_hdr(skb)->protocol == IPPROTO_ICMP) { ieee80211_queue_work(rtwdev->hw, &rtwdev->btc.icmp_notify_work); return PACKET_ICMP; } return PACKET_MAX; } static void rtw89_core_tx_update_llc_hdr(struct rtw89_dev *rtwdev, struct rtw89_tx_desc_info *desc_info, struct sk_buff *skb) { struct ieee80211_hdr *hdr = (void *)skb->data; __le16 fc = hdr->frame_control; desc_info->hdr_llc_len = ieee80211_hdrlen(fc); desc_info->hdr_llc_len >>= 1; /* in unit of 2 bytes */ } static void rtw89_core_tx_wake(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req) { if (!RTW89_CHK_FW_FEATURE(TX_WAKE, &rtwdev->fw)) return; if (!test_bit(RTW89_FLAG_LOW_POWER_MODE, rtwdev->flags)) return; if (tx_req->tx_type != RTW89_CORE_TX_TYPE_MGMT) return; rtw89_mac_notify_wake(rtwdev); } static void rtw89_core_tx_update_desc_info(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req) { struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; struct sk_buff *skb = tx_req->skb; struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); struct ieee80211_hdr *hdr = (void *)skb->data; enum rtw89_core_tx_type tx_type; enum btc_pkt_type pkt_type; bool is_bmc; u16 seq; seq = (le16_to_cpu(hdr->seq_ctrl) & IEEE80211_SCTL_SEQ) >> 4; if (tx_req->tx_type != RTW89_CORE_TX_TYPE_FWCMD) { tx_type = rtw89_core_get_tx_type(rtwdev, skb); tx_req->tx_type = tx_type; } is_bmc = (is_broadcast_ether_addr(hdr->addr1) || is_multicast_ether_addr(hdr->addr1)); desc_info->seq = seq; desc_info->pkt_size = skb->len; desc_info->is_bmc = is_bmc; desc_info->wd_page = true; desc_info->hiq = info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM; switch (tx_req->tx_type) { case RTW89_CORE_TX_TYPE_MGMT: rtw89_core_tx_update_mgmt_info(rtwdev, tx_req); break; case RTW89_CORE_TX_TYPE_DATA: rtw89_core_tx_update_data_info(rtwdev, tx_req); pkt_type = rtw89_core_tx_btc_spec_pkt_notify(rtwdev, tx_req); rtw89_core_tx_update_he_qos_htc(rtwdev, tx_req, pkt_type); rtw89_core_tx_update_ampdu_info(rtwdev, tx_req, pkt_type); rtw89_core_tx_update_llc_hdr(rtwdev, desc_info, skb); break; case RTW89_CORE_TX_TYPE_FWCMD: rtw89_core_tx_update_h2c_info(rtwdev, tx_req); break; } } void rtw89_core_tx_kick_off(struct rtw89_dev *rtwdev, u8 qsel) { u8 ch_dma; ch_dma = rtw89_core_get_ch_dma(rtwdev, qsel); rtw89_hci_tx_kick_off(rtwdev, ch_dma); } int rtw89_h2c_tx(struct rtw89_dev *rtwdev, struct sk_buff *skb, bool fwdl) { struct rtw89_core_tx_request tx_req = {0}; u32 cnt; int ret; if (!test_bit(RTW89_FLAG_POWERON, rtwdev->flags)) { rtw89_debug(rtwdev, RTW89_DBG_FW, "ignore h2c due to power is off with firmware state=%d\n", test_bit(RTW89_FLAG_FW_RDY, rtwdev->flags)); return 0; } tx_req.skb = skb; tx_req.tx_type = RTW89_CORE_TX_TYPE_FWCMD; if (fwdl) tx_req.desc_info.fw_dl = true; rtw89_core_tx_update_desc_info(rtwdev, &tx_req); if (!fwdl) rtw89_hex_dump(rtwdev, RTW89_DBG_FW, "H2C: ", skb->data, skb->len); cnt = rtw89_hci_check_and_reclaim_tx_resource(rtwdev, RTW89_TXCH_CH12); if (cnt == 0) { rtw89_err(rtwdev, "no tx fwcmd resource\n"); return -ENOSPC; } ret = rtw89_hci_tx_write(rtwdev, &tx_req); if (ret) { rtw89_err(rtwdev, "failed to transmit skb to HCI\n"); return ret; } rtw89_hci_tx_kick_off(rtwdev, RTW89_TXCH_CH12); return 0; } int rtw89_core_tx_write(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, struct ieee80211_sta *sta, struct sk_buff *skb, int *qsel) { struct rtw89_core_tx_request tx_req = {0}; struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; int ret; tx_req.skb = skb; tx_req.sta = sta; tx_req.vif = vif; rtw89_traffic_stats_accu(rtwdev, &rtwdev->stats, skb, true); rtw89_traffic_stats_accu(rtwdev, &rtwvif->stats, skb, true); rtw89_core_tx_update_desc_info(rtwdev, &tx_req); rtw89_core_tx_wake(rtwdev, &tx_req); ret = rtw89_hci_tx_write(rtwdev, &tx_req); if (ret) { rtw89_err(rtwdev, "failed to transmit skb to HCI\n"); return ret; } if (qsel) *qsel = tx_req.desc_info.qsel; return 0; } static __le32 rtw89_build_txwd_body0(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_BODY0_WP_OFFSET, desc_info->wp_offset) | FIELD_PREP(RTW89_TXWD_BODY0_WD_INFO_EN, desc_info->en_wd_info) | FIELD_PREP(RTW89_TXWD_BODY0_CHANNEL_DMA, desc_info->ch_dma) | FIELD_PREP(RTW89_TXWD_BODY0_HDR_LLC_LEN, desc_info->hdr_llc_len) | FIELD_PREP(RTW89_TXWD_BODY0_WD_PAGE, desc_info->wd_page) | FIELD_PREP(RTW89_TXWD_BODY0_FW_DL, desc_info->fw_dl) | FIELD_PREP(RTW89_TXWD_BODY0_HW_SSN_SEL, desc_info->hw_ssn_sel) | FIELD_PREP(RTW89_TXWD_BODY0_HW_SSN_MODE, desc_info->hw_seq_mode); return cpu_to_le32(dword); } static __le32 rtw89_build_txwd_body0_v1(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_BODY0_WP_OFFSET_V1, desc_info->wp_offset) | FIELD_PREP(RTW89_TXWD_BODY0_WD_INFO_EN, desc_info->en_wd_info) | FIELD_PREP(RTW89_TXWD_BODY0_CHANNEL_DMA, desc_info->ch_dma) | FIELD_PREP(RTW89_TXWD_BODY0_HDR_LLC_LEN, desc_info->hdr_llc_len) | FIELD_PREP(RTW89_TXWD_BODY0_WD_PAGE, desc_info->wd_page) | FIELD_PREP(RTW89_TXWD_BODY0_FW_DL, desc_info->fw_dl); return cpu_to_le32(dword); } static __le32 rtw89_build_txwd_body1_v1(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_BODY1_ADDR_INFO_NUM, desc_info->addr_info_nr) | FIELD_PREP(RTW89_TXWD_BODY1_SEC_KEYID, desc_info->sec_keyid) | FIELD_PREP(RTW89_TXWD_BODY1_SEC_TYPE, desc_info->sec_type); return cpu_to_le32(dword); } static __le32 rtw89_build_txwd_body2(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_BODY2_TID_INDICATE, desc_info->tid_indicate) | FIELD_PREP(RTW89_TXWD_BODY2_QSEL, desc_info->qsel) | FIELD_PREP(RTW89_TXWD_BODY2_TXPKT_SIZE, desc_info->pkt_size) | FIELD_PREP(RTW89_TXWD_BODY2_MACID, desc_info->mac_id); return cpu_to_le32(dword); } static __le32 rtw89_build_txwd_body3(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_BODY3_SW_SEQ, desc_info->seq) | FIELD_PREP(RTW89_TXWD_BODY3_AGG_EN, desc_info->agg_en) | FIELD_PREP(RTW89_TXWD_BODY3_BK, desc_info->bk); return cpu_to_le32(dword); } static __le32 rtw89_build_txwd_body4(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_BODY4_SEC_IV_L0, desc_info->sec_seq[0]) | FIELD_PREP(RTW89_TXWD_BODY4_SEC_IV_L1, desc_info->sec_seq[1]); return cpu_to_le32(dword); } static __le32 rtw89_build_txwd_body5(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_BODY5_SEC_IV_H2, desc_info->sec_seq[2]) | FIELD_PREP(RTW89_TXWD_BODY5_SEC_IV_H3, desc_info->sec_seq[3]) | FIELD_PREP(RTW89_TXWD_BODY5_SEC_IV_H4, desc_info->sec_seq[4]) | FIELD_PREP(RTW89_TXWD_BODY5_SEC_IV_H5, desc_info->sec_seq[5]); return cpu_to_le32(dword); } static __le32 rtw89_build_txwd_body7_v1(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_BODY7_USE_RATE_V1, desc_info->use_rate) | FIELD_PREP(RTW89_TXWD_BODY7_DATA_RATE, desc_info->data_rate); return cpu_to_le32(dword); } static __le32 rtw89_build_txwd_info0(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_INFO0_USE_RATE, desc_info->use_rate) | FIELD_PREP(RTW89_TXWD_INFO0_DATA_RATE, desc_info->data_rate) | FIELD_PREP(RTW89_TXWD_INFO0_DISDATAFB, desc_info->dis_data_fb) | FIELD_PREP(RTW89_TXWD_INFO0_MULTIPORT_ID, desc_info->port); return cpu_to_le32(dword); } static __le32 rtw89_build_txwd_info0_v1(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_INFO0_DISDATAFB, desc_info->dis_data_fb); return cpu_to_le32(dword); } static __le32 rtw89_build_txwd_info1(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_INFO1_MAX_AGGNUM, desc_info->ampdu_num) | FIELD_PREP(RTW89_TXWD_INFO1_A_CTRL_BSR, desc_info->a_ctrl_bsr) | FIELD_PREP(RTW89_TXWD_INFO1_DATA_RTY_LOWEST_RATE, desc_info->data_retry_lowest_rate); return cpu_to_le32(dword); } static __le32 rtw89_build_txwd_info2(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_INFO2_AMPDU_DENSITY, desc_info->ampdu_density) | FIELD_PREP(RTW89_TXWD_INFO2_SEC_TYPE, desc_info->sec_type) | FIELD_PREP(RTW89_TXWD_INFO2_SEC_HW_ENC, desc_info->sec_en) | FIELD_PREP(RTW89_TXWD_INFO2_SEC_CAM_IDX, desc_info->sec_cam_idx); return cpu_to_le32(dword); } static __le32 rtw89_build_txwd_info2_v1(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_INFO2_AMPDU_DENSITY, desc_info->ampdu_density) | FIELD_PREP(RTW89_TXWD_INFO2_FORCE_KEY_EN, desc_info->sec_en) | FIELD_PREP(RTW89_TXWD_INFO2_SEC_CAM_IDX, desc_info->sec_cam_idx); return cpu_to_le32(dword); } static __le32 rtw89_build_txwd_info4(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(RTW89_TXWD_INFO4_RTS_EN, 1) | FIELD_PREP(RTW89_TXWD_INFO4_HW_RTS_EN, 1); return cpu_to_le32(dword); } void rtw89_core_fill_txdesc(struct rtw89_dev *rtwdev, struct rtw89_tx_desc_info *desc_info, void *txdesc) { struct rtw89_txwd_body *txwd_body = (struct rtw89_txwd_body *)txdesc; struct rtw89_txwd_info *txwd_info; txwd_body->dword0 = rtw89_build_txwd_body0(desc_info); txwd_body->dword2 = rtw89_build_txwd_body2(desc_info); txwd_body->dword3 = rtw89_build_txwd_body3(desc_info); if (!desc_info->en_wd_info) return; txwd_info = (struct rtw89_txwd_info *)(txwd_body + 1); txwd_info->dword0 = rtw89_build_txwd_info0(desc_info); txwd_info->dword1 = rtw89_build_txwd_info1(desc_info); txwd_info->dword2 = rtw89_build_txwd_info2(desc_info); txwd_info->dword4 = rtw89_build_txwd_info4(desc_info); } EXPORT_SYMBOL(rtw89_core_fill_txdesc); void rtw89_core_fill_txdesc_v1(struct rtw89_dev *rtwdev, struct rtw89_tx_desc_info *desc_info, void *txdesc) { struct rtw89_txwd_body_v1 *txwd_body = (struct rtw89_txwd_body_v1 *)txdesc; struct rtw89_txwd_info *txwd_info; txwd_body->dword0 = rtw89_build_txwd_body0_v1(desc_info); txwd_body->dword1 = rtw89_build_txwd_body1_v1(desc_info); txwd_body->dword2 = rtw89_build_txwd_body2(desc_info); txwd_body->dword3 = rtw89_build_txwd_body3(desc_info); if (desc_info->sec_en) { txwd_body->dword4 = rtw89_build_txwd_body4(desc_info); txwd_body->dword5 = rtw89_build_txwd_body5(desc_info); } txwd_body->dword7 = rtw89_build_txwd_body7_v1(desc_info); if (!desc_info->en_wd_info) return; txwd_info = (struct rtw89_txwd_info *)(txwd_body + 1); txwd_info->dword0 = rtw89_build_txwd_info0_v1(desc_info); txwd_info->dword1 = rtw89_build_txwd_info1(desc_info); txwd_info->dword2 = rtw89_build_txwd_info2_v1(desc_info); txwd_info->dword4 = rtw89_build_txwd_info4(desc_info); } EXPORT_SYMBOL(rtw89_core_fill_txdesc_v1); static __le32 rtw89_build_txwd_fwcmd0_v1(struct rtw89_tx_desc_info *desc_info) { u32 dword = FIELD_PREP(AX_RXD_RPKT_LEN_MASK, desc_info->pkt_size) | FIELD_PREP(AX_RXD_RPKT_TYPE_MASK, desc_info->fw_dl ? RTW89_CORE_RX_TYPE_FWDL : RTW89_CORE_RX_TYPE_H2C); return cpu_to_le32(dword); } void rtw89_core_fill_txdesc_fwcmd_v1(struct rtw89_dev *rtwdev, struct rtw89_tx_desc_info *desc_info, void *txdesc) { struct rtw89_rxdesc_short *txwd_v1 = (struct rtw89_rxdesc_short *)txdesc; txwd_v1->dword0 = rtw89_build_txwd_fwcmd0_v1(desc_info); } EXPORT_SYMBOL(rtw89_core_fill_txdesc_fwcmd_v1); static int rtw89_core_rx_process_mac_ppdu(struct rtw89_dev *rtwdev, struct sk_buff *skb, struct rtw89_rx_phy_ppdu *phy_ppdu) { bool rx_cnt_valid = false; u8 plcp_size = 0; u8 usr_num = 0; u8 *phy_sts; rx_cnt_valid = RTW89_GET_RXINFO_RX_CNT_VLD(skb->data); plcp_size = RTW89_GET_RXINFO_PLCP_LEN(skb->data) << 3; usr_num = RTW89_GET_RXINFO_USR_NUM(skb->data); if (usr_num > RTW89_PPDU_MAX_USR) { rtw89_warn(rtwdev, "Invalid user number in mac info\n"); return -EINVAL; } phy_sts = skb->data + RTW89_PPDU_MAC_INFO_SIZE; phy_sts += usr_num * RTW89_PPDU_MAC_INFO_USR_SIZE; /* 8-byte alignment */ if (usr_num & BIT(0)) phy_sts += RTW89_PPDU_MAC_INFO_USR_SIZE; if (rx_cnt_valid) phy_sts += RTW89_PPDU_MAC_RX_CNT_SIZE; phy_sts += plcp_size; phy_ppdu->buf = phy_sts; phy_ppdu->len = skb->data + skb->len - phy_sts; return 0; } static void rtw89_core_rx_process_phy_ppdu_iter(void *data, struct ieee80211_sta *sta) { struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; struct rtw89_rx_phy_ppdu *phy_ppdu = (struct rtw89_rx_phy_ppdu *)data; if (rtwsta->mac_id == phy_ppdu->mac_id && phy_ppdu->to_self) ewma_rssi_add(&rtwsta->avg_rssi, phy_ppdu->rssi_avg); } #define VAR_LEN 0xff #define VAR_LEN_UNIT 8 static u16 rtw89_core_get_phy_status_ie_len(struct rtw89_dev *rtwdev, u8 *addr) { static const u8 physts_ie_len_tab[32] = { 16, 32, 24, 24, 8, 8, 8, 8, VAR_LEN, 8, VAR_LEN, 176, VAR_LEN, VAR_LEN, VAR_LEN, VAR_LEN, VAR_LEN, VAR_LEN, 16, 24, VAR_LEN, VAR_LEN, VAR_LEN, 0, 24, 24, 24, 24, 32, 32, 32, 32 }; u16 ie_len; u8 ie; ie = RTW89_GET_PHY_STS_IE_TYPE(addr); if (physts_ie_len_tab[ie] != VAR_LEN) ie_len = physts_ie_len_tab[ie]; else ie_len = RTW89_GET_PHY_STS_IE_LEN(addr) * VAR_LEN_UNIT; return ie_len; } static void rtw89_core_parse_phy_status_ie01(struct rtw89_dev *rtwdev, u8 *addr, struct rtw89_rx_phy_ppdu *phy_ppdu) { s16 cfo; phy_ppdu->chan_idx = RTW89_GET_PHY_STS_IE01_CH_IDX(addr); if (phy_ppdu->rate < RTW89_HW_RATE_OFDM6) return; /* sign conversion for S(12,2) */ cfo = sign_extend32(RTW89_GET_PHY_STS_IE01_CFO(addr), 11); rtw89_phy_cfo_parse(rtwdev, cfo, phy_ppdu); } static int rtw89_core_process_phy_status_ie(struct rtw89_dev *rtwdev, u8 *addr, struct rtw89_rx_phy_ppdu *phy_ppdu) { u8 ie; ie = RTW89_GET_PHY_STS_IE_TYPE(addr); switch (ie) { case RTW89_PHYSTS_IE01_CMN_OFDM: rtw89_core_parse_phy_status_ie01(rtwdev, addr, phy_ppdu); break; default: break; } return 0; } static void rtw89_core_update_phy_ppdu(struct rtw89_rx_phy_ppdu *phy_ppdu) { s8 *rssi = phy_ppdu->rssi; u8 *buf = phy_ppdu->buf; phy_ppdu->ie = RTW89_GET_PHY_STS_IE_MAP(buf); phy_ppdu->rssi_avg = RTW89_GET_PHY_STS_RSSI_AVG(buf); rssi[RF_PATH_A] = RTW89_RSSI_RAW_TO_DBM(RTW89_GET_PHY_STS_RSSI_A(buf)); rssi[RF_PATH_B] = RTW89_RSSI_RAW_TO_DBM(RTW89_GET_PHY_STS_RSSI_B(buf)); rssi[RF_PATH_C] = RTW89_RSSI_RAW_TO_DBM(RTW89_GET_PHY_STS_RSSI_C(buf)); rssi[RF_PATH_D] = RTW89_RSSI_RAW_TO_DBM(RTW89_GET_PHY_STS_RSSI_D(buf)); } static int rtw89_core_rx_process_phy_ppdu(struct rtw89_dev *rtwdev, struct rtw89_rx_phy_ppdu *phy_ppdu) { if (RTW89_GET_PHY_STS_LEN(phy_ppdu->buf) << 3 != phy_ppdu->len) { rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "phy ppdu len mismatch\n"); return -EINVAL; } rtw89_core_update_phy_ppdu(phy_ppdu); ieee80211_iterate_stations_atomic(rtwdev->hw, rtw89_core_rx_process_phy_ppdu_iter, phy_ppdu); return 0; } static int rtw89_core_rx_parse_phy_sts(struct rtw89_dev *rtwdev, struct rtw89_rx_phy_ppdu *phy_ppdu) { u16 ie_len; u8 *pos, *end; /* mark invalid reports and bypass them */ if (phy_ppdu->ie < RTW89_CCK_PKT) return -EINVAL; pos = (u8 *)phy_ppdu->buf + PHY_STS_HDR_LEN; end = (u8 *)phy_ppdu->buf + phy_ppdu->len; while (pos < end) { ie_len = rtw89_core_get_phy_status_ie_len(rtwdev, pos); rtw89_core_process_phy_status_ie(rtwdev, pos, phy_ppdu); pos += ie_len; if (pos > end || ie_len == 0) { rtw89_debug(rtwdev, RTW89_DBG_TXRX, "phy status parse failed\n"); return -EINVAL; } } return 0; } static void rtw89_core_rx_process_phy_sts(struct rtw89_dev *rtwdev, struct rtw89_rx_phy_ppdu *phy_ppdu) { int ret; ret = rtw89_core_rx_parse_phy_sts(rtwdev, phy_ppdu); if (ret) rtw89_debug(rtwdev, RTW89_DBG_TXRX, "parse phy sts failed\n"); else phy_ppdu->valid = true; } static u8 rtw89_rxdesc_to_nl_he_gi(struct rtw89_dev *rtwdev, const struct rtw89_rx_desc_info *desc_info, bool rx_status) { switch (desc_info->gi_ltf) { case RTW89_GILTF_SGI_4XHE08: case RTW89_GILTF_2XHE08: case RTW89_GILTF_1XHE08: return NL80211_RATE_INFO_HE_GI_0_8; case RTW89_GILTF_2XHE16: case RTW89_GILTF_1XHE16: return NL80211_RATE_INFO_HE_GI_1_6; case RTW89_GILTF_LGI_4XHE32: return NL80211_RATE_INFO_HE_GI_3_2; default: rtw89_warn(rtwdev, "invalid gi_ltf=%d", desc_info->gi_ltf); return rx_status ? NL80211_RATE_INFO_HE_GI_3_2 : U8_MAX; } } static bool rtw89_core_rx_ppdu_match(struct rtw89_dev *rtwdev, struct rtw89_rx_desc_info *desc_info, struct ieee80211_rx_status *status) { u8 band = desc_info->bb_sel ? RTW89_PHY_1 : RTW89_PHY_0; u8 data_rate_mode, bw, rate_idx = MASKBYTE0, gi_ltf; u16 data_rate; bool ret; data_rate = desc_info->data_rate; data_rate_mode = GET_DATA_RATE_MODE(data_rate); if (data_rate_mode == DATA_RATE_MODE_NON_HT) { rate_idx = GET_DATA_RATE_NOT_HT_IDX(data_rate); /* rate_idx is still hardware value here */ } else if (data_rate_mode == DATA_RATE_MODE_HT) { rate_idx = GET_DATA_RATE_HT_IDX(data_rate); } else if (data_rate_mode == DATA_RATE_MODE_VHT) { rate_idx = GET_DATA_RATE_VHT_HE_IDX(data_rate); } else if (data_rate_mode == DATA_RATE_MODE_HE) { rate_idx = GET_DATA_RATE_VHT_HE_IDX(data_rate); } else { rtw89_warn(rtwdev, "invalid RX rate mode %d\n", data_rate_mode); } bw = rtw89_hw_to_rate_info_bw(desc_info->bw); gi_ltf = rtw89_rxdesc_to_nl_he_gi(rtwdev, desc_info, false); ret = rtwdev->ppdu_sts.curr_rx_ppdu_cnt[band] == desc_info->ppdu_cnt && status->rate_idx == rate_idx && status->he_gi == gi_ltf && status->bw == bw; return ret; } struct rtw89_vif_rx_stats_iter_data { struct rtw89_dev *rtwdev; struct rtw89_rx_phy_ppdu *phy_ppdu; struct rtw89_rx_desc_info *desc_info; struct sk_buff *skb; const u8 *bssid; }; static void rtw89_stats_trigger_frame(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, struct sk_buff *skb) { struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; struct ieee80211_trigger *tf = (struct ieee80211_trigger *)skb->data; u8 *pos, *end, type; u16 aid; if (!ether_addr_equal(vif->bss_conf.bssid, tf->ta) || rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION || rtwvif->net_type == RTW89_NET_TYPE_NO_LINK) return; type = le64_get_bits(tf->common_info, IEEE80211_TRIGGER_TYPE_MASK); if (type != IEEE80211_TRIGGER_TYPE_BASIC) return; end = (u8 *)tf + skb->len; pos = tf->variable; while (end - pos >= RTW89_TF_BASIC_USER_INFO_SZ) { aid = RTW89_GET_TF_USER_INFO_AID12(pos); rtw89_debug(rtwdev, RTW89_DBG_TXRX, "[TF] aid: %d, ul_mcs: %d, rua: %d\n", aid, RTW89_GET_TF_USER_INFO_UL_MCS(pos), RTW89_GET_TF_USER_INFO_RUA(pos)); if (aid == RTW89_TF_PAD) break; if (aid == vif->cfg.aid) { rtwvif->stats.rx_tf_acc++; rtwdev->stats.rx_tf_acc++; break; } pos += RTW89_TF_BASIC_USER_INFO_SZ; } } static void rtw89_vif_rx_stats_iter(void *data, u8 *mac, struct ieee80211_vif *vif) { struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; struct rtw89_vif_rx_stats_iter_data *iter_data = data; struct rtw89_dev *rtwdev = iter_data->rtwdev; struct rtw89_pkt_stat *pkt_stat = &rtwdev->phystat.cur_pkt_stat; struct rtw89_rx_desc_info *desc_info = iter_data->desc_info; struct sk_buff *skb = iter_data->skb; struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; const u8 *bssid = iter_data->bssid; if (ieee80211_is_trigger(hdr->frame_control)) { rtw89_stats_trigger_frame(rtwdev, vif, skb); return; } if (!ether_addr_equal(vif->bss_conf.bssid, bssid)) return; if (ieee80211_is_beacon(hdr->frame_control)) pkt_stat->beacon_nr++; if (!ether_addr_equal(vif->addr, hdr->addr1)) return; if (desc_info->data_rate < RTW89_HW_RATE_NR) pkt_stat->rx_rate_cnt[desc_info->data_rate]++; rtw89_traffic_stats_accu(rtwdev, &rtwvif->stats, skb, false); } static void rtw89_core_rx_stats(struct rtw89_dev *rtwdev, struct rtw89_rx_phy_ppdu *phy_ppdu, struct rtw89_rx_desc_info *desc_info, struct sk_buff *skb) { struct rtw89_vif_rx_stats_iter_data iter_data; rtw89_traffic_stats_accu(rtwdev, &rtwdev->stats, skb, false); iter_data.rtwdev = rtwdev; iter_data.phy_ppdu = phy_ppdu; iter_data.desc_info = desc_info; iter_data.skb = skb; iter_data.bssid = get_hdr_bssid((struct ieee80211_hdr *)skb->data); rtw89_iterate_vifs_bh(rtwdev, rtw89_vif_rx_stats_iter, &iter_data); } static void rtw89_correct_cck_chan(struct rtw89_dev *rtwdev, struct ieee80211_rx_status *status) { const struct rtw89_chan_rcd *rcd = rtw89_chan_rcd_get(rtwdev, RTW89_SUB_ENTITY_0); u16 chan = rcd->prev_primary_channel; u8 band = rcd->prev_band_type == RTW89_BAND_2G ? NL80211_BAND_2GHZ : NL80211_BAND_5GHZ; if (status->band != NL80211_BAND_2GHZ && status->encoding == RX_ENC_LEGACY && status->rate_idx < RTW89_HW_RATE_OFDM6) { status->freq = ieee80211_channel_to_frequency(chan, band); status->band = band; } } static void rtw89_core_hw_to_sband_rate(struct ieee80211_rx_status *rx_status) { if (rx_status->band == NL80211_BAND_2GHZ || rx_status->encoding != RX_ENC_LEGACY) return; /* Some control frames' freq(ACKs in this case) are reported wrong due * to FW notify timing, set to lowest rate to prevent overflow. */ if (rx_status->rate_idx < RTW89_HW_RATE_OFDM6) { rx_status->rate_idx = 0; return; } /* No 4 CCK rates for non-2G */ rx_status->rate_idx -= 4; } static void rtw89_core_rx_to_mac80211(struct rtw89_dev *rtwdev, struct rtw89_rx_phy_ppdu *phy_ppdu, struct rtw89_rx_desc_info *desc_info, struct sk_buff *skb_ppdu, struct ieee80211_rx_status *rx_status) { struct napi_struct *napi = &rtwdev->napi; /* In low power mode, napi isn't scheduled. Receive it to netif. */ if (unlikely(!test_bit(NAPI_STATE_SCHED, &napi->state))) napi = NULL; rtw89_core_hw_to_sband_rate(rx_status); rtw89_core_rx_stats(rtwdev, phy_ppdu, desc_info, skb_ppdu); /* In low power mode, it does RX in thread context. */ local_bh_disable(); ieee80211_rx_napi(rtwdev->hw, NULL, skb_ppdu, napi); local_bh_enable(); rtwdev->napi_budget_countdown--; } static void rtw89_core_rx_pending_skb(struct rtw89_dev *rtwdev, struct rtw89_rx_phy_ppdu *phy_ppdu, struct rtw89_rx_desc_info *desc_info, struct sk_buff *skb) { u8 band = desc_info->bb_sel ? RTW89_PHY_1 : RTW89_PHY_0; int curr = rtwdev->ppdu_sts.curr_rx_ppdu_cnt[band]; struct sk_buff *skb_ppdu = NULL, *tmp; struct ieee80211_rx_status *rx_status; if (curr > RTW89_MAX_PPDU_CNT) return; skb_queue_walk_safe(&rtwdev->ppdu_sts.rx_queue[band], skb_ppdu, tmp) { skb_unlink(skb_ppdu, &rtwdev->ppdu_sts.rx_queue[band]); rx_status = IEEE80211_SKB_RXCB(skb_ppdu); if (rtw89_core_rx_ppdu_match(rtwdev, desc_info, rx_status)) rtw89_chip_query_ppdu(rtwdev, phy_ppdu, rx_status); rtw89_correct_cck_chan(rtwdev, rx_status); rtw89_core_rx_to_mac80211(rtwdev, phy_ppdu, desc_info, skb_ppdu, rx_status); } } static void rtw89_core_rx_process_ppdu_sts(struct rtw89_dev *rtwdev, struct rtw89_rx_desc_info *desc_info, struct sk_buff *skb) { struct rtw89_rx_phy_ppdu phy_ppdu = {.buf = skb->data, .valid = false, .len = skb->len, .to_self = desc_info->addr1_match, .rate = desc_info->data_rate, .mac_id = desc_info->mac_id}; int ret; if (desc_info->mac_info_valid) rtw89_core_rx_process_mac_ppdu(rtwdev, skb, &phy_ppdu); ret = rtw89_core_rx_process_phy_ppdu(rtwdev, &phy_ppdu); if (ret) rtw89_debug(rtwdev, RTW89_DBG_TXRX, "process ppdu failed\n"); rtw89_core_rx_process_phy_sts(rtwdev, &phy_ppdu); rtw89_core_rx_pending_skb(rtwdev, &phy_ppdu, desc_info, skb); dev_kfree_skb_any(skb); } static void rtw89_core_rx_process_report(struct rtw89_dev *rtwdev, struct rtw89_rx_desc_info *desc_info, struct sk_buff *skb) { switch (desc_info->pkt_type) { case RTW89_CORE_RX_TYPE_C2H: rtw89_fw_c2h_irqsafe(rtwdev, skb); break; case RTW89_CORE_RX_TYPE_PPDU_STAT: rtw89_core_rx_process_ppdu_sts(rtwdev, desc_info, skb); break; default: rtw89_debug(rtwdev, RTW89_DBG_TXRX, "unhandled pkt_type=%d\n", desc_info->pkt_type); dev_kfree_skb_any(skb); break; } } void rtw89_core_query_rxdesc(struct rtw89_dev *rtwdev, struct rtw89_rx_desc_info *desc_info, u8 *data, u32 data_offset) { const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_rxdesc_short *rxd_s; struct rtw89_rxdesc_long *rxd_l; u8 shift_len, drv_info_len; rxd_s = (struct rtw89_rxdesc_short *)(data + data_offset); desc_info->pkt_size = RTW89_GET_RXWD_PKT_SIZE(rxd_s); desc_info->drv_info_size = RTW89_GET_RXWD_DRV_INFO_SIZE(rxd_s); desc_info->long_rxdesc = RTW89_GET_RXWD_LONG_RXD(rxd_s); desc_info->pkt_type = RTW89_GET_RXWD_RPKT_TYPE(rxd_s); desc_info->mac_info_valid = RTW89_GET_RXWD_MAC_INFO_VALID(rxd_s); if (chip->chip_id == RTL8852C) desc_info->bw = RTW89_GET_RXWD_BW_V1(rxd_s); else desc_info->bw = RTW89_GET_RXWD_BW(rxd_s); desc_info->data_rate = RTW89_GET_RXWD_DATA_RATE(rxd_s); desc_info->gi_ltf = RTW89_GET_RXWD_GI_LTF(rxd_s); desc_info->user_id = RTW89_GET_RXWD_USER_ID(rxd_s); desc_info->sr_en = RTW89_GET_RXWD_SR_EN(rxd_s); desc_info->ppdu_cnt = RTW89_GET_RXWD_PPDU_CNT(rxd_s); desc_info->ppdu_type = RTW89_GET_RXWD_PPDU_TYPE(rxd_s); desc_info->free_run_cnt = RTW89_GET_RXWD_FREE_RUN_CNT(rxd_s); desc_info->icv_err = RTW89_GET_RXWD_ICV_ERR(rxd_s); desc_info->crc32_err = RTW89_GET_RXWD_CRC32_ERR(rxd_s); desc_info->hw_dec = RTW89_GET_RXWD_HW_DEC(rxd_s); desc_info->sw_dec = RTW89_GET_RXWD_SW_DEC(rxd_s); desc_info->addr1_match = RTW89_GET_RXWD_A1_MATCH(rxd_s); shift_len = desc_info->shift << 1; /* 2-byte unit */ drv_info_len = desc_info->drv_info_size << 3; /* 8-byte unit */ desc_info->offset = data_offset + shift_len + drv_info_len; desc_info->ready = true; if (!desc_info->long_rxdesc) return; rxd_l = (struct rtw89_rxdesc_long *)(data + data_offset); desc_info->frame_type = RTW89_GET_RXWD_TYPE(rxd_l); desc_info->addr_cam_valid = RTW89_GET_RXWD_ADDR_CAM_VLD(rxd_l); desc_info->addr_cam_id = RTW89_GET_RXWD_ADDR_CAM_ID(rxd_l); desc_info->sec_cam_id = RTW89_GET_RXWD_SEC_CAM_ID(rxd_l); desc_info->mac_id = RTW89_GET_RXWD_MAC_ID(rxd_l); desc_info->rx_pl_id = RTW89_GET_RXWD_RX_PL_ID(rxd_l); } EXPORT_SYMBOL(rtw89_core_query_rxdesc); struct rtw89_core_iter_rx_status { struct rtw89_dev *rtwdev; struct ieee80211_rx_status *rx_status; struct rtw89_rx_desc_info *desc_info; u8 mac_id; }; static void rtw89_core_stats_sta_rx_status_iter(void *data, struct ieee80211_sta *sta) { struct rtw89_core_iter_rx_status *iter_data = (struct rtw89_core_iter_rx_status *)data; struct ieee80211_rx_status *rx_status = iter_data->rx_status; struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; struct rtw89_rx_desc_info *desc_info = iter_data->desc_info; u8 mac_id = iter_data->mac_id; if (mac_id != rtwsta->mac_id) return; rtwsta->rx_status = *rx_status; rtwsta->rx_hw_rate = desc_info->data_rate; } static void rtw89_core_stats_sta_rx_status(struct rtw89_dev *rtwdev, struct rtw89_rx_desc_info *desc_info, struct ieee80211_rx_status *rx_status) { struct rtw89_core_iter_rx_status iter_data; if (!desc_info->addr1_match || !desc_info->long_rxdesc) return; if (desc_info->frame_type != RTW89_RX_TYPE_DATA) return; iter_data.rtwdev = rtwdev; iter_data.rx_status = rx_status; iter_data.desc_info = desc_info; iter_data.mac_id = desc_info->mac_id; ieee80211_iterate_stations_atomic(rtwdev->hw, rtw89_core_stats_sta_rx_status_iter, &iter_data); } static void rtw89_core_update_rx_status(struct rtw89_dev *rtwdev, struct rtw89_rx_desc_info *desc_info, struct ieee80211_rx_status *rx_status) { const struct cfg80211_chan_def *chandef = rtw89_chandef_get(rtwdev, RTW89_SUB_ENTITY_0); const struct rtw89_chan *cur = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); u16 data_rate; u8 data_rate_mode; /* currently using single PHY */ rx_status->freq = chandef->chan->center_freq; rx_status->band = chandef->chan->band; if (rtwdev->scanning && RTW89_CHK_FW_FEATURE(SCAN_OFFLOAD, &rtwdev->fw)) { u8 chan = cur->primary_channel; u8 band = cur->band_type; enum nl80211_band nl_band; nl_band = rtw89_hw_to_nl80211_band(band); rx_status->freq = ieee80211_channel_to_frequency(chan, nl_band); rx_status->band = nl_band; } if (desc_info->icv_err || desc_info->crc32_err) rx_status->flag |= RX_FLAG_FAILED_FCS_CRC; if (desc_info->hw_dec && !(desc_info->sw_dec || desc_info->icv_err)) rx_status->flag |= RX_FLAG_DECRYPTED; rx_status->bw = rtw89_hw_to_rate_info_bw(desc_info->bw); data_rate = desc_info->data_rate; data_rate_mode = GET_DATA_RATE_MODE(data_rate); if (data_rate_mode == DATA_RATE_MODE_NON_HT) { rx_status->encoding = RX_ENC_LEGACY; rx_status->rate_idx = GET_DATA_RATE_NOT_HT_IDX(data_rate); /* convert rate_idx after we get the correct band */ } else if (data_rate_mode == DATA_RATE_MODE_HT) { rx_status->encoding = RX_ENC_HT; rx_status->rate_idx = GET_DATA_RATE_HT_IDX(data_rate); if (desc_info->gi_ltf) rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI; } else if (data_rate_mode == DATA_RATE_MODE_VHT) { rx_status->encoding = RX_ENC_VHT; rx_status->rate_idx = GET_DATA_RATE_VHT_HE_IDX(data_rate); rx_status->nss = GET_DATA_RATE_NSS(data_rate) + 1; if (desc_info->gi_ltf) rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI; } else if (data_rate_mode == DATA_RATE_MODE_HE) { rx_status->encoding = RX_ENC_HE; rx_status->rate_idx = GET_DATA_RATE_VHT_HE_IDX(data_rate); rx_status->nss = GET_DATA_RATE_NSS(data_rate) + 1; } else { rtw89_warn(rtwdev, "invalid RX rate mode %d\n", data_rate_mode); } /* he_gi is used to match ppdu, so we always fill it. */ rx_status->he_gi = rtw89_rxdesc_to_nl_he_gi(rtwdev, desc_info, true); rx_status->flag |= RX_FLAG_MACTIME_START; rx_status->mactime = desc_info->free_run_cnt; rtw89_core_stats_sta_rx_status(rtwdev, desc_info, rx_status); } static enum rtw89_ps_mode rtw89_update_ps_mode(struct rtw89_dev *rtwdev) { const struct rtw89_chip_info *chip = rtwdev->chip; if (rtw89_disable_ps_mode || !chip->ps_mode_supported) return RTW89_PS_MODE_NONE; if (chip->ps_mode_supported & BIT(RTW89_PS_MODE_PWR_GATED)) return RTW89_PS_MODE_PWR_GATED; if (chip->ps_mode_supported & BIT(RTW89_PS_MODE_CLK_GATED)) return RTW89_PS_MODE_CLK_GATED; if (chip->ps_mode_supported & BIT(RTW89_PS_MODE_RFOFF)) return RTW89_PS_MODE_RFOFF; return RTW89_PS_MODE_NONE; } static void rtw89_core_flush_ppdu_rx_queue(struct rtw89_dev *rtwdev, struct rtw89_rx_desc_info *desc_info) { struct rtw89_ppdu_sts_info *ppdu_sts = &rtwdev->ppdu_sts; u8 band = desc_info->bb_sel ? RTW89_PHY_1 : RTW89_PHY_0; struct ieee80211_rx_status *rx_status; struct sk_buff *skb_ppdu, *tmp; skb_queue_walk_safe(&ppdu_sts->rx_queue[band], skb_ppdu, tmp) { skb_unlink(skb_ppdu, &ppdu_sts->rx_queue[band]); rx_status = IEEE80211_SKB_RXCB(skb_ppdu); rtw89_core_rx_to_mac80211(rtwdev, NULL, desc_info, skb_ppdu, rx_status); } } void rtw89_core_rx(struct rtw89_dev *rtwdev, struct rtw89_rx_desc_info *desc_info, struct sk_buff *skb) { struct ieee80211_rx_status *rx_status; struct rtw89_ppdu_sts_info *ppdu_sts = &rtwdev->ppdu_sts; u8 ppdu_cnt = desc_info->ppdu_cnt; u8 band = desc_info->bb_sel ? RTW89_PHY_1 : RTW89_PHY_0; if (desc_info->pkt_type != RTW89_CORE_RX_TYPE_WIFI) { rtw89_core_rx_process_report(rtwdev, desc_info, skb); return; } if (ppdu_sts->curr_rx_ppdu_cnt[band] != ppdu_cnt) { rtw89_core_flush_ppdu_rx_queue(rtwdev, desc_info); ppdu_sts->curr_rx_ppdu_cnt[band] = ppdu_cnt; } rx_status = IEEE80211_SKB_RXCB(skb); memset(rx_status, 0, sizeof(*rx_status)); rtw89_core_update_rx_status(rtwdev, desc_info, rx_status); if (desc_info->long_rxdesc && BIT(desc_info->frame_type) & PPDU_FILTER_BITMAP) skb_queue_tail(&ppdu_sts->rx_queue[band], skb); else rtw89_core_rx_to_mac80211(rtwdev, NULL, desc_info, skb, rx_status); } EXPORT_SYMBOL(rtw89_core_rx); void rtw89_core_napi_start(struct rtw89_dev *rtwdev) { if (test_and_set_bit(RTW89_FLAG_NAPI_RUNNING, rtwdev->flags)) return; napi_enable(&rtwdev->napi); } EXPORT_SYMBOL(rtw89_core_napi_start); void rtw89_core_napi_stop(struct rtw89_dev *rtwdev) { if (!test_and_clear_bit(RTW89_FLAG_NAPI_RUNNING, rtwdev->flags)) return; napi_synchronize(&rtwdev->napi); napi_disable(&rtwdev->napi); } EXPORT_SYMBOL(rtw89_core_napi_stop); void rtw89_core_napi_init(struct rtw89_dev *rtwdev) { init_dummy_netdev(&rtwdev->netdev); netif_napi_add(&rtwdev->netdev, &rtwdev->napi, - rtwdev->hci.ops->napi_poll, NAPI_POLL_WEIGHT); + rtwdev->hci.ops->napi_poll); } EXPORT_SYMBOL(rtw89_core_napi_init); void rtw89_core_napi_deinit(struct rtw89_dev *rtwdev) { rtw89_core_napi_stop(rtwdev); netif_napi_del(&rtwdev->napi); } EXPORT_SYMBOL(rtw89_core_napi_deinit); static void rtw89_core_ba_work(struct work_struct *work) { struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev, ba_work); struct rtw89_txq *rtwtxq, *tmp; int ret; spin_lock_bh(&rtwdev->ba_lock); list_for_each_entry_safe(rtwtxq, tmp, &rtwdev->ba_list, list) { struct ieee80211_txq *txq = rtw89_txq_to_txq(rtwtxq); struct ieee80211_sta *sta = txq->sta; struct rtw89_sta *rtwsta = sta ? (struct rtw89_sta *)sta->drv_priv : NULL; u8 tid = txq->tid; if (!sta) { rtw89_warn(rtwdev, "cannot start BA without sta\n"); goto skip_ba_work; } if (rtwsta->disassoc) { rtw89_debug(rtwdev, RTW89_DBG_TXRX, "cannot start BA with disassoc sta\n"); goto skip_ba_work; } ret = ieee80211_start_tx_ba_session(sta, tid, 0); if (ret) { rtw89_debug(rtwdev, RTW89_DBG_TXRX, "failed to setup BA session for %pM:%2d: %d\n", sta->addr, tid, ret); if (ret == -EINVAL) set_bit(RTW89_TXQ_F_BLOCK_BA, &rtwtxq->flags); } skip_ba_work: list_del_init(&rtwtxq->list); } spin_unlock_bh(&rtwdev->ba_lock); } static void rtw89_core_free_sta_pending_ba(struct rtw89_dev *rtwdev, struct ieee80211_sta *sta) { struct rtw89_txq *rtwtxq, *tmp; spin_lock_bh(&rtwdev->ba_lock); list_for_each_entry_safe(rtwtxq, tmp, &rtwdev->ba_list, list) { struct ieee80211_txq *txq = rtw89_txq_to_txq(rtwtxq); if (sta == txq->sta) list_del_init(&rtwtxq->list); } spin_unlock_bh(&rtwdev->ba_lock); } static void rtw89_core_free_sta_pending_forbid_ba(struct rtw89_dev *rtwdev, struct ieee80211_sta *sta) { struct rtw89_txq *rtwtxq, *tmp; spin_lock_bh(&rtwdev->ba_lock); list_for_each_entry_safe(rtwtxq, tmp, &rtwdev->forbid_ba_list, list) { struct ieee80211_txq *txq = rtw89_txq_to_txq(rtwtxq); if (sta == txq->sta) { clear_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags); list_del_init(&rtwtxq->list); } } spin_unlock_bh(&rtwdev->ba_lock); } static void rtw89_core_stop_tx_ba_session(struct rtw89_dev *rtwdev, struct rtw89_txq *rtwtxq) { struct ieee80211_txq *txq = rtw89_txq_to_txq(rtwtxq); struct ieee80211_sta *sta = txq->sta; struct rtw89_sta *rtwsta = sta_to_rtwsta_safe(sta); if (unlikely(!rtwsta) || unlikely(rtwsta->disassoc)) return; if (!test_bit(RTW89_TXQ_F_AMPDU, &rtwtxq->flags) || test_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags)) return; spin_lock_bh(&rtwdev->ba_lock); if (!list_empty(&rtwtxq->list)) { list_del_init(&rtwtxq->list); goto out; } set_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags); list_add_tail(&rtwtxq->list, &rtwdev->forbid_ba_list); ieee80211_stop_tx_ba_session(sta, txq->tid); cancel_delayed_work(&rtwdev->forbid_ba_work); ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->forbid_ba_work, RTW89_FORBID_BA_TIMER); out: spin_unlock_bh(&rtwdev->ba_lock); } static void rtw89_core_txq_check_agg(struct rtw89_dev *rtwdev, struct rtw89_txq *rtwtxq, struct sk_buff *skb) { struct ieee80211_hw *hw = rtwdev->hw; struct ieee80211_txq *txq = rtw89_txq_to_txq(rtwtxq); struct ieee80211_sta *sta = txq->sta; struct rtw89_sta *rtwsta = sta ? (struct rtw89_sta *)sta->drv_priv : NULL; if (unlikely(skb->protocol == cpu_to_be16(ETH_P_PAE))) { rtw89_core_stop_tx_ba_session(rtwdev, rtwtxq); return; } if (unlikely(!sta)) return; if (test_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags)) return; if (unlikely(test_bit(RTW89_TXQ_F_BLOCK_BA, &rtwtxq->flags))) return; if (test_bit(RTW89_TXQ_F_AMPDU, &rtwtxq->flags)) { IEEE80211_SKB_CB(skb)->flags |= IEEE80211_TX_CTL_AMPDU; return; } spin_lock_bh(&rtwdev->ba_lock); if (!rtwsta->disassoc && list_empty(&rtwtxq->list)) { list_add_tail(&rtwtxq->list, &rtwdev->ba_list); ieee80211_queue_work(hw, &rtwdev->ba_work); } spin_unlock_bh(&rtwdev->ba_lock); } static void rtw89_core_txq_push(struct rtw89_dev *rtwdev, struct rtw89_txq *rtwtxq, unsigned long frame_cnt, unsigned long byte_cnt) { struct ieee80211_txq *txq = rtw89_txq_to_txq(rtwtxq); struct ieee80211_vif *vif = txq->vif; struct ieee80211_sta *sta = txq->sta; struct sk_buff *skb; unsigned long i; int ret; rcu_read_lock(); for (i = 0; i < frame_cnt; i++) { skb = ieee80211_tx_dequeue_ni(rtwdev->hw, txq); if (!skb) { rtw89_debug(rtwdev, RTW89_DBG_TXRX, "dequeue a NULL skb\n"); goto out; } rtw89_core_txq_check_agg(rtwdev, rtwtxq, skb); ret = rtw89_core_tx_write(rtwdev, vif, sta, skb, NULL); if (ret) { rtw89_err(rtwdev, "failed to push txq: %d\n", ret); ieee80211_free_txskb(rtwdev->hw, skb); break; } } out: rcu_read_unlock(); } static u32 rtw89_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev, u8 tid) { u8 qsel, ch_dma; qsel = rtw89_core_get_qsel(rtwdev, tid); ch_dma = rtw89_core_get_ch_dma(rtwdev, qsel); return rtw89_hci_check_and_reclaim_tx_resource(rtwdev, ch_dma); } static bool rtw89_core_txq_agg_wait(struct rtw89_dev *rtwdev, struct ieee80211_txq *txq, unsigned long *frame_cnt, bool *sched_txq, bool *reinvoke) { struct rtw89_txq *rtwtxq = (struct rtw89_txq *)txq->drv_priv; struct ieee80211_sta *sta = txq->sta; struct rtw89_sta *rtwsta = sta ? (struct rtw89_sta *)sta->drv_priv : NULL; if (!sta || rtwsta->max_agg_wait <= 0) return false; if (rtwdev->stats.tx_tfc_lv <= RTW89_TFC_MID) return false; if (*frame_cnt > 1) { *frame_cnt -= 1; *sched_txq = true; *reinvoke = true; rtwtxq->wait_cnt = 1; return false; } if (*frame_cnt == 1 && rtwtxq->wait_cnt < rtwsta->max_agg_wait) { *reinvoke = true; rtwtxq->wait_cnt++; return true; } rtwtxq->wait_cnt = 0; return false; } static void rtw89_core_txq_schedule(struct rtw89_dev *rtwdev, u8 ac, bool *reinvoke) { struct ieee80211_hw *hw = rtwdev->hw; struct ieee80211_txq *txq; struct rtw89_txq *rtwtxq; unsigned long frame_cnt; unsigned long byte_cnt; u32 tx_resource; bool sched_txq; ieee80211_txq_schedule_start(hw, ac); while ((txq = ieee80211_next_txq(hw, ac))) { rtwtxq = (struct rtw89_txq *)txq->drv_priv; tx_resource = rtw89_check_and_reclaim_tx_resource(rtwdev, txq->tid); sched_txq = false; ieee80211_txq_get_depth(txq, &frame_cnt, &byte_cnt); if (rtw89_core_txq_agg_wait(rtwdev, txq, &frame_cnt, &sched_txq, reinvoke)) { ieee80211_return_txq(hw, txq, true); continue; } frame_cnt = min_t(unsigned long, frame_cnt, tx_resource); rtw89_core_txq_push(rtwdev, rtwtxq, frame_cnt, byte_cnt); ieee80211_return_txq(hw, txq, sched_txq); if (frame_cnt != 0) rtw89_core_tx_kick_off(rtwdev, rtw89_core_get_qsel(rtwdev, txq->tid)); /* bound of tx_resource could get stuck due to burst traffic */ if (frame_cnt == tx_resource) *reinvoke = true; } ieee80211_txq_schedule_end(hw, ac); } static void rtw89_ips_work(struct work_struct *work) { struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev, ips_work); mutex_lock(&rtwdev->mutex); if (rtwdev->hw->conf.flags & IEEE80211_CONF_IDLE) rtw89_enter_ips(rtwdev); mutex_unlock(&rtwdev->mutex); } static void rtw89_core_txq_work(struct work_struct *w) { struct rtw89_dev *rtwdev = container_of(w, struct rtw89_dev, txq_work); bool reinvoke = false; u8 ac; for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) rtw89_core_txq_schedule(rtwdev, ac, &reinvoke); if (reinvoke) { /* reinvoke to process the last frame */ mod_delayed_work(rtwdev->txq_wq, &rtwdev->txq_reinvoke_work, 1); } } static void rtw89_core_txq_reinvoke_work(struct work_struct *w) { struct rtw89_dev *rtwdev = container_of(w, struct rtw89_dev, txq_reinvoke_work.work); queue_work(rtwdev->txq_wq, &rtwdev->txq_work); } static void rtw89_forbid_ba_work(struct work_struct *w) { struct rtw89_dev *rtwdev = container_of(w, struct rtw89_dev, forbid_ba_work.work); struct rtw89_txq *rtwtxq, *tmp; spin_lock_bh(&rtwdev->ba_lock); list_for_each_entry_safe(rtwtxq, tmp, &rtwdev->forbid_ba_list, list) { clear_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags); list_del_init(&rtwtxq->list); } spin_unlock_bh(&rtwdev->ba_lock); } static enum rtw89_tfc_lv rtw89_get_traffic_level(struct rtw89_dev *rtwdev, u32 throughput, u64 cnt) { if (cnt < 100) return RTW89_TFC_IDLE; if (throughput > 50) return RTW89_TFC_HIGH; if (throughput > 10) return RTW89_TFC_MID; if (throughput > 2) return RTW89_TFC_LOW; return RTW89_TFC_ULTRA_LOW; } static bool rtw89_traffic_stats_calc(struct rtw89_dev *rtwdev, struct rtw89_traffic_stats *stats) { enum rtw89_tfc_lv tx_tfc_lv = stats->tx_tfc_lv; enum rtw89_tfc_lv rx_tfc_lv = stats->rx_tfc_lv; stats->tx_throughput_raw = (u32)(stats->tx_unicast >> RTW89_TP_SHIFT); stats->rx_throughput_raw = (u32)(stats->rx_unicast >> RTW89_TP_SHIFT); ewma_tp_add(&stats->tx_ewma_tp, stats->tx_throughput_raw); ewma_tp_add(&stats->rx_ewma_tp, stats->rx_throughput_raw); stats->tx_throughput = ewma_tp_read(&stats->tx_ewma_tp); stats->rx_throughput = ewma_tp_read(&stats->rx_ewma_tp); stats->tx_tfc_lv = rtw89_get_traffic_level(rtwdev, stats->tx_throughput, stats->tx_cnt); stats->rx_tfc_lv = rtw89_get_traffic_level(rtwdev, stats->rx_throughput, stats->rx_cnt); stats->tx_avg_len = stats->tx_cnt ? DIV_ROUND_DOWN_ULL(stats->tx_unicast, stats->tx_cnt) : 0; stats->rx_avg_len = stats->rx_cnt ? DIV_ROUND_DOWN_ULL(stats->rx_unicast, stats->rx_cnt) : 0; stats->tx_unicast = 0; stats->rx_unicast = 0; stats->tx_cnt = 0; stats->rx_cnt = 0; stats->rx_tf_periodic = stats->rx_tf_acc; stats->rx_tf_acc = 0; if (tx_tfc_lv != stats->tx_tfc_lv || rx_tfc_lv != stats->rx_tfc_lv) return true; return false; } static bool rtw89_traffic_stats_track(struct rtw89_dev *rtwdev) { struct rtw89_vif *rtwvif; bool tfc_changed; tfc_changed = rtw89_traffic_stats_calc(rtwdev, &rtwdev->stats); rtw89_for_each_rtwvif(rtwdev, rtwvif) rtw89_traffic_stats_calc(rtwdev, &rtwvif->stats); return tfc_changed; } static void rtw89_vif_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) { if (rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION) return; if (rtwvif->stats.tx_tfc_lv == RTW89_TFC_IDLE && rtwvif->stats.rx_tfc_lv == RTW89_TFC_IDLE) rtw89_enter_lps(rtwdev, rtwvif->mac_id); } static void rtw89_enter_lps_track(struct rtw89_dev *rtwdev) { struct rtw89_vif *rtwvif; rtw89_for_each_rtwvif(rtwdev, rtwvif) rtw89_vif_enter_lps(rtwdev, rtwvif); } void rtw89_traffic_stats_init(struct rtw89_dev *rtwdev, struct rtw89_traffic_stats *stats) { stats->tx_unicast = 0; stats->rx_unicast = 0; stats->tx_cnt = 0; stats->rx_cnt = 0; ewma_tp_init(&stats->tx_ewma_tp); ewma_tp_init(&stats->rx_ewma_tp); } static void rtw89_track_work(struct work_struct *work) { struct rtw89_dev *rtwdev = container_of(work, struct rtw89_dev, track_work.work); bool tfc_changed; mutex_lock(&rtwdev->mutex); if (!test_bit(RTW89_FLAG_RUNNING, rtwdev->flags)) goto out; ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->track_work, RTW89_TRACK_WORK_PERIOD); tfc_changed = rtw89_traffic_stats_track(rtwdev); if (rtwdev->scanning) goto out; rtw89_leave_lps(rtwdev); if (tfc_changed) { rtw89_hci_recalc_int_mit(rtwdev); rtw89_btc_ntfy_wl_sta(rtwdev); } rtw89_mac_bf_monitor_track(rtwdev); rtw89_phy_stat_track(rtwdev); rtw89_phy_env_monitor_track(rtwdev); rtw89_phy_dig(rtwdev); rtw89_chip_rfk_track(rtwdev); rtw89_phy_ra_update(rtwdev); rtw89_phy_cfo_track(rtwdev); if (rtwdev->lps_enabled && !rtwdev->btc.lps) rtw89_enter_lps_track(rtwdev); out: mutex_unlock(&rtwdev->mutex); } u8 rtw89_core_acquire_bit_map(unsigned long *addr, unsigned long size) { unsigned long bit; bit = find_first_zero_bit(addr, size); if (bit < size) set_bit(bit, addr); return bit; } void rtw89_core_release_bit_map(unsigned long *addr, u8 bit) { clear_bit(bit, addr); } void rtw89_core_release_all_bits_map(unsigned long *addr, unsigned int nbits) { bitmap_zero(addr, nbits); } int rtw89_core_acquire_sta_ba_entry(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx) { const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_cam_info *cam_info = &rtwdev->cam_info; struct rtw89_ba_cam_entry *entry = NULL, *tmp; u8 idx; int i; lockdep_assert_held(&rtwdev->mutex); idx = rtw89_core_acquire_bit_map(cam_info->ba_cam_map, chip->bacam_num); if (idx == chip->bacam_num) { /* allocate a static BA CAM to tid=0/5, so replace the existing * one if BA CAM is full. Hardware will process the original tid * automatically. */ if (tid != 0 && tid != 5) return -ENOSPC; for_each_set_bit(i, cam_info->ba_cam_map, chip->bacam_num) { tmp = &cam_info->ba_cam_entry[i]; if (tmp->tid == 0 || tmp->tid == 5) continue; idx = i; entry = tmp; list_del(&entry->list); break; } if (!entry) return -ENOSPC; } else { entry = &cam_info->ba_cam_entry[idx]; } entry->tid = tid; list_add_tail(&entry->list, &rtwsta->ba_cam_list); *cam_idx = idx; return 0; } int rtw89_core_release_sta_ba_entry(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx) { struct rtw89_cam_info *cam_info = &rtwdev->cam_info; struct rtw89_ba_cam_entry *entry = NULL, *tmp; u8 idx; lockdep_assert_held(&rtwdev->mutex); list_for_each_entry_safe(entry, tmp, &rtwsta->ba_cam_list, list) { if (entry->tid != tid) continue; idx = entry - cam_info->ba_cam_entry; list_del(&entry->list); rtw89_core_release_bit_map(cam_info->ba_cam_map, idx); *cam_idx = idx; return 0; } return -ENOENT; } #define RTW89_TYPE_MAPPING(_type) \ case NL80211_IFTYPE_ ## _type: \ rtwvif->wifi_role = RTW89_WIFI_ROLE_ ## _type; \ break void rtw89_vif_type_mapping(struct ieee80211_vif *vif, bool assoc) { struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; switch (vif->type) { RTW89_TYPE_MAPPING(ADHOC); RTW89_TYPE_MAPPING(STATION); RTW89_TYPE_MAPPING(AP); RTW89_TYPE_MAPPING(MONITOR); RTW89_TYPE_MAPPING(MESH_POINT); default: WARN_ON(1); break; } switch (vif->type) { case NL80211_IFTYPE_AP: case NL80211_IFTYPE_MESH_POINT: rtwvif->net_type = RTW89_NET_TYPE_AP_MODE; rtwvif->self_role = RTW89_SELF_ROLE_AP; break; case NL80211_IFTYPE_ADHOC: rtwvif->net_type = RTW89_NET_TYPE_AD_HOC; rtwvif->self_role = RTW89_SELF_ROLE_CLIENT; break; case NL80211_IFTYPE_STATION: if (assoc) { rtwvif->net_type = RTW89_NET_TYPE_INFRA; rtwvif->trigger = vif->bss_conf.he_support; } else { rtwvif->net_type = RTW89_NET_TYPE_NO_LINK; rtwvif->trigger = false; } rtwvif->self_role = RTW89_SELF_ROLE_CLIENT; rtwvif->addr_cam.sec_ent_mode = RTW89_ADDR_CAM_SEC_NORMAL; break; default: WARN_ON(1); break; } } int rtw89_core_sta_add(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, struct ieee80211_sta *sta) { struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; int i; rtwsta->rtwvif = rtwvif; rtwsta->prev_rssi = 0; INIT_LIST_HEAD(&rtwsta->ba_cam_list); for (i = 0; i < ARRAY_SIZE(sta->txq); i++) rtw89_core_txq_init(rtwdev, sta->txq[i]); ewma_rssi_init(&rtwsta->avg_rssi); if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls) { /* for station mode, assign the mac_id from itself */ rtwsta->mac_id = rtwvif->mac_id; rtw89_btc_ntfy_role_info(rtwdev, rtwvif, rtwsta, BTC_ROLE_MSTS_STA_CONN_START); rtw89_chip_rfk_channel(rtwdev); } else if (vif->type == NL80211_IFTYPE_AP || sta->tdls) { rtwsta->mac_id = rtw89_core_acquire_bit_map(rtwdev->mac_id_map, RTW89_MAX_MAC_ID_NUM); } return 0; } int rtw89_core_sta_disassoc(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, struct ieee80211_sta *sta) { struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; rtwdev->total_sta_assoc--; rtwsta->disassoc = true; return 0; } int rtw89_core_sta_disconnect(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, struct ieee80211_sta *sta) { struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; int ret; rtw89_mac_bf_monitor_calc(rtwdev, sta, true); rtw89_mac_bf_disassoc(rtwdev, vif, sta); rtw89_core_free_sta_pending_ba(rtwdev, sta); rtw89_core_free_sta_pending_forbid_ba(rtwdev, sta); if (vif->type == NL80211_IFTYPE_AP || sta->tdls) rtw89_cam_deinit_addr_cam(rtwdev, &rtwsta->addr_cam); if (sta->tdls) rtw89_cam_deinit_bssid_cam(rtwdev, &rtwsta->bssid_cam); if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls) rtw89_vif_type_mapping(vif, false); ret = rtw89_fw_h2c_assoc_cmac_tbl(rtwdev, vif, sta); if (ret) { rtw89_warn(rtwdev, "failed to send h2c cmac table\n"); return ret; } ret = rtw89_fw_h2c_join_info(rtwdev, rtwvif, rtwsta, true); if (ret) { rtw89_warn(rtwdev, "failed to send h2c join info\n"); return ret; } if (vif->type == NL80211_IFTYPE_AP || sta->tdls) { ret = rtw89_fw_h2c_role_maintain(rtwdev, rtwvif, rtwsta, RTW89_ROLE_REMOVE); if (ret) { rtw89_warn(rtwdev, "failed to send h2c role info\n"); return ret; } } /* update cam aid mac_id net_type */ ret = rtw89_fw_h2c_cam(rtwdev, rtwvif, rtwsta, NULL); if (ret) { rtw89_warn(rtwdev, "failed to send h2c cam\n"); return ret; } return ret; } int rtw89_core_sta_assoc(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, struct ieee80211_sta *sta) { struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; struct rtw89_bssid_cam_entry *bssid_cam = rtw89_get_bssid_cam_of(rtwvif, rtwsta); int ret; if (vif->type == NL80211_IFTYPE_AP || sta->tdls) { ret = rtw89_mac_set_macid_pause(rtwdev, rtwsta->mac_id, false); if (ret) { rtw89_warn(rtwdev, "failed to send h2c macid pause\n"); return ret; } ret = rtw89_fw_h2c_role_maintain(rtwdev, rtwvif, rtwsta, RTW89_ROLE_CREATE); if (ret) { rtw89_warn(rtwdev, "failed to send h2c role info\n"); return ret; } if (sta->tdls) { ret = rtw89_cam_init_bssid_cam(rtwdev, rtwvif, bssid_cam, sta->addr); if (ret) { rtw89_warn(rtwdev, "failed to send h2c init bssid cam for TDLS\n"); return ret; } } ret = rtw89_cam_init_addr_cam(rtwdev, &rtwsta->addr_cam, bssid_cam); if (ret) { rtw89_warn(rtwdev, "failed to send h2c init addr cam\n"); return ret; } } ret = rtw89_fw_h2c_assoc_cmac_tbl(rtwdev, vif, sta); if (ret) { rtw89_warn(rtwdev, "failed to send h2c cmac table\n"); return ret; } ret = rtw89_fw_h2c_join_info(rtwdev, rtwvif, rtwsta, false); if (ret) { rtw89_warn(rtwdev, "failed to send h2c join info\n"); return ret; } /* update cam aid mac_id net_type */ rtw89_fw_h2c_cam(rtwdev, rtwvif, rtwsta, NULL); if (ret) { rtw89_warn(rtwdev, "failed to send h2c cam\n"); return ret; } ret = rtw89_fw_h2c_general_pkt(rtwdev, rtwsta->mac_id); if (ret) { rtw89_warn(rtwdev, "failed to send h2c general packet\n"); return ret; } rtwdev->total_sta_assoc++; rtw89_phy_ra_assoc(rtwdev, sta); rtw89_mac_bf_assoc(rtwdev, vif, sta); rtw89_mac_bf_monitor_calc(rtwdev, sta, false); if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls) { rtw89_btc_ntfy_role_info(rtwdev, rtwvif, rtwsta, BTC_ROLE_MSTS_STA_CONN_END); rtw89_core_get_no_ul_ofdma_htc(rtwdev, &rtwsta->htc_template); } return ret; } int rtw89_core_sta_remove(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, struct ieee80211_sta *sta) { struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls) rtw89_btc_ntfy_role_info(rtwdev, rtwvif, rtwsta, BTC_ROLE_MSTS_STA_DIS_CONN); else if (vif->type == NL80211_IFTYPE_AP || sta->tdls) rtw89_core_release_bit_map(rtwdev->mac_id_map, rtwsta->mac_id); return 0; } static void rtw89_init_ht_cap(struct rtw89_dev *rtwdev, struct ieee80211_sta_ht_cap *ht_cap) { static const __le16 highest[RF_PATH_MAX] = { cpu_to_le16(150), cpu_to_le16(300), cpu_to_le16(450), cpu_to_le16(600), }; struct rtw89_hal *hal = &rtwdev->hal; u8 nss = hal->rx_nss; int i; ht_cap->ht_supported = true; ht_cap->cap = 0; ht_cap->cap |= IEEE80211_HT_CAP_SGI_20 | IEEE80211_HT_CAP_MAX_AMSDU | IEEE80211_HT_CAP_TX_STBC | (1 << IEEE80211_HT_CAP_RX_STBC_SHIFT); ht_cap->cap |= IEEE80211_HT_CAP_LDPC_CODING; ht_cap->cap |= IEEE80211_HT_CAP_SUP_WIDTH_20_40 | IEEE80211_HT_CAP_DSSSCCK40 | IEEE80211_HT_CAP_SGI_40; ht_cap->ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K; ht_cap->ampdu_density = IEEE80211_HT_MPDU_DENSITY_NONE; ht_cap->mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED; for (i = 0; i < nss; i++) ht_cap->mcs.rx_mask[i] = 0xFF; ht_cap->mcs.rx_mask[4] = 0x01; ht_cap->mcs.rx_highest = highest[nss - 1]; } static void rtw89_init_vht_cap(struct rtw89_dev *rtwdev, struct ieee80211_sta_vht_cap *vht_cap) { static const __le16 highest_bw80[RF_PATH_MAX] = { cpu_to_le16(433), cpu_to_le16(867), cpu_to_le16(1300), cpu_to_le16(1733), }; static const __le16 highest_bw160[RF_PATH_MAX] = { cpu_to_le16(867), cpu_to_le16(1733), cpu_to_le16(2600), cpu_to_le16(3467), }; const struct rtw89_chip_info *chip = rtwdev->chip; const __le16 *highest = chip->support_bw160 ? highest_bw160 : highest_bw80; struct rtw89_hal *hal = &rtwdev->hal; u16 tx_mcs_map = 0, rx_mcs_map = 0; u8 sts_cap = 3; int i; for (i = 0; i < 8; i++) { if (i < hal->tx_nss) tx_mcs_map |= IEEE80211_VHT_MCS_SUPPORT_0_9 << (i * 2); else tx_mcs_map |= IEEE80211_VHT_MCS_NOT_SUPPORTED << (i * 2); if (i < hal->rx_nss) rx_mcs_map |= IEEE80211_VHT_MCS_SUPPORT_0_9 << (i * 2); else rx_mcs_map |= IEEE80211_VHT_MCS_NOT_SUPPORTED << (i * 2); } vht_cap->vht_supported = true; vht_cap->cap = IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454 | IEEE80211_VHT_CAP_SHORT_GI_80 | IEEE80211_VHT_CAP_RXSTBC_1 | IEEE80211_VHT_CAP_HTC_VHT | IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK | 0; vht_cap->cap |= IEEE80211_VHT_CAP_TXSTBC; vht_cap->cap |= IEEE80211_VHT_CAP_RXLDPC; vht_cap->cap |= IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE | IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE; vht_cap->cap |= sts_cap << IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT; if (chip->support_bw160) vht_cap->cap |= IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ | IEEE80211_VHT_CAP_SHORT_GI_160; vht_cap->vht_mcs.rx_mcs_map = cpu_to_le16(rx_mcs_map); vht_cap->vht_mcs.tx_mcs_map = cpu_to_le16(tx_mcs_map); vht_cap->vht_mcs.rx_highest = highest[hal->rx_nss - 1]; vht_cap->vht_mcs.tx_highest = highest[hal->tx_nss - 1]; } #define RTW89_SBAND_IFTYPES_NR 2 static void rtw89_init_he_cap(struct rtw89_dev *rtwdev, enum nl80211_band band, struct ieee80211_supported_band *sband) { const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_hal *hal = &rtwdev->hal; struct ieee80211_sband_iftype_data *iftype_data; bool no_ng16 = (chip->chip_id == RTL8852A && hal->cv == CHIP_CBV) || (chip->chip_id == RTL8852B && hal->cv == CHIP_CAV); u16 mcs_map = 0; int i; int nss = hal->rx_nss; int idx = 0; iftype_data = kcalloc(RTW89_SBAND_IFTYPES_NR, sizeof(*iftype_data), GFP_KERNEL); if (!iftype_data) return; for (i = 0; i < 8; i++) { if (i < nss) mcs_map |= IEEE80211_HE_MCS_SUPPORT_0_11 << (i * 2); else mcs_map |= IEEE80211_HE_MCS_NOT_SUPPORTED << (i * 2); } for (i = 0; i < NUM_NL80211_IFTYPES; i++) { struct ieee80211_sta_he_cap *he_cap; u8 *mac_cap_info; u8 *phy_cap_info; switch (i) { case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_AP: break; default: continue; } if (idx >= RTW89_SBAND_IFTYPES_NR) { rtw89_warn(rtwdev, "run out of iftype_data\n"); break; } iftype_data[idx].types_mask = BIT(i); he_cap = &iftype_data[idx].he_cap; mac_cap_info = he_cap->he_cap_elem.mac_cap_info; phy_cap_info = he_cap->he_cap_elem.phy_cap_info; he_cap->has_he = true; mac_cap_info[0] = IEEE80211_HE_MAC_CAP0_HTC_HE; if (i == NL80211_IFTYPE_STATION) mac_cap_info[1] = IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US; mac_cap_info[2] = IEEE80211_HE_MAC_CAP2_ALL_ACK | IEEE80211_HE_MAC_CAP2_BSR; mac_cap_info[3] = IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_2; if (i == NL80211_IFTYPE_AP) mac_cap_info[3] |= IEEE80211_HE_MAC_CAP3_OMI_CONTROL; mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_OPS | IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU; if (i == NL80211_IFTYPE_STATION) mac_cap_info[5] = IEEE80211_HE_MAC_CAP5_HT_VHT_TRIG_FRAME_RX; if (band == NL80211_BAND_2GHZ) { phy_cap_info[0] = IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G; } else { phy_cap_info[0] = IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G; if (chip->support_bw160) phy_cap_info[0] |= IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G; } phy_cap_info[1] = IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A | IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD | IEEE80211_HE_PHY_CAP1_HE_LTF_AND_GI_FOR_HE_PPDUS_0_8US; phy_cap_info[2] = IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US | IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ | IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ | IEEE80211_HE_PHY_CAP2_DOPPLER_TX; phy_cap_info[3] = IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_16_QAM; if (i == NL80211_IFTYPE_STATION) phy_cap_info[3] |= IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_16_QAM | IEEE80211_HE_PHY_CAP3_DCM_MAX_TX_NSS_2; if (i == NL80211_IFTYPE_AP) phy_cap_info[3] |= IEEE80211_HE_PHY_CAP3_RX_PARTIAL_BW_SU_IN_20MHZ_MU; phy_cap_info[4] = IEEE80211_HE_PHY_CAP4_SU_BEAMFORMEE | IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_4; if (chip->support_bw160) phy_cap_info[4] |= IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4; phy_cap_info[5] = no_ng16 ? 0 : IEEE80211_HE_PHY_CAP5_NG16_SU_FEEDBACK | IEEE80211_HE_PHY_CAP5_NG16_MU_FEEDBACK; phy_cap_info[6] = IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_42_SU | IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_75_MU | IEEE80211_HE_PHY_CAP6_TRIG_SU_BEAMFORMING_FB | IEEE80211_HE_PHY_CAP6_PARTIAL_BW_EXT_RANGE; phy_cap_info[7] = IEEE80211_HE_PHY_CAP7_POWER_BOOST_FACTOR_SUPP | IEEE80211_HE_PHY_CAP7_HE_SU_MU_PPDU_4XLTF_AND_08_US_GI | IEEE80211_HE_PHY_CAP7_MAX_NC_1; phy_cap_info[8] = IEEE80211_HE_PHY_CAP8_HE_ER_SU_PPDU_4XLTF_AND_08_US_GI | IEEE80211_HE_PHY_CAP8_HE_ER_SU_1XLTF_AND_08_US_GI | IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_996; if (chip->support_bw160) phy_cap_info[8] |= IEEE80211_HE_PHY_CAP8_20MHZ_IN_160MHZ_HE_PPDU | IEEE80211_HE_PHY_CAP8_80MHZ_IN_160MHZ_HE_PPDU; phy_cap_info[9] = IEEE80211_HE_PHY_CAP9_LONGER_THAN_16_SIGB_OFDM_SYM | IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU | IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_COMP_SIGB | IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_NON_COMP_SIGB | u8_encode_bits(IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US, IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK); if (i == NL80211_IFTYPE_STATION) phy_cap_info[9] |= IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU; he_cap->he_mcs_nss_supp.rx_mcs_80 = cpu_to_le16(mcs_map); he_cap->he_mcs_nss_supp.tx_mcs_80 = cpu_to_le16(mcs_map); if (chip->support_bw160) { he_cap->he_mcs_nss_supp.rx_mcs_160 = cpu_to_le16(mcs_map); he_cap->he_mcs_nss_supp.tx_mcs_160 = cpu_to_le16(mcs_map); } if (band == NL80211_BAND_6GHZ) { __le16 capa; capa = le16_encode_bits(IEEE80211_HT_MPDU_DENSITY_NONE, IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START) | le16_encode_bits(IEEE80211_VHT_MAX_AMPDU_1024K, IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP) | le16_encode_bits(IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454, IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN); iftype_data[idx].he_6ghz_capa.capa = capa; } idx++; } sband->iftype_data = iftype_data; sband->n_iftype_data = idx; } static int rtw89_core_set_supported_band(struct rtw89_dev *rtwdev) { struct ieee80211_hw *hw = rtwdev->hw; struct ieee80211_supported_band *sband_2ghz = NULL, *sband_5ghz = NULL; struct ieee80211_supported_band *sband_6ghz = NULL; u32 size = sizeof(struct ieee80211_supported_band); u8 support_bands = rtwdev->chip->support_bands; if (support_bands & BIT(NL80211_BAND_2GHZ)) { sband_2ghz = kmemdup(&rtw89_sband_2ghz, size, GFP_KERNEL); if (!sband_2ghz) goto err; rtw89_init_ht_cap(rtwdev, &sband_2ghz->ht_cap); rtw89_init_he_cap(rtwdev, NL80211_BAND_2GHZ, sband_2ghz); hw->wiphy->bands[NL80211_BAND_2GHZ] = sband_2ghz; } if (support_bands & BIT(NL80211_BAND_5GHZ)) { sband_5ghz = kmemdup(&rtw89_sband_5ghz, size, GFP_KERNEL); if (!sband_5ghz) goto err; rtw89_init_ht_cap(rtwdev, &sband_5ghz->ht_cap); rtw89_init_vht_cap(rtwdev, &sband_5ghz->vht_cap); rtw89_init_he_cap(rtwdev, NL80211_BAND_5GHZ, sband_5ghz); hw->wiphy->bands[NL80211_BAND_5GHZ] = sband_5ghz; } if (support_bands & BIT(NL80211_BAND_6GHZ)) { sband_6ghz = kmemdup(&rtw89_sband_6ghz, size, GFP_KERNEL); if (!sband_6ghz) goto err; rtw89_init_he_cap(rtwdev, NL80211_BAND_6GHZ, sband_6ghz); hw->wiphy->bands[NL80211_BAND_6GHZ] = sband_6ghz; } return 0; err: hw->wiphy->bands[NL80211_BAND_2GHZ] = NULL; hw->wiphy->bands[NL80211_BAND_5GHZ] = NULL; hw->wiphy->bands[NL80211_BAND_6GHZ] = NULL; if (sband_2ghz) kfree(sband_2ghz->iftype_data); if (sband_5ghz) kfree(sband_5ghz->iftype_data); if (sband_6ghz) kfree(sband_6ghz->iftype_data); kfree(sband_2ghz); kfree(sband_5ghz); kfree(sband_6ghz); return -ENOMEM; } static void rtw89_core_clr_supported_band(struct rtw89_dev *rtwdev) { struct ieee80211_hw *hw = rtwdev->hw; kfree(hw->wiphy->bands[NL80211_BAND_2GHZ]->iftype_data); kfree(hw->wiphy->bands[NL80211_BAND_5GHZ]->iftype_data); if (hw->wiphy->bands[NL80211_BAND_6GHZ]) kfree(hw->wiphy->bands[NL80211_BAND_6GHZ]->iftype_data); kfree(hw->wiphy->bands[NL80211_BAND_2GHZ]); kfree(hw->wiphy->bands[NL80211_BAND_5GHZ]); kfree(hw->wiphy->bands[NL80211_BAND_6GHZ]); hw->wiphy->bands[NL80211_BAND_2GHZ] = NULL; hw->wiphy->bands[NL80211_BAND_5GHZ] = NULL; hw->wiphy->bands[NL80211_BAND_6GHZ] = NULL; } static void rtw89_core_ppdu_sts_init(struct rtw89_dev *rtwdev) { int i; for (i = 0; i < RTW89_PHY_MAX; i++) skb_queue_head_init(&rtwdev->ppdu_sts.rx_queue[i]); for (i = 0; i < RTW89_PHY_MAX; i++) rtwdev->ppdu_sts.curr_rx_ppdu_cnt[i] = U8_MAX; } void rtw89_core_update_beacon_work(struct work_struct *work) { struct rtw89_dev *rtwdev; struct rtw89_vif *rtwvif = container_of(work, struct rtw89_vif, update_beacon_work); if (rtwvif->net_type != RTW89_NET_TYPE_AP_MODE) return; rtwdev = rtwvif->rtwdev; mutex_lock(&rtwdev->mutex); rtw89_fw_h2c_update_beacon(rtwdev, rtwvif); mutex_unlock(&rtwdev->mutex); } int rtw89_core_start(struct rtw89_dev *rtwdev) { int ret; rtwdev->mac.qta_mode = RTW89_QTA_SCC; ret = rtw89_mac_init(rtwdev); if (ret) { rtw89_err(rtwdev, "mac init fail, ret:%d\n", ret); return ret; } rtw89_btc_ntfy_poweron(rtwdev); /* efuse process */ /* pre-config BB/RF, BB reset/RFC reset */ rtw89_chip_disable_bb_rf(rtwdev); ret = rtw89_chip_enable_bb_rf(rtwdev); if (ret) return ret; rtw89_phy_init_bb_reg(rtwdev); rtw89_phy_init_rf_reg(rtwdev); rtw89_btc_ntfy_init(rtwdev, BTC_MODE_NORMAL); rtw89_phy_dm_init(rtwdev); rtw89_mac_cfg_ppdu_status(rtwdev, RTW89_MAC_0, true); rtw89_mac_update_rts_threshold(rtwdev, RTW89_MAC_0); ret = rtw89_hci_start(rtwdev); if (ret) { rtw89_err(rtwdev, "failed to start hci\n"); return ret; } ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->track_work, RTW89_TRACK_WORK_PERIOD); set_bit(RTW89_FLAG_RUNNING, rtwdev->flags); rtw89_btc_ntfy_radio_state(rtwdev, BTC_RFCTRL_WL_ON); rtw89_fw_h2c_fw_log(rtwdev, rtwdev->fw.fw_log_enable); rtw89_fw_h2c_init_ba_cam(rtwdev); return 0; } void rtw89_core_stop(struct rtw89_dev *rtwdev) { struct rtw89_btc *btc = &rtwdev->btc; /* Prvent to stop twice; enter_ips and ops_stop */ if (!test_bit(RTW89_FLAG_RUNNING, rtwdev->flags)) return; rtw89_btc_ntfy_radio_state(rtwdev, BTC_RFCTRL_WL_OFF); clear_bit(RTW89_FLAG_RUNNING, rtwdev->flags); mutex_unlock(&rtwdev->mutex); cancel_work_sync(&rtwdev->c2h_work); cancel_work_sync(&btc->eapol_notify_work); cancel_work_sync(&btc->arp_notify_work); cancel_work_sync(&btc->dhcp_notify_work); cancel_work_sync(&btc->icmp_notify_work); cancel_delayed_work_sync(&rtwdev->txq_reinvoke_work); cancel_delayed_work_sync(&rtwdev->track_work); cancel_delayed_work_sync(&rtwdev->coex_act1_work); cancel_delayed_work_sync(&rtwdev->coex_bt_devinfo_work); cancel_delayed_work_sync(&rtwdev->coex_rfk_chk_work); cancel_delayed_work_sync(&rtwdev->cfo_track_work); cancel_delayed_work_sync(&rtwdev->forbid_ba_work); mutex_lock(&rtwdev->mutex); rtw89_btc_ntfy_poweroff(rtwdev); rtw89_hci_flush_queues(rtwdev, BIT(rtwdev->hw->queues) - 1, true); rtw89_mac_flush_txq(rtwdev, BIT(rtwdev->hw->queues) - 1, true); rtw89_hci_stop(rtwdev); rtw89_hci_deinit(rtwdev); rtw89_mac_pwr_off(rtwdev); rtw89_hci_reset(rtwdev); } int rtw89_core_init(struct rtw89_dev *rtwdev) { struct rtw89_btc *btc = &rtwdev->btc; int ret; u8 band; INIT_LIST_HEAD(&rtwdev->ba_list); INIT_LIST_HEAD(&rtwdev->forbid_ba_list); INIT_LIST_HEAD(&rtwdev->rtwvifs_list); INIT_LIST_HEAD(&rtwdev->early_h2c_list); for (band = NL80211_BAND_2GHZ; band < NUM_NL80211_BANDS; band++) { if (!(rtwdev->chip->support_bands & BIT(band))) continue; INIT_LIST_HEAD(&rtwdev->scan_info.pkt_list[band]); } INIT_WORK(&rtwdev->ba_work, rtw89_core_ba_work); INIT_WORK(&rtwdev->txq_work, rtw89_core_txq_work); INIT_DELAYED_WORK(&rtwdev->txq_reinvoke_work, rtw89_core_txq_reinvoke_work); INIT_DELAYED_WORK(&rtwdev->track_work, rtw89_track_work); INIT_DELAYED_WORK(&rtwdev->coex_act1_work, rtw89_coex_act1_work); INIT_DELAYED_WORK(&rtwdev->coex_bt_devinfo_work, rtw89_coex_bt_devinfo_work); INIT_DELAYED_WORK(&rtwdev->coex_rfk_chk_work, rtw89_coex_rfk_chk_work); INIT_DELAYED_WORK(&rtwdev->cfo_track_work, rtw89_phy_cfo_track_work); INIT_DELAYED_WORK(&rtwdev->forbid_ba_work, rtw89_forbid_ba_work); rtwdev->txq_wq = alloc_workqueue("rtw89_tx_wq", WQ_UNBOUND | WQ_HIGHPRI, 0); spin_lock_init(&rtwdev->ba_lock); spin_lock_init(&rtwdev->rpwm_lock); mutex_init(&rtwdev->mutex); mutex_init(&rtwdev->rf_mutex); rtwdev->total_sta_assoc = 0; INIT_WORK(&rtwdev->c2h_work, rtw89_fw_c2h_work); INIT_WORK(&rtwdev->ips_work, rtw89_ips_work); skb_queue_head_init(&rtwdev->c2h_queue); rtw89_core_ppdu_sts_init(rtwdev); rtw89_traffic_stats_init(rtwdev, &rtwdev->stats); rtwdev->ps_mode = rtw89_update_ps_mode(rtwdev); rtwdev->hal.rx_fltr = DEFAULT_AX_RX_FLTR; INIT_WORK(&btc->eapol_notify_work, rtw89_btc_ntfy_eapol_packet_work); INIT_WORK(&btc->arp_notify_work, rtw89_btc_ntfy_arp_packet_work); INIT_WORK(&btc->dhcp_notify_work, rtw89_btc_ntfy_dhcp_packet_work); INIT_WORK(&btc->icmp_notify_work, rtw89_btc_ntfy_icmp_packet_work); ret = rtw89_load_firmware(rtwdev); if (ret) { rtw89_warn(rtwdev, "no firmware loaded\n"); return ret; } rtw89_ser_init(rtwdev); rtw89_entity_init(rtwdev); return 0; } EXPORT_SYMBOL(rtw89_core_init); void rtw89_core_deinit(struct rtw89_dev *rtwdev) { rtw89_ser_deinit(rtwdev); rtw89_unload_firmware(rtwdev); rtw89_fw_free_all_early_h2c(rtwdev); destroy_workqueue(rtwdev->txq_wq); mutex_destroy(&rtwdev->rf_mutex); mutex_destroy(&rtwdev->mutex); } EXPORT_SYMBOL(rtw89_core_deinit); void rtw89_core_scan_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, const u8 *mac_addr, bool hw_scan) { const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); rtwdev->scanning = true; rtw89_leave_lps(rtwdev); if (hw_scan && (rtwdev->hw->conf.flags & IEEE80211_CONF_IDLE)) rtw89_leave_ips(rtwdev); ether_addr_copy(rtwvif->mac_addr, mac_addr); rtw89_btc_ntfy_scan_start(rtwdev, RTW89_PHY_0, chan->band_type); rtw89_chip_rfk_scan(rtwdev, true); rtw89_hci_recalc_int_mit(rtwdev); rtw89_fw_h2c_cam(rtwdev, rtwvif, NULL, mac_addr); } void rtw89_core_scan_complete(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, bool hw_scan) { struct rtw89_vif *rtwvif = vif ? (struct rtw89_vif *)vif->drv_priv : NULL; if (!rtwvif) return; ether_addr_copy(rtwvif->mac_addr, vif->addr); rtw89_fw_h2c_cam(rtwdev, rtwvif, NULL, NULL); rtw89_chip_rfk_scan(rtwdev, false); rtw89_btc_ntfy_scan_finish(rtwdev, RTW89_PHY_0); rtwdev->scanning = false; rtwdev->dig.bypass_dig = true; if (hw_scan && (rtwdev->hw->conf.flags & IEEE80211_CONF_IDLE)) ieee80211_queue_work(rtwdev->hw, &rtwdev->ips_work); } static void rtw89_read_chip_ver(struct rtw89_dev *rtwdev) { const struct rtw89_chip_info *chip = rtwdev->chip; u8 cv; cv = rtw89_read32_mask(rtwdev, R_AX_SYS_CFG1, B_AX_CHIP_VER_MASK); if (chip->chip_id == RTL8852A && cv <= CHIP_CBV) { if (rtw89_read32(rtwdev, R_AX_GPIO0_7_FUNC_SEL) == RTW89_R32_DEAD) cv = CHIP_CAV; else cv = CHIP_CBV; } rtwdev->hal.cv = cv; } static void rtw89_core_setup_phycap(struct rtw89_dev *rtwdev) { rtwdev->hal.support_cckpd = !(rtwdev->chip->chip_id == RTL8852A && rtwdev->hal.cv <= CHIP_CBV) && !(rtwdev->chip->chip_id == RTL8852B && rtwdev->hal.cv <= CHIP_CAV); rtwdev->hal.support_igi = rtwdev->chip->chip_id == RTL8852A && rtwdev->hal.cv <= CHIP_CBV; } static int rtw89_chip_efuse_info_setup(struct rtw89_dev *rtwdev) { int ret; ret = rtw89_mac_partial_init(rtwdev); if (ret) return ret; ret = rtw89_parse_efuse_map(rtwdev); if (ret) return ret; ret = rtw89_parse_phycap_map(rtwdev); if (ret) return ret; ret = rtw89_mac_setup_phycap(rtwdev); if (ret) return ret; rtw89_core_setup_phycap(rtwdev); rtw89_mac_pwr_off(rtwdev); return 0; } static int rtw89_chip_board_info_setup(struct rtw89_dev *rtwdev) { rtw89_chip_fem_setup(rtwdev); return 0; } int rtw89_chip_info_setup(struct rtw89_dev *rtwdev) { int ret; rtw89_read_chip_ver(rtwdev); ret = rtw89_wait_firmware_completion(rtwdev); if (ret) { rtw89_err(rtwdev, "failed to wait firmware completion\n"); return ret; } ret = rtw89_fw_recognize(rtwdev); if (ret) { rtw89_err(rtwdev, "failed to recognize firmware\n"); return ret; } ret = rtw89_chip_efuse_info_setup(rtwdev); if (ret) return ret; ret = rtw89_chip_board_info_setup(rtwdev); if (ret) return ret; return 0; } EXPORT_SYMBOL(rtw89_chip_info_setup); static int rtw89_core_register_hw(struct rtw89_dev *rtwdev) { struct ieee80211_hw *hw = rtwdev->hw; struct rtw89_efuse *efuse = &rtwdev->efuse; int ret; int tx_headroom = IEEE80211_HT_CTL_LEN; hw->vif_data_size = sizeof(struct rtw89_vif); hw->sta_data_size = sizeof(struct rtw89_sta); hw->txq_data_size = sizeof(struct rtw89_txq); hw->chanctx_data_size = sizeof(struct rtw89_chanctx_cfg); SET_IEEE80211_PERM_ADDR(hw, efuse->addr); hw->extra_tx_headroom = tx_headroom; hw->queues = IEEE80211_NUM_ACS; hw->max_rx_aggregation_subframes = RTW89_MAX_RX_AGG_NUM; hw->max_tx_aggregation_subframes = RTW89_MAX_TX_AGG_NUM; ieee80211_hw_set(hw, SIGNAL_DBM); ieee80211_hw_set(hw, HAS_RATE_CONTROL); ieee80211_hw_set(hw, MFP_CAPABLE); ieee80211_hw_set(hw, REPORTS_TX_ACK_STATUS); ieee80211_hw_set(hw, AMPDU_AGGREGATION); ieee80211_hw_set(hw, RX_INCLUDES_FCS); ieee80211_hw_set(hw, TX_AMSDU); ieee80211_hw_set(hw, SUPPORT_FAST_XMIT); ieee80211_hw_set(hw, SUPPORTS_AMSDU_IN_AMPDU); ieee80211_hw_set(hw, SUPPORTS_PS); ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS); ieee80211_hw_set(hw, SINGLE_SCAN_ON_ALL_BANDS); ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID); hw->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION) | BIT(NL80211_IFTYPE_AP); hw->wiphy->available_antennas_tx = BIT(rtwdev->chip->rf_path_num) - 1; hw->wiphy->available_antennas_rx = BIT(rtwdev->chip->rf_path_num) - 1; hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS | WIPHY_FLAG_TDLS_EXTERNAL_SETUP; hw->wiphy->features |= NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR; hw->wiphy->max_scan_ssids = RTW89_SCANOFLD_MAX_SSID; hw->wiphy->max_scan_ie_len = RTW89_SCANOFLD_MAX_IE_LEN; wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_CAN_REPLACE_PTK0); ret = rtw89_core_set_supported_band(rtwdev); if (ret) { rtw89_err(rtwdev, "failed to set supported band\n"); return ret; } hw->wiphy->reg_notifier = rtw89_regd_notifier; hw->wiphy->sar_capa = &rtw89_sar_capa; ret = ieee80211_register_hw(hw); if (ret) { rtw89_err(rtwdev, "failed to register hw\n"); goto err; } ret = rtw89_regd_init(rtwdev, rtw89_regd_notifier); if (ret) { rtw89_err(rtwdev, "failed to init regd\n"); goto err; } return 0; err: return ret; } static void rtw89_core_unregister_hw(struct rtw89_dev *rtwdev) { struct ieee80211_hw *hw = rtwdev->hw; ieee80211_unregister_hw(hw); rtw89_core_clr_supported_band(rtwdev); } int rtw89_core_register(struct rtw89_dev *rtwdev) { int ret; ret = rtw89_core_register_hw(rtwdev); if (ret) { rtw89_err(rtwdev, "failed to register core hw\n"); return ret; } rtw89_debugfs_init(rtwdev); return 0; } EXPORT_SYMBOL(rtw89_core_register); void rtw89_core_unregister(struct rtw89_dev *rtwdev) { rtw89_core_unregister_hw(rtwdev); } EXPORT_SYMBOL(rtw89_core_unregister); struct rtw89_dev *rtw89_alloc_ieee80211_hw(struct device *device, u32 bus_data_size, const struct rtw89_chip_info *chip) { struct ieee80211_hw *hw; struct rtw89_dev *rtwdev; struct ieee80211_ops *ops; u32 driver_data_size; u32 early_feat_map = 0; bool no_chanctx; rtw89_early_fw_feature_recognize(device, chip, &early_feat_map); ops = kmemdup(&rtw89_ops, sizeof(rtw89_ops), GFP_KERNEL); if (!ops) goto err; no_chanctx = chip->support_chanctx_num == 0 || !(early_feat_map & BIT(RTW89_FW_FEATURE_SCAN_OFFLOAD)); if (no_chanctx) { ops->add_chanctx = NULL; ops->remove_chanctx = NULL; ops->change_chanctx = NULL; ops->assign_vif_chanctx = NULL; ops->unassign_vif_chanctx = NULL; } driver_data_size = sizeof(struct rtw89_dev) + bus_data_size; hw = ieee80211_alloc_hw(driver_data_size, ops); if (!hw) goto err; rtwdev = hw->priv; rtwdev->hw = hw; rtwdev->dev = device; rtwdev->ops = ops; rtwdev->chip = chip; rtw89_debug(rtwdev, RTW89_DBG_FW, "probe driver %s chanctx\n", no_chanctx ? "without" : "with"); return rtwdev; err: kfree(ops); return NULL; } EXPORT_SYMBOL(rtw89_alloc_ieee80211_hw); void rtw89_free_ieee80211_hw(struct rtw89_dev *rtwdev) { kfree(rtwdev->ops); ieee80211_free_hw(rtwdev->hw); } EXPORT_SYMBOL(rtw89_free_ieee80211_hw); MODULE_AUTHOR("Realtek Corporation"); MODULE_DESCRIPTION("Realtek 802.11ax wireless core module"); MODULE_LICENSE("Dual BSD/GPL");