diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c index 39bbd6a37cf3..356962cfad99 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c @@ -1,1018 +1,1018 @@ /*- * Copyright (c) 2021-2022 NVIDIA corporation & affiliates. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "opt_kern_tls.h" #include "opt_rss.h" #include "opt_ratelimit.h" #include #include #include #include #include #include #ifdef KERN_TLS static if_snd_tag_free_t mlx5e_tls_rx_snd_tag_free; static if_snd_tag_modify_t mlx5e_tls_rx_snd_tag_modify; static const struct if_snd_tag_sw mlx5e_tls_rx_snd_tag_sw = { .snd_tag_modify = mlx5e_tls_rx_snd_tag_modify, .snd_tag_free = mlx5e_tls_rx_snd_tag_free, .type = IF_SND_TAG_TYPE_TLS_RX }; MALLOC_DEFINE(M_MLX5E_TLS_RX, "MLX5E_TLS_RX", "MLX5 ethernet HW TLS RX"); /* software TLS RX context */ struct mlx5_ifc_sw_tls_rx_cntx_bits { struct mlx5_ifc_tls_static_params_bits param; struct mlx5_ifc_tls_progress_params_bits progress; struct { uint8_t key_data[8][0x20]; uint8_t key_len[0x20]; } key; }; CTASSERT(MLX5_ST_SZ_BYTES(sw_tls_rx_cntx) <= sizeof(((struct mlx5e_tls_rx_tag *)NULL)->crypto_params)); CTASSERT(MLX5_ST_SZ_BYTES(mkc) == sizeof(((struct mlx5e_tx_umr_wqe *)NULL)->mkc)); static const char *mlx5e_tls_rx_stats_desc[] = { MLX5E_TLS_RX_STATS(MLX5E_STATS_DESC) }; static void mlx5e_tls_rx_work(struct work_struct *); static bool mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(struct mlx5e_tls_rx_tag *, uint32_t, uint32_t *, uint64_t *); CTASSERT((MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param) % 16) == 0); static uint32_t mlx5e_tls_rx_get_ch(struct mlx5e_priv *priv, uint32_t flowid, uint32_t flowtype) { u32 ch; #ifdef RSS u32 temp; #endif /* keep this code synced with mlx5e_select_queue() */ ch = priv->params.num_channels; #ifdef RSS if (rss_hash2bucket(flowid, flowtype, &temp) == 0) ch = temp % ch; else #endif ch = (flowid % 128) % ch; return (ch); } /* * This function gets a pointer to an internal queue, IQ, based on the * provided "flowid" and "flowtype". The IQ returned may in some rare * cases not be activated or running, but this is all handled by the * "mlx5e_iq_get_producer_index()" function. * * The idea behind this function is to spread the IQ traffic as much * as possible and to avoid congestion on the same IQ when processing * RX traffic. */ static struct mlx5e_iq * mlx5e_tls_rx_get_iq(struct mlx5e_priv *priv, uint32_t flowid, uint32_t flowtype) { /* * NOTE: The channels array is only freed at detach * and it safe to return a pointer to the send tag * inside the channels structure as long as we * reference the priv. */ return (&priv->channel[mlx5e_tls_rx_get_ch(priv, flowid, flowtype)].iq); } static void mlx5e_tls_rx_send_static_parameters_cb(void *arg) { struct mlx5e_tls_rx_tag *ptag; ptag = (struct mlx5e_tls_rx_tag *)arg; m_snd_tag_rele(&ptag->tag); } /* * This function sends the so-called TLS RX static parameters to the * hardware. These parameters are temporarily stored in the * "crypto_params" field of the TLS RX tag. Most importantly this * function sets the TCP sequence number (32-bit) and TLS record * number (64-bit) where the decryption can resume. * * Zero is returned upon success. Else some error happend. */ static int mlx5e_tls_rx_send_static_parameters(struct mlx5e_iq *iq, struct mlx5e_tls_rx_tag *ptag) { const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_umr_wqe) + MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param), MLX5_SEND_WQE_DS); struct mlx5e_tx_umr_wqe *wqe; int pi; mtx_lock(&iq->lock); pi = mlx5e_iq_get_producer_index(iq); if (pi < 0) { mtx_unlock(&iq->lock); return (-ENOMEM); } wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi); memset(wqe, 0, sizeof(*wqe)); wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) | MLX5_OPCODE_UMR | (MLX5_OPCODE_MOD_UMR_TLS_TIR_STATIC_PARAMS << 24)); wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt); wqe->ctrl.imm = cpu_to_be32(ptag->tirn << 8); wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL; /* fill out UMR control segment */ wqe->umr.flags = 0x80; /* inline data */ wqe->umr.bsf_octowords = cpu_to_be16(MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param) / 16); /* copy in the static crypto parameters */ memcpy(wqe + 1, MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, param), MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param)); /* copy data for doorbell */ memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32)); iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); iq->data[pi].callback = &mlx5e_tls_rx_send_static_parameters_cb; iq->data[pi].arg = ptag; m_snd_tag_ref(&ptag->tag); iq->pc += iq->data[pi].num_wqebbs; mlx5e_iq_notify_hw(iq); mtx_unlock(&iq->lock); return (0); /* success */ } static void mlx5e_tls_rx_send_progress_parameters_cb(void *arg) { struct mlx5e_tls_rx_tag *ptag; ptag = (struct mlx5e_tls_rx_tag *)arg; complete(&ptag->progress_complete); } CTASSERT(MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, progress) == sizeof(((struct mlx5e_tx_psv_wqe *)NULL)->psv)); /* * This function resets the state of the TIR context to start * searching for a valid TLS header and is used only when allocating * the TLS RX tag. * * Zero is returned upon success, else some error happened. */ static int mlx5e_tls_rx_send_progress_parameters_sync(struct mlx5e_iq *iq, struct mlx5e_tls_rx_tag *ptag) { const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_psv_wqe), MLX5_SEND_WQE_DS); struct mlx5e_priv *priv; struct mlx5e_tx_psv_wqe *wqe; int pi; mtx_lock(&iq->lock); pi = mlx5e_iq_get_producer_index(iq); if (pi < 0) { mtx_unlock(&iq->lock); return (-ENOMEM); } wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi); memset(wqe, 0, sizeof(*wqe)); wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) | MLX5_OPCODE_SET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIR_PROGRESS_PARAMS << 24)); wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt); wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; /* copy in the PSV control segment */ memcpy(&wqe->psv, MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, progress), sizeof(wqe->psv)); /* copy data for doorbell */ memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32)); iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); iq->data[pi].callback = &mlx5e_tls_rx_send_progress_parameters_cb; iq->data[pi].arg = ptag; iq->pc += iq->data[pi].num_wqebbs; init_completion(&ptag->progress_complete); mlx5e_iq_notify_hw(iq); mtx_unlock(&iq->lock); while (1) { if (wait_for_completion_timeout(&ptag->progress_complete, hz) != 0) break; priv = container_of(iq, struct mlx5e_channel, iq)->priv; if (priv->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR || pci_channel_offline(priv->mdev->pdev) != 0) return (-EWOULDBLOCK); } return (0); /* success */ } CTASSERT(MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE >= MLX5_ST_SZ_BYTES(tls_progress_params)); CTASSERT(MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE <= PAGE_SIZE); struct mlx5e_get_tls_progress_params_wqe { struct mlx5_wqe_ctrl_seg ctrl; struct mlx5_seg_get_psv psv; }; static void mlx5e_tls_rx_receive_progress_parameters_cb(void *arg) { struct mlx5e_tls_rx_tag *ptag; struct mlx5e_iq *iq; uint32_t tcp_curr_sn_he; uint32_t tcp_next_sn_he; uint64_t tls_rcd_num; void *buffer; ptag = (struct mlx5e_tls_rx_tag *)arg; buffer = mlx5e_tls_rx_get_progress_buffer(ptag); MLX5E_TLS_RX_TAG_LOCK(ptag); ptag->tcp_resync_pending = 0; switch (MLX5_GET(tls_progress_params, buffer, record_tracker_state)) { case MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING: break; default: goto done; } switch (MLX5_GET(tls_progress_params, buffer, auth_state)) { case MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD: break; default: goto done; } tcp_curr_sn_he = MLX5_GET(tls_progress_params, buffer, hw_resync_tcp_sn); if (mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(ptag, tcp_curr_sn_he, &tcp_next_sn_he, &tls_rcd_num)) { MLX5_SET64(sw_tls_rx_cntx, ptag->crypto_params, param.initial_record_number, tls_rcd_num); MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, param.resync_tcp_sn, tcp_curr_sn_he); iq = mlx5e_tls_rx_get_iq( container_of(ptag->tls_rx, struct mlx5e_priv, tls_rx), ptag->flowid, ptag->flowtype); if (mlx5e_tls_rx_send_static_parameters(iq, ptag) != 0) MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1); } done: MLX5E_TLS_RX_TAG_UNLOCK(ptag); m_snd_tag_rele(&ptag->tag); } /* * This function queries the hardware for the current state of the TIR * in question. It is typically called when encrypted data is received * to re-establish hardware decryption of received TLS data. * * Zero is returned upon success, else some error happened. */ static int mlx5e_tls_rx_receive_progress_parameters(struct mlx5e_iq *iq, struct mlx5e_tls_rx_tag *ptag) { struct mlx5e_get_tls_progress_params_wqe *wqe; const u32 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS); u64 dma_address; int pi; mtx_lock(&iq->lock); pi = mlx5e_iq_get_producer_index(iq); if (pi < 0) { mtx_unlock(&iq->lock); return (-ENOMEM); } mlx5e_iq_load_memory_single(iq, pi, mlx5e_tls_rx_get_progress_buffer(ptag), MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE, &dma_address, BUS_DMASYNC_PREREAD); wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi); memset(wqe, 0, sizeof(*wqe)); wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) | MLX5_OPCODE_GET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIR_PROGRESS_PARAMS << 24)); wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt); wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; wqe->psv.num_psv = 1 << 4; wqe->psv.l_key = iq->mkey_be; wqe->psv.psv_index[0] = cpu_to_be32(ptag->tirn); wqe->psv.va = cpu_to_be64(dma_address); /* copy data for doorbell */ memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32)); iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); iq->data[pi].callback = &mlx5e_tls_rx_receive_progress_parameters_cb; iq->data[pi].arg = ptag; m_snd_tag_ref(&ptag->tag); iq->pc += iq->data[pi].num_wqebbs; mlx5e_iq_notify_hw(iq); mtx_unlock(&iq->lock); return (0); /* success */ } /* * This is the import function for TLS RX tags. */ static int mlx5e_tls_rx_tag_import(void *arg, void **store, int cnt, int domain, int flags) { struct mlx5e_tls_rx_tag *ptag; int i; for (i = 0; i != cnt; i++) { ptag = malloc_domainset(sizeof(*ptag), M_MLX5E_TLS_RX, mlx5_dev_domainset(arg), flags | M_ZERO); mtx_init(&ptag->mtx, "mlx5-tls-rx-tag-mtx", NULL, MTX_DEF); INIT_WORK(&ptag->work, mlx5e_tls_rx_work); store[i] = ptag; } return (i); } /* * This is the release function for TLS RX tags. */ static void mlx5e_tls_rx_tag_release(void *arg, void **store, int cnt) { struct mlx5e_tls_rx_tag *ptag; int i; for (i = 0; i != cnt; i++) { ptag = store[i]; flush_work(&ptag->work); mtx_destroy(&ptag->mtx); free(ptag, M_MLX5E_TLS_RX); } } /* * This is a convenience function to free TLS RX tags. It resets some * selected fields, updates the number of resources and returns the * TLS RX tag to the UMA pool of free tags. */ static void mlx5e_tls_rx_tag_zfree(struct mlx5e_tls_rx_tag *ptag) { /* make sure any unhandled taskqueue events are ignored */ ptag->state = MLX5E_TLS_RX_ST_FREED; /* reset some variables */ ptag->dek_index = 0; ptag->dek_index_ok = 0; ptag->tirn = 0; ptag->flow_rule = NULL; ptag->tcp_resync_active = 0; ptag->tcp_resync_pending = 0; /* avoid leaking keys */ memset(ptag->crypto_params, 0, sizeof(ptag->crypto_params)); /* update number of resources in use */ atomic_add_32(&ptag->tls_rx->num_resources, -1U); /* return tag to UMA */ uma_zfree(ptag->tls_rx->zone, ptag); } /* * This function enables TLS RX support for the given NIC, if all * needed firmware capabilites are present. */ int mlx5e_tls_rx_init(struct mlx5e_priv *priv) { struct mlx5e_tls_rx *ptls = &priv->tls_rx; struct sysctl_oid *node; uint32_t x; if (MLX5_CAP_GEN(priv->mdev, tls_rx) == 0 || MLX5_CAP_GEN(priv->mdev, log_max_dek) == 0 || MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version) == 0) return (0); ptls->wq = create_singlethread_workqueue("mlx5-tls-rx-wq"); if (ptls->wq == NULL) return (ENOMEM); sysctl_ctx_init(&ptls->ctx); snprintf(ptls->zname, sizeof(ptls->zname), "mlx5_%u_tls_rx", device_get_unit(priv->mdev->pdev->dev.bsddev)); ptls->zone = uma_zcache_create(ptls->zname, sizeof(struct mlx5e_tls_rx_tag), NULL, NULL, NULL, NULL, mlx5e_tls_rx_tag_import, mlx5e_tls_rx_tag_release, priv->mdev, UMA_ZONE_UNMANAGED); /* shared between RX and TX TLS */ ptls->max_resources = 1U << (MLX5_CAP_GEN(priv->mdev, log_max_dek) - 1); for (x = 0; x != MLX5E_TLS_RX_STATS_NUM; x++) ptls->stats.arg[x] = counter_u64_alloc(M_WAITOK); ptls->init = 1; node = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO, "tls_rx", CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "Hardware TLS receive offload"); if (node == NULL) return (0); mlx5e_create_counter_stats(&ptls->ctx, SYSCTL_CHILDREN(node), "stats", mlx5e_tls_rx_stats_desc, MLX5E_TLS_RX_STATS_NUM, ptls->stats.arg); return (0); } /* * This function disables TLS RX support for the given NIC. */ void mlx5e_tls_rx_cleanup(struct mlx5e_priv *priv) { struct mlx5e_tls_rx *ptls = &priv->tls_rx; uint32_t x; if (ptls->init == 0) return; ptls->init = 0; flush_workqueue(ptls->wq); sysctl_ctx_free(&ptls->ctx); uma_zdestroy(ptls->zone); destroy_workqueue(ptls->wq); /* check if all resources are freed */ MPASS(priv->tls_rx.num_resources == 0); for (x = 0; x != MLX5E_TLS_RX_STATS_NUM; x++) counter_u64_free(ptls->stats.arg[x]); } /* * This function is used to serialize sleeping firmware operations * needed in order to establish and destroy a TLS RX tag. */ static void mlx5e_tls_rx_work(struct work_struct *work) { struct mlx5e_tls_rx_tag *ptag; struct mlx5e_priv *priv; int err; ptag = container_of(work, struct mlx5e_tls_rx_tag, work); priv = container_of(ptag->tls_rx, struct mlx5e_priv, tls_rx); switch (ptag->state) { case MLX5E_TLS_RX_ST_INIT: /* try to allocate new TIR context */ err = mlx5_tls_open_tir(priv->mdev, priv->tdn, priv->channel[mlx5e_tls_rx_get_ch(priv, ptag->flowid, ptag->flowtype)].rqtn, &ptag->tirn); if (err) { MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1); break; } MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, progress.pd, ptag->tirn); /* try to allocate a DEK context ID */ err = mlx5_encryption_key_create(priv->mdev, priv->pdn, MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, key.key_data), MLX5_GET(sw_tls_rx_cntx, ptag->crypto_params, key.key_len), &ptag->dek_index); if (err) { MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1); break; } MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, param.dek_index, ptag->dek_index); ptag->dek_index_ok = 1; MLX5E_TLS_RX_TAG_LOCK(ptag); if (ptag->state == MLX5E_TLS_RX_ST_INIT) ptag->state = MLX5E_TLS_RX_ST_SETUP; MLX5E_TLS_RX_TAG_UNLOCK(ptag); break; case MLX5E_TLS_RX_ST_RELEASE: /* remove flow rule for incoming traffic, if any */ if (ptag->flow_rule != NULL) mlx5e_accel_fs_del_inpcb(ptag->flow_rule); /* try to destroy DEK context by ID */ if (ptag->dek_index_ok) mlx5_encryption_key_destroy(priv->mdev, ptag->dek_index); /* try to destroy TIR context by ID */ if (ptag->tirn != 0) mlx5_tls_close_tir(priv->mdev, ptag->tirn); /* free tag */ mlx5e_tls_rx_tag_zfree(ptag); break; default: break; } } /* * This function translates the crypto parameters into the format used * by the firmware and hardware. Currently only AES-128 and AES-256 is * supported for TLS v1.2 and TLS v1.3. * * Returns zero on success, else an error happened. */ static int mlx5e_tls_rx_set_params(void *ctx, struct inpcb *inp, const struct tls_session_params *en) { uint32_t tcp_sn_he; uint64_t tls_sn_he; MLX5_SET(sw_tls_rx_cntx, ctx, param.const_2, 2); if (en->tls_vminor == TLS_MINOR_VER_TWO) MLX5_SET(sw_tls_rx_cntx, ctx, param.tls_version, 2); /* v1.2 */ else MLX5_SET(sw_tls_rx_cntx, ctx, param.tls_version, 3); /* v1.3 */ MLX5_SET(sw_tls_rx_cntx, ctx, param.const_1, 1); MLX5_SET(sw_tls_rx_cntx, ctx, param.encryption_standard, 1); /* TLS */ /* copy the initial vector in place */ switch (en->iv_len) { case MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param.gcm_iv): case MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param.gcm_iv) + MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param.implicit_iv): memcpy(MLX5_ADDR_OF(sw_tls_rx_cntx, ctx, param.gcm_iv), en->iv, en->iv_len); break; default: return (EINVAL); } if (en->cipher_key_len <= MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, key.key_data)) { memcpy(MLX5_ADDR_OF(sw_tls_rx_cntx, ctx, key.key_data), en->cipher_key, en->cipher_key_len); MLX5_SET(sw_tls_rx_cntx, ctx, key.key_len, en->cipher_key_len); } else { return (EINVAL); } if (__predict_false(inp == NULL || ktls_get_rx_sequence(inp, &tcp_sn_he, &tls_sn_he) != 0)) return (EINVAL); MLX5_SET64(sw_tls_rx_cntx, ctx, param.initial_record_number, tls_sn_he); MLX5_SET(sw_tls_rx_cntx, ctx, param.resync_tcp_sn, tcp_sn_he); return (0); } /* Verify zero default */ CTASSERT(MLX5E_TLS_RX_ST_INIT == 0); /* - * This functino is responsible for allocating a TLS RX tag. It is a + * This function is responsible for allocating a TLS RX tag. It is a * callback function invoked by the network stack. * * Returns zero on success else an error happened. */ int mlx5e_tls_rx_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, struct m_snd_tag **ppmt) { struct mlx5e_iq *iq; struct mlx5e_priv *priv; struct mlx5e_tls_rx_tag *ptag; struct mlx5_flow_rule *flow_rule; const struct tls_session_params *en; uint32_t value; int error; priv = ifp->if_softc; if (unlikely(priv->gone != 0 || priv->tls_rx.init == 0 || params->hdr.flowtype == M_HASHTYPE_NONE)) return (EOPNOTSUPP); /* allocate new tag from zone, if any */ ptag = uma_zalloc(priv->tls_rx.zone, M_NOWAIT); if (ptag == NULL) return (ENOMEM); /* sanity check default values */ MPASS(ptag->dek_index == 0); MPASS(ptag->dek_index_ok == 0); /* setup TLS RX tag */ ptag->tls_rx = &priv->tls_rx; ptag->flowtype = params->hdr.flowtype; ptag->flowid = params->hdr.flowid; value = atomic_fetchadd_32(&priv->tls_rx.num_resources, 1U); /* check resource limits */ if (value >= priv->tls_rx.max_resources) { error = ENOMEM; goto failure; } en = ¶ms->tls_rx.tls->params; /* only TLS v1.2 and v1.3 is currently supported */ if (en->tls_vmajor != TLS_MAJOR_VER_ONE || (en->tls_vminor != TLS_MINOR_VER_TWO #ifdef TLS_MINOR_VER_THREE && en->tls_vminor != TLS_MINOR_VER_THREE #endif )) { error = EPROTONOSUPPORT; goto failure; } switch (en->cipher_algorithm) { case CRYPTO_AES_NIST_GCM_16: switch (en->cipher_key_len) { case 128 / 8: if (en->tls_vminor == TLS_MINOR_VER_TWO) { if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_128) == 0) { error = EPROTONOSUPPORT; goto failure; } } else { if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_128) == 0) { error = EPROTONOSUPPORT; goto failure; } } error = mlx5e_tls_rx_set_params( ptag->crypto_params, params->tls_rx.inp, en); if (error) goto failure; break; case 256 / 8: if (en->tls_vminor == TLS_MINOR_VER_TWO) { if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_256) == 0) { error = EPROTONOSUPPORT; goto failure; } } else { if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_256) == 0) { error = EPROTONOSUPPORT; goto failure; } } error = mlx5e_tls_rx_set_params( ptag->crypto_params, params->tls_rx.inp, en); if (error) goto failure; break; default: error = EINVAL; goto failure; } break; default: error = EPROTONOSUPPORT; goto failure; } /* store pointer to mbuf tag */ MPASS(ptag->tag.refcount == 0); m_snd_tag_init(&ptag->tag, ifp, &mlx5e_tls_rx_snd_tag_sw); *ppmt = &ptag->tag; /* reset state */ ptag->state = MLX5E_TLS_RX_ST_INIT; queue_work(priv->tls_rx.wq, &ptag->work); flush_work(&ptag->work); /* check that worker task completed successfully */ MLX5E_TLS_RX_TAG_LOCK(ptag); if (ptag->state == MLX5E_TLS_RX_ST_SETUP) { ptag->state = MLX5E_TLS_RX_ST_READY; error = 0; } else { error = ENOMEM; } MLX5E_TLS_RX_TAG_UNLOCK(ptag); if (unlikely(error)) goto cleanup; iq = mlx5e_tls_rx_get_iq(priv, ptag->flowid, ptag->flowtype); /* establish connection between DEK and TIR */ if (mlx5e_tls_rx_send_static_parameters(iq, ptag) != 0) { MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1); error = ENOMEM; goto cleanup; } MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, progress.auth_state, MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD); MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, progress.record_tracker_state, MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START); /* reset state to all zeros */ if (mlx5e_tls_rx_send_progress_parameters_sync(iq, ptag) != 0) { MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1); error = ENOMEM; goto cleanup; } if (ifp->if_pcp != IFNET_PCP_NONE || params->tls_rx.vlan_id != 0) { /* create flow rule for TLS RX traffic (tagged) */ flow_rule = mlx5e_accel_fs_add_inpcb(priv, params->tls_rx.inp, ptag->tirn, MLX5_FS_DEFAULT_FLOW_TAG, params->tls_rx.vlan_id); } else { /* create flow rule for TLS RX traffic (untagged) */ flow_rule = mlx5e_accel_fs_add_inpcb(priv, params->tls_rx.inp, ptag->tirn, MLX5_FS_DEFAULT_FLOW_TAG, MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN); } if (IS_ERR_OR_NULL(flow_rule)) { MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1); error = ENOMEM; goto cleanup; } ptag->flow_rule = flow_rule; return (0); cleanup: m_snd_tag_rele(&ptag->tag); return (error); failure: mlx5e_tls_rx_tag_zfree(ptag); return (error); } /* * This function adds the TCP sequence number and TLS record number in * host endian format to a small database. When TLS records have the * same length, they are simply accumulated by counting instead of * separated entries in the TLS database. The dimension of the * database is such that it cannot store more than 1GByte of * continuous TCP data to avoid issues with TCP sequence number wrap * around. A record length of zero bytes has special meaning and means * that resync completed and all data in the database can be * discarded. This function is called after the TCP stack has * re-assembled all TCP fragments due to out of order packet reception * and all TCP sequence numbers should be sequential. * * This function returns true if a so-called TLS RX resync operation * is in progress. Else no such operation is in progress. */ static bool mlx5e_tls_rx_snd_tag_add_tcp_sequence(struct mlx5e_tls_rx_tag *ptag, uint32_t tcp_sn_he, uint32_t len, uint64_t tls_rcd) { uint16_t i, j, n; if (ptag->tcp_resync_active == 0 || ptag->tcp_resync_next != tcp_sn_he || len == 0) { /* start over again or terminate */ ptag->tcp_resync_active = (len != 0); ptag->tcp_resync_len[0] = len; ptag->tcp_resync_num[0] = 1; ptag->tcp_resync_pc = (len != 0); ptag->tcp_resync_cc = 0; ptag->tcp_resync_start = tcp_sn_he; ptag->rcd_resync_start = tls_rcd; } else { i = (ptag->tcp_resync_pc - 1) & (MLX5E_TLS_RX_RESYNC_MAX - 1); n = ptag->tcp_resync_pc - ptag->tcp_resync_cc; /* check if same length like last time */ if (ptag->tcp_resync_len[i] == len && ptag->tcp_resync_num[i] != MLX5E_TLS_RX_NUM_MAX) { /* use existing entry */ ptag->tcp_resync_num[i]++; } else if (n == MLX5E_TLS_RX_RESYNC_MAX) { j = ptag->tcp_resync_cc++ & (MLX5E_TLS_RX_RESYNC_MAX - 1); /* adjust starting TCP sequence number */ ptag->rcd_resync_start += ptag->tcp_resync_num[j]; ptag->tcp_resync_start += ptag->tcp_resync_len[j] * ptag->tcp_resync_num[j]; i = ptag->tcp_resync_pc++ & (MLX5E_TLS_RX_RESYNC_MAX - 1); /* store new entry */ ptag->tcp_resync_len[i] = len; ptag->tcp_resync_num[i] = 1; } else { i = ptag->tcp_resync_pc++ & (MLX5E_TLS_RX_RESYNC_MAX - 1); /* add new entry */ ptag->tcp_resync_len[i] = len; ptag->tcp_resync_num[i] = 1; } } /* store next TCP SN in host endian format */ ptag->tcp_resync_next = tcp_sn_he + len; return (ptag->tcp_resync_active); } /* * This function checks if the given TCP sequence number points to the * beginning of a valid TLS header. * * Returns true if a match is found. Else false. */ static bool mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(struct mlx5e_tls_rx_tag *ptag, uint32_t tcp_sn_he, uint32_t *p_next_tcp_sn_he, uint64_t *p_tls_rcd) { uint16_t i, j; uint32_t off = 0; uint32_t rcd = 0; uint32_t delta; uint32_t leap; for (i = ptag->tcp_resync_cc; i != ptag->tcp_resync_pc; i++) { delta = tcp_sn_he - off - ptag->tcp_resync_start; /* check if subtraction went negative */ if ((int32_t)delta < 0) break; j = i & (MLX5E_TLS_RX_RESYNC_MAX - 1); leap = ptag->tcp_resync_len[j] * ptag->tcp_resync_num[j]; if (delta < leap) { if ((delta % ptag->tcp_resync_len[j]) == 0) { *p_next_tcp_sn_he = tcp_sn_he + ptag->tcp_resync_len[j]; *p_tls_rcd = ptag->rcd_resync_start + (uint64_t)rcd + (uint64_t)(delta / ptag->tcp_resync_len[j]); return (true); /* success */ } break; /* invalid offset */ } rcd += ptag->tcp_resync_num[j]; off += leap; } return (false); /* not found */ } /* * This is a callback function from the network stack to keep track of * TLS RX TCP sequence numbers. * * Returns zero on success else an error happened. */ static int mlx5e_tls_rx_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params) { struct mlx5e_tls_rx_tag *ptag; struct mlx5e_priv *priv; struct mlx5e_iq *iq; int err; ptag = container_of(pmt, struct mlx5e_tls_rx_tag, tag); priv = container_of(ptag->tls_rx, struct mlx5e_priv, tls_rx); if (unlikely(priv->gone != 0)) return (ENXIO); iq = mlx5e_tls_rx_get_iq(priv, ptag->flowid, ptag->flowtype); MLX5E_TLS_RX_TAG_LOCK(ptag); if (mlx5e_tls_rx_snd_tag_add_tcp_sequence(ptag, params->tls_rx.tls_hdr_tcp_sn, params->tls_rx.tls_rec_length, params->tls_rx.tls_seq_number) && ptag->tcp_resync_pending == 0) { err = mlx5e_tls_rx_receive_progress_parameters(iq, ptag); if (err != 0) { MLX5E_TLS_RX_STAT_INC(ptag, rx_resync_err, 1); } else { ptag->tcp_resync_pending = 1; MLX5E_TLS_RX_STAT_INC(ptag, rx_resync_ok, 1); } } else { err = 0; } MLX5E_TLS_RX_TAG_UNLOCK(ptag); return (-err); } /* * This function frees a TLS RX tag in a non-blocking way. */ static void mlx5e_tls_rx_snd_tag_free(struct m_snd_tag *pmt) { struct mlx5e_tls_rx_tag *ptag = container_of(pmt, struct mlx5e_tls_rx_tag, tag); struct mlx5e_priv *priv; MLX5E_TLS_RX_TAG_LOCK(ptag); ptag->state = MLX5E_TLS_RX_ST_RELEASE; MLX5E_TLS_RX_TAG_UNLOCK(ptag); priv = ptag->tag.ifp->if_softc; queue_work(priv->tls_rx.wq, &ptag->work); } #else int mlx5e_tls_rx_init(struct mlx5e_priv *priv) { return (0); } void mlx5e_tls_rx_cleanup(struct mlx5e_priv *priv) { /* NOP */ } #endif