Index: sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h =================================================================== --- sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h +++ sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h @@ -606,18 +606,16 @@ #define MPA_V2_RDMA_READ_RTR 0x4000 #define MPA_V2_IRD_ORD_MASK 0x3FFF -/* Fixme: Use atomic_read for kref.count as same as Linux */ #define c4iw_put_ep(ep) { \ CTR4(KTR_IW_CXGBE, "put_ep (%s:%u) ep %p, refcnt %d", \ - __func__, __LINE__, ep, (ep)->kref.count); \ - WARN_ON((ep)->kref.count < 1); \ + __func__, __LINE__, ep, atomic_read(&(ep)->kref.refcount)); \ + WARN_ON(atomic_read(&(ep)->kref.refcount) < 1); \ kref_put(&((ep)->kref), _c4iw_free_ep); \ } -/* Fixme: Use atomic_read for kref.count as same as Linux */ #define c4iw_get_ep(ep) { \ CTR4(KTR_IW_CXGBE, "get_ep (%s:%u) ep %p, refcnt %d", \ - __func__, __LINE__, ep, (ep)->kref.count); \ + __func__, __LINE__, ep, atomic_read(&(ep)->kref.refcount)); \ kref_get(&((ep)->kref)); \ } Index: sys/dev/cxgbe/iw_cxgbe/qp.c =================================================================== --- sys/dev/cxgbe/iw_cxgbe/qp.c +++ sys/dev/cxgbe/iw_cxgbe/qp.c @@ -1257,8 +1257,7 @@ case C4IW_QP_STATE_RTS: switch (attrs->next_state) { case C4IW_QP_STATE_CLOSING: - //Fixme: Use atomic_read as same as Linux - BUG_ON(qhp->ep->com.kref.count < 2); + BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); set_state(qhp, C4IW_QP_STATE_CLOSING); ep = qhp->ep; if (!internal) { Index: sys/ofed/drivers/infiniband/core/cma.c =================================================================== --- sys/ofed/drivers/infiniband/core/cma.c +++ sys/ofed/drivers/infiniband/core/cma.c @@ -2400,10 +2400,9 @@ { struct rdma_id_private *cur_id; struct sockaddr *addr, *cur_addr; - struct hlist_node *node; addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr; - hlist_for_each_entry(cur_id, node, &bind_list->owners, node) { + hlist_for_each_entry(cur_id, &bind_list->owners, node) { if (id_priv == cur_id) continue; Index: sys/ofed/drivers/infiniband/core/fmr_pool.c =================================================================== --- sys/ofed/drivers/infiniband/core/fmr_pool.c +++ sys/ofed/drivers/infiniband/core/fmr_pool.c @@ -118,14 +118,13 @@ { struct hlist_head *bucket; struct ib_pool_fmr *fmr; - struct hlist_node *pos; if (!pool->cache_bucket) return NULL; bucket = pool->cache_bucket + ib_fmr_hash(*page_list); - hlist_for_each_entry(fmr, pos, bucket, cache_node) + hlist_for_each_entry(fmr, bucket, cache_node) if (io_virtual_address == fmr->io_virtual_address && page_list_len == fmr->page_list_len && !memcmp(page_list, fmr->page_list, Index: sys/ofed/drivers/infiniband/core/uverbs_main.c =================================================================== --- sys/ofed/drivers/infiniband/core/uverbs_main.c +++ sys/ofed/drivers/infiniband/core/uverbs_main.c @@ -1168,7 +1168,7 @@ if (!dev) return -ENODEV; - return sprintf(buf, "%d\n", dev->ref.count); + return sprintf(buf, "%d\n", atomic_read(&dev->ref.refcount)); } static DEVICE_ATTR(ref_cnt, S_IRUGO, show_dev_ref_cnt, NULL); Index: sys/ofed/drivers/infiniband/hw/mlx4/main.c =================================================================== --- sys/ofed/drivers/infiniband/hw/mlx4/main.c +++ sys/ofed/drivers/infiniband/hw/mlx4/main.c @@ -2556,7 +2556,7 @@ } err_map: - iounmap(ibdev->priv_uar.map); + iounmap(ibdev->priv_uar.map, PAGE_SIZE); mlx4_ib_free_eqs(dev, ibdev); err_uar: @@ -2679,7 +2679,7 @@ pr_warn("failure unregistering notifier\n"); ibdev->iboe.nb.notifier_call = NULL; } - iounmap(ibdev->priv_uar.map); + iounmap(ibdev->priv_uar.map, PAGE_SIZE); for (p = 0; p < ibdev->num_ports; ++p) { if (mlx4_ib_port_link_layer(&ibdev->ib_dev, p + 1) == Index: sys/ofed/drivers/infiniband/hw/mthca/mthca_av.c =================================================================== --- sys/ofed/drivers/infiniband/hw/mthca/mthca_av.c +++ sys/ofed/drivers/infiniband/hw/mthca/mthca_av.c @@ -368,7 +368,8 @@ return; if (dev->av_table.av_map) - iounmap(dev->av_table.av_map); + iounmap(dev->av_table.av_map, + dev->av_table.num_ddr_avs * MTHCA_AV_SIZE); pci_pool_destroy(dev->av_table.pool); mthca_alloc_cleanup(&dev->av_table.alloc); } Index: sys/ofed/drivers/infiniband/hw/mthca/mthca_catas.c =================================================================== --- sys/ofed/drivers/infiniband/hw/mthca/mthca_catas.c +++ sys/ofed/drivers/infiniband/hw/mthca/mthca_catas.c @@ -174,7 +174,7 @@ del_timer_sync(&dev->catas_err.timer); if (dev->catas_err.map) - iounmap(dev->catas_err.map); + iounmap(dev->catas_err.map, dev->catas_err.size * 4); spin_lock_irq(&catas_lock); list_del(&dev->catas_err.list); Index: sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.c =================================================================== --- sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.c +++ sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.c @@ -490,7 +490,7 @@ MTHCA_MAILBOX_SIZE, MTHCA_MAILBOX_SIZE, 0); if (!dev->cmd.pool) { - iounmap(dev->hcr); + iounmap(dev->hcr, MTHCA_HCR_SIZE); return -ENOMEM; } @@ -500,9 +500,9 @@ void mthca_cmd_cleanup(struct mthca_dev *dev) { pci_pool_destroy(dev->cmd.pool); - iounmap(dev->hcr); + iounmap(dev->hcr, MTHCA_HCR_SIZE); if (dev->cmd.flags & MTHCA_CMD_POST_DOORBELLS) - iounmap(dev->cmd.dbell_map); + iounmap(dev->cmd.dbell_map, dev->cmd.dbell_size); } /* @@ -721,6 +721,7 @@ addr = pci_resource_start(dev->pdev, 2) + ((pci_resource_len(dev->pdev, 2) - 1) & base); dev->cmd.dbell_map = ioremap(addr, max_off + sizeof(u32)); + dev->cmd.dbell_size = max_off + sizeof(u32); if (!dev->cmd.dbell_map) return; Index: sys/ofed/drivers/infiniband/hw/mthca/mthca_dev.h =================================================================== --- sys/ofed/drivers/infiniband/hw/mthca/mthca_dev.h +++ sys/ofed/drivers/infiniband/hw/mthca/mthca_dev.h @@ -128,6 +128,7 @@ struct mthca_cmd_context *context; u16 token_mask; u32 flags; + u32 dbell_size; void __iomem *dbell_map; u16 dbell_offsets[MTHCA_CMD_NUM_DBELL_DWORDS]; }; @@ -217,6 +218,8 @@ struct mthca_icm_table *mtt_table; struct mthca_icm_table *mpt_table; struct { + unsigned long mpt_size; + unsigned long mtt_size; void __iomem *mpt_base; void __iomem *mtt_base; struct mthca_buddy mtt_buddy; Index: sys/ofed/drivers/infiniband/hw/mthca/mthca_eq.c =================================================================== --- sys/ofed/drivers/infiniband/hw/mthca/mthca_eq.c +++ sys/ofed/drivers/infiniband/hw/mthca/mthca_eq.c @@ -687,7 +687,7 @@ dev->fw.arbel.eq_arm_base) + 4, 4, &dev->eq_regs.arbel.eq_arm)) { mthca_err(dev, "Couldn't map EQ arm register, aborting.\n"); - iounmap(dev->clr_base); + iounmap(dev->clr_base, MTHCA_CLR_INT_SIZE); return -ENOMEM; } @@ -696,8 +696,8 @@ MTHCA_EQ_SET_CI_SIZE, &dev->eq_regs.arbel.eq_set_ci_base)) { mthca_err(dev, "Couldn't map EQ CI register, aborting.\n"); - iounmap(dev->eq_regs.arbel.eq_arm); - iounmap(dev->clr_base); + iounmap(dev->eq_regs.arbel.eq_arm, 4); + iounmap(dev->clr_base, MTHCA_CLR_INT_SIZE); return -ENOMEM; } } else { @@ -713,7 +713,7 @@ &dev->eq_regs.tavor.ecr_base)) { mthca_err(dev, "Couldn't map ecr register, " "aborting.\n"); - iounmap(dev->clr_base); + iounmap(dev->clr_base, MTHCA_CLR_INT_SIZE); return -ENOMEM; } } @@ -725,12 +725,12 @@ static void mthca_unmap_eq_regs(struct mthca_dev *dev) { if (mthca_is_memfree(dev)) { - iounmap(dev->eq_regs.arbel.eq_set_ci_base); - iounmap(dev->eq_regs.arbel.eq_arm); - iounmap(dev->clr_base); + iounmap(dev->eq_regs.arbel.eq_set_ci_base, MTHCA_EQ_SET_CI_SIZE); + iounmap(dev->eq_regs.arbel.eq_arm, 4); + iounmap(dev->clr_base, MTHCA_CLR_INT_SIZE); } else { - iounmap(dev->eq_regs.tavor.ecr_base); - iounmap(dev->clr_base); + iounmap(dev->eq_regs.tavor.ecr_base, MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE); + iounmap(dev->clr_base, MTHCA_CLR_INT_SIZE); } } Index: sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c =================================================================== --- sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c +++ sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c @@ -917,7 +917,7 @@ mthca_cleanup_pd_table(dev); err_kar_unmap: - iounmap(dev->kar); + iounmap(dev->kar, PAGE_SIZE); err_uar_free: mthca_uar_free(dev, &dev->driver_uar); @@ -1186,7 +1186,7 @@ mthca_cleanup_mr_table(mdev); mthca_cleanup_pd_table(mdev); - iounmap(mdev->kar); + iounmap(mdev->kar, PAGE_SIZE); mthca_uar_free(mdev, &mdev->driver_uar); mthca_cleanup_uar_table(mdev); mthca_close_hca(mdev); Index: sys/ofed/drivers/infiniband/hw/mthca/mthca_mr.c =================================================================== --- sys/ofed/drivers/infiniband/hw/mthca/mthca_mr.c +++ sys/ofed/drivers/infiniband/hw/mthca/mthca_mr.c @@ -894,8 +894,9 @@ ((pci_resource_len(dev->pdev, 4) - 1) & dev->mr_table.mpt_base); + dev->mr_table.tavor_fmr.mpt_size = mpts * sizeof(struct mthca_mpt_entry); dev->mr_table.tavor_fmr.mpt_base = - ioremap(addr, mpts * sizeof(struct mthca_mpt_entry)); + ioremap(addr, dev->mr_table.tavor_fmr.mpt_size); if (!dev->mr_table.tavor_fmr.mpt_base) { mthca_warn(dev, "MPT ioremap for FMR failed.\n"); @@ -907,8 +908,9 @@ ((pci_resource_len(dev->pdev, 4) - 1) & dev->mr_table.mtt_base); + dev->mr_table.tavor_fmr.mtt_size = mtts * dev->limits.mtt_seg_size; dev->mr_table.tavor_fmr.mtt_base = - ioremap(addr, mtts * dev->limits.mtt_seg_size); + ioremap(addr, dev->mr_table.tavor_fmr.mtt_size); if (!dev->mr_table.tavor_fmr.mtt_base) { mthca_warn(dev, "MTT ioremap for FMR failed.\n"); err = -ENOMEM; @@ -953,11 +955,13 @@ err_fmr_mtt_buddy: if (dev->mr_table.tavor_fmr.mtt_base) - iounmap(dev->mr_table.tavor_fmr.mtt_base); + iounmap(dev->mr_table.tavor_fmr.mtt_base, + dev->mr_table.tavor_fmr.mtt_size); err_fmr_mtt: if (dev->mr_table.tavor_fmr.mpt_base) - iounmap(dev->mr_table.tavor_fmr.mpt_base); + iounmap(dev->mr_table.tavor_fmr.mpt_base, + dev->mr_table.tavor_fmr.mpt_size); err_fmr_mpt: mthca_buddy_cleanup(&dev->mr_table.mtt_buddy); @@ -977,9 +981,11 @@ mthca_buddy_cleanup(&dev->mr_table.mtt_buddy); if (dev->mr_table.tavor_fmr.mtt_base) - iounmap(dev->mr_table.tavor_fmr.mtt_base); + iounmap(dev->mr_table.tavor_fmr.mtt_base, + dev->mr_table.tavor_fmr.mtt_size); if (dev->mr_table.tavor_fmr.mpt_base) - iounmap(dev->mr_table.tavor_fmr.mpt_base); + iounmap(dev->mr_table.tavor_fmr.mpt_base, + dev->mr_table.tavor_fmr.mpt_size); mthca_alloc_cleanup(&dev->mr_table.mpt_alloc); } Index: sys/ofed/drivers/infiniband/hw/mthca/mthca_reset.c =================================================================== --- sys/ofed/drivers/infiniband/hw/mthca/mthca_reset.c +++ sys/ofed/drivers/infiniband/hw/mthca/mthca_reset.c @@ -163,7 +163,7 @@ } writel(MTHCA_RESET_VALUE, reset); - iounmap(reset); + iounmap(reset, 4); } /* Docs say to wait one second before accessing device */ Index: sys/ofed/drivers/net/mlx4/alloc.c =================================================================== --- sys/ofed/drivers/net/mlx4/alloc.c +++ sys/ofed/drivers/net/mlx4/alloc.c @@ -285,7 +285,7 @@ buf->direct.map); else { if (BITS_PER_LONG == 64 && buf->direct.buf) - vunmap(buf->direct.buf); + vunmap(buf->direct.buf, buf->nbufs); for (i = 0; i < buf->nbufs; ++i) if (buf->page_list[i].buf) Index: sys/ofed/drivers/net/mlx4/catas.c =================================================================== --- sys/ofed/drivers/net/mlx4/catas.c +++ sys/ofed/drivers/net/mlx4/catas.c @@ -158,7 +158,7 @@ del_timer_sync(&priv->catas_err.timer); if (priv->catas_err.map) { - iounmap(priv->catas_err.map); + iounmap(priv->catas_err.map, priv->fw.catas_size * 4); priv->catas_err.map = NULL; } Index: sys/ofed/drivers/net/mlx4/cmd.c =================================================================== --- sys/ofed/drivers/net/mlx4/cmd.c +++ sys/ofed/drivers/net/mlx4/cmd.c @@ -2239,7 +2239,7 @@ err_comm_admin: kfree(priv->mfunc.master.slave_state); err_comm: - iounmap(priv->mfunc.comm); + iounmap(priv->mfunc.comm, MLX4_COMM_PAGESIZE); err_vhcr: dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, priv->mfunc.vhcr, @@ -2296,7 +2296,7 @@ err_hcr: if (!mlx4_is_slave(dev)) - iounmap(priv->cmd.hcr); + iounmap(priv->cmd.hcr, MLX4_HCR_SIZE); return -ENOMEM; } @@ -2317,7 +2317,7 @@ kfree(priv->mfunc.master.vf_oper); } - iounmap(priv->mfunc.comm); + iounmap(priv->mfunc.comm, MLX4_COMM_PAGESIZE); } void mlx4_cmd_cleanup(struct mlx4_dev *dev) @@ -2327,7 +2327,7 @@ pci_pool_destroy(priv->cmd.pool); if (!mlx4_is_slave(dev)) - iounmap(priv->cmd.hcr); + iounmap(priv->cmd.hcr, MLX4_HCR_SIZE); if (mlx4_is_mfunc(dev)) dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, priv->mfunc.vhcr, priv->mfunc.vhcr_dma); Index: sys/ofed/drivers/net/mlx4/en_main.c =================================================================== --- sys/ofed/drivers/net/mlx4/en_main.c +++ sys/ofed/drivers/net/mlx4/en_main.c @@ -164,7 +164,7 @@ ret = mlx4_mr_free(dev, &mdev->mr); if (ret) mlx4_err(mdev, "Error deregistering MR. The system may have become unstable."); - iounmap(mdev->uar_map); + iounmap(mdev->uar_map, PAGE_SIZE); mlx4_uar_free(dev, &mdev->priv_uar); mlx4_pd_free(dev, mdev->priv_pdn); kfree(mdev); @@ -277,7 +277,7 @@ mlx4_err(mdev, "Error deregistering MR. The system may have become unstable."); err_map: if (mdev->uar_map) - iounmap(mdev->uar_map); + iounmap(mdev->uar_map, PAGE_SIZE); err_uar: mlx4_uar_free(dev, &mdev->priv_uar); err_pd: Index: sys/ofed/drivers/net/mlx4/en_netdev.c =================================================================== --- sys/ofed/drivers/net/mlx4/en_netdev.c +++ sys/ofed/drivers/net/mlx4/en_netdev.c @@ -264,11 +264,10 @@ mlx4_en_filter_find(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, u8 ip_proto, __be16 src_port, __be16 dst_port) { - struct hlist_node *elem; struct mlx4_en_filter *filter; struct mlx4_en_filter *ret = NULL; - hlist_for_each_entry(filter, elem, + hlist_for_each_entry(filter, filter_hash_bucket(priv, src_ip, dst_ip, src_port, dst_port), filter_chain) { Index: sys/ofed/drivers/net/mlx4/en_resources.c =================================================================== --- sys/ofed/drivers/net/mlx4/en_resources.c +++ sys/ofed/drivers/net/mlx4/en_resources.c @@ -111,7 +111,7 @@ if (BITS_PER_LONG == 64 || buf->nbufs == 1) return; - vunmap(buf->direct.buf); + vunmap(buf->direct.buf, buf->nbufs); } void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event) Index: sys/ofed/drivers/net/mlx4/eq.c =================================================================== --- sys/ofed/drivers/net/mlx4/eq.c +++ sys/ofed/drivers/net/mlx4/eq.c @@ -895,7 +895,7 @@ for (i = 0; i < mlx4_num_eq_uar(dev); ++i) if (priv->eq_table.uar_map[i]) { - iounmap(priv->eq_table.uar_map[i]); + iounmap(priv->eq_table.uar_map[i], PAGE_SIZE); priv->eq_table.uar_map[i] = NULL; } } @@ -1102,7 +1102,7 @@ { struct mlx4_priv *priv = mlx4_priv(dev); - iounmap(priv->clr_base); + iounmap(priv->clr_base, MLX4_CLR_INT_SIZE); } int mlx4_alloc_eq_table(struct mlx4_dev *dev) Index: sys/ofed/drivers/net/mlx4/main.c =================================================================== --- sys/ofed/drivers/net/mlx4/main.c +++ sys/ofed/drivers/net/mlx4/main.c @@ -1786,7 +1786,7 @@ struct mlx4_priv *priv = mlx4_priv(dev); if (priv->clock_mapping) - iounmap(priv->clock_mapping); + iounmap(priv->clock_mapping, MLX4_CLOCK_SIZE); } static void mlx4_close_hca(struct mlx4_dev *dev) @@ -2854,7 +2854,7 @@ mlx4_cleanup_pd_table(dev); err_kar_unmap: - iounmap(priv->kar); + iounmap(priv->kar, PAGE_SIZE); err_uar_free: mlx4_uar_free(dev, &priv->driver_uar); @@ -3074,7 +3074,7 @@ } ret = readl(owner); - iounmap(owner); + iounmap(owner, MLX4_OWNER_SIZE); return (int) !!ret; } @@ -3093,7 +3093,7 @@ } writel(0, owner); msleep(1000); - iounmap(owner); + iounmap(owner, MLX4_OWNER_SIZE); } static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data) @@ -3484,7 +3484,7 @@ mlx4_free_resource_tracker(dev, RES_TR_FREE_STRUCTS_ONLY); - iounmap(priv->kar); + iounmap(priv->kar, PAGE_SIZE); mlx4_uar_free(dev, &priv->driver_uar); mlx4_cleanup_uar_table(dev); if (!mlx4_is_slave(dev)) Index: sys/ofed/drivers/net/mlx4/pd.c =================================================================== --- sys/ofed/drivers/net/mlx4/pd.c +++ sys/ofed/drivers/net/mlx4/pd.c @@ -227,7 +227,7 @@ unamp_uar: bf->uar = NULL; - iounmap(uar->map); + iounmap(uar->map, PAGE_SIZE); free_uar: mlx4_uar_free(dev, uar); @@ -257,7 +257,7 @@ list_del(&bf->uar->bf_list); io_mapping_unmap(bf->uar->bf_map); - iounmap(bf->uar->map); + iounmap(bf->uar->map, PAGE_SIZE); mlx4_uar_free(dev, bf->uar); kfree(bf->uar); } else if (list_empty(&bf->uar->bf_list)) Index: sys/ofed/drivers/net/mlx4/reset.c =================================================================== --- sys/ofed/drivers/net/mlx4/reset.c +++ sys/ofed/drivers/net/mlx4/reset.c @@ -111,13 +111,13 @@ if (sem) { mlx4_err(dev, "Failed to obtain HW semaphore, aborting\n"); err = -EAGAIN; - iounmap(reset); + iounmap(reset, MLX4_RESET_SIZE); goto out; } /* actually hit reset */ writel(MLX4_RESET_VALUE, reset + MLX4_RESET_OFFSET); - iounmap(reset); + iounmap(reset, MLX4_RESET_SIZE); /* wait half a second before accessing device */ msleep(500); Index: sys/ofed/include/linux/bitops.h =================================================================== --- sys/ofed/include/linux/bitops.h +++ sys/ofed/include/linux/bitops.h @@ -2,7 +2,7 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -29,16 +29,21 @@ #ifndef _LINUX_BITOPS_H_ #define _LINUX_BITOPS_H_ +#include +#include + +#define BIT(nr) (1UL << (nr)) #ifdef __LP64__ #define BITS_PER_LONG 64 #else #define BITS_PER_LONG 32 #endif -#define BIT_MASK(n) (~0UL >> (BITS_PER_LONG - (n))) +#define BIT_MASK(nr) (1UL << ((nr) & (BITS_PER_LONG - 1))) #define BITS_TO_LONGS(n) howmany((n), BITS_PER_LONG) -#define BIT_WORD(nr) ((nr) / BITS_PER_LONG) +#define BIT_WORD(nr) ((nr) / BITS_PER_LONG) +#define GENMASK(lo, hi) (((2UL << ((hi) - (lo))) - 1UL) << (lo)) -#define BITS_PER_BYTE 8 +#define BITS_PER_BYTE 8 static inline int __ffs(int mask) @@ -78,50 +83,6 @@ } static inline unsigned long -find_first_bit(unsigned long *addr, unsigned long size) -{ - long mask; - int bit; - - for (bit = 0; size >= BITS_PER_LONG; - size -= BITS_PER_LONG, bit += BITS_PER_LONG, addr++) { - if (*addr == 0) - continue; - return (bit + __ffsl(*addr)); - } - if (size) { - mask = (*addr) & BIT_MASK(size); - if (mask) - bit += __ffsl(mask); - else - bit += size; - } - return (bit); -} - -static inline unsigned long -find_first_zero_bit(unsigned long *addr, unsigned long size) -{ - long mask; - int bit; - - for (bit = 0; size >= BITS_PER_LONG; - size -= BITS_PER_LONG, bit += BITS_PER_LONG, addr++) { - if (~(*addr) == 0) - continue; - return (bit + __ffsl(~(*addr))); - } - if (size) { - mask = ~(*addr) & BIT_MASK(size); - if (mask) - bit += __ffsl(mask); - else - bit += size; - } - return (bit); -} - -static inline unsigned long find_last_bit(unsigned long *addr, unsigned long size) { long mask; @@ -134,7 +95,7 @@ bit = BITS_PER_LONG * pos; addr += pos; if (offs) { - mask = (*addr) & BIT_MASK(offs); + mask = (*addr) & (BIT_MASK(offs) - 1UL); if (mask) return (bit + __flsl(mask)); } @@ -148,82 +109,55 @@ } static inline unsigned long -find_next_bit(unsigned long *addr, unsigned long size, unsigned long offset) +find_next_bit(const unsigned long *addr, unsigned long size, + unsigned long offset) { - long mask; - int offs; - int bit; - int pos; + const unsigned long mm = BITS_PER_LONG - 1; + unsigned long mask = BIT_MASK(offset) - 1UL; - if (offset >= size) - return (size); - pos = offset / BITS_PER_LONG; - offs = offset % BITS_PER_LONG; - bit = BITS_PER_LONG * pos; - addr += pos; - if (offs) { - mask = (*addr) & ~BIT_MASK(offs); - if (mask) - return (bit + __ffsl(mask)); - if (size - bit <= BITS_PER_LONG) - return (size); - bit += BITS_PER_LONG; - addr++; - } - for (size -= bit; size >= BITS_PER_LONG; - size -= BITS_PER_LONG, bit += BITS_PER_LONG, addr++) { - if (*addr == 0) - continue; - return (bit + __ffsl(*addr)); - } - if (size) { - mask = (*addr) & BIT_MASK(size); - if (mask) - bit += __ffsl(mask); - else - bit += size; - } - return (bit); + while (offset < size) { + mask = addr[BIT_WORD(offset)] & ~mask; + if (mask != 0) { + offset = __ffsl(mask) | (offset & ~mm); + break; + } + offset += ((~offset) & mm) + 1; + } + if (offset > size) + offset = size; + return (offset); } static inline unsigned long -find_next_zero_bit(unsigned long *addr, unsigned long size, +find_next_zero_bit(const unsigned long *addr, unsigned long size, unsigned long offset) { - long mask; - int offs; - int bit; - int pos; + const unsigned long mm = BITS_PER_LONG - 1; + unsigned long mask = BIT_MASK(offset) - 1UL; - if (offset >= size) - return (size); - pos = offset / BITS_PER_LONG; - offs = offset % BITS_PER_LONG; - bit = BITS_PER_LONG * pos; - addr += pos; - if (offs) { - mask = ~(*addr) & ~BIT_MASK(offs); - if (mask) - return (bit + __ffsl(mask)); - if (size - bit <= BITS_PER_LONG) - return (size); - bit += BITS_PER_LONG; - addr++; - } - for (size -= bit; size >= BITS_PER_LONG; - size -= BITS_PER_LONG, bit += BITS_PER_LONG, addr++) { - if (~(*addr) == 0) - continue; - return (bit + __ffsl(~(*addr))); - } - if (size) { - mask = ~(*addr) & BIT_MASK(size); - if (mask) - bit += __ffsl(mask); - else - bit += size; - } - return (bit); + while (offset < size) { + mask = (~addr[BIT_WORD(offset)]) & ~mask; + if (mask != 0) { + offset = __ffsl(mask) | (offset & ~mm); + break; + } + offset += ((~offset) & mm) + 1; + } + if (offset > size) + offset = size; + return (offset); +} + +static inline unsigned long +find_first_bit(unsigned long *addr, unsigned long size) +{ + return (find_next_bit(addr, size, 0)); +} + +static inline unsigned long +find_first_zero_bit(unsigned long *addr, unsigned long size) +{ + return (find_next_zero_bit(addr, size, 0)); } static inline void @@ -245,13 +179,13 @@ memset(addr, 0xff, len); tail = size & (BITS_PER_LONG - 1); if (tail) - addr[size / BITS_PER_LONG] = BIT_MASK(tail); + addr[size / BITS_PER_LONG] = BIT_MASK(tail) - 1UL; } static inline int bitmap_full(unsigned long *addr, int size) { - long mask; + unsigned long mask; int tail; int len; int i; @@ -262,7 +196,7 @@ return (0); tail = size & (BITS_PER_LONG - 1); if (tail) { - mask = BIT_MASK(tail); + mask = BIT_MASK(tail) - 1UL; if ((addr[i] & mask) != mask) return (0); } @@ -283,38 +217,36 @@ return (0); tail = size & (BITS_PER_LONG - 1); if (tail) { - mask = BIT_MASK(tail); + mask = BIT_MASK(tail) - 1UL; if ((addr[i] & mask) != 0) return (0); } return (1); } -#define NBLONG (NBBY * sizeof(long)) - #define __set_bit(i, a) \ - atomic_set_long(&((volatile long *)(a))[(i)/NBLONG], 1UL << ((i) % NBLONG)) + atomic_set_long(&((volatile long *)(a))[BIT_WORD(i)], BIT_MASK(i)) #define set_bit(i, a) \ - atomic_set_long(&((volatile long *)(a))[(i)/NBLONG], 1UL << ((i) % NBLONG)) + atomic_set_long(&((volatile long *)(a))[BIT_WORD(i)], BIT_MASK(i)) #define __clear_bit(i, a) \ - atomic_clear_long(&((volatile long *)(a))[(i)/NBLONG], 1UL << ((i) % NBLONG)) + atomic_clear_long(&((volatile long *)(a))[BIT_WORD(i)], BIT_MASK(i)) #define clear_bit(i, a) \ - atomic_clear_long(&((volatile long *)(a))[(i)/NBLONG], 1UL << ((i) % NBLONG)) + atomic_clear_long(&((volatile long *)(a))[BIT_WORD(i)], BIT_MASK(i)) #define test_bit(i, a) \ - !!(atomic_load_acq_long(&((volatile long *)(a))[(i)/NBLONG]) & \ - (1UL << ((i) % NBLONG))) + ((atomic_load_acq_long(&((volatile long *)(a))[BIT_WORD(i)]) & \ + (BIT_MASK(i))) != 0) static inline long test_and_clear_bit(long bit, long *var) { long val; - var += bit / (sizeof(long) * NBBY); - bit %= sizeof(long) * NBBY; + var += BIT_WORD(bit); + bit %= BITS_PER_LONG; bit = (1UL << bit); do { val = *(volatile long *)var; @@ -328,8 +260,8 @@ { long val; - var += bit / (sizeof(long) * NBBY); - bit %= sizeof(long) * NBBY; + var += BIT_WORD(bit); + bit %= BITS_PER_LONG; bit = (1UL << bit); do { val = *(volatile long *)var; @@ -338,14 +270,9 @@ return !!(val & bit); } - -#define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) % BITS_PER_LONG)) -#define BITMAP_LAST_WORD_MASK(nbits) \ -( \ - ((nbits) % BITS_PER_LONG) ? \ - (1UL<<((nbits) % BITS_PER_LONG))-1 : ~0UL \ -) - +#define BITMAP_FIRST_WORD_MASK(start) (-BIT_MASK(start)) +#define BITMAP_LAST_WORD_MASK(nbits) \ + (((nbits) & (BITS_PER_LONG - 1)) ? (BIT_MASK(nbits) - 1UL) : -1UL) static inline void bitmap_set(unsigned long *map, int start, int nr) @@ -390,36 +317,28 @@ } enum { - REG_OP_ISFREE, /* true if region is all zero bits */ - REG_OP_ALLOC, /* set all bits in region */ - REG_OP_RELEASE, /* clear all bits in region */ + REG_OP_ISFREE, + REG_OP_ALLOC, + REG_OP_RELEASE, }; -static int __reg_op(unsigned long *bitmap, int pos, int order, int reg_op) +static inline int __reg_op(unsigned long *bitmap, int pos, int order, int reg_op) { - int nbits_reg; /* number of bits in region */ - int index; /* index first long of region in bitmap */ - int offset; /* bit offset region in bitmap[index] */ - int nlongs_reg; /* num longs spanned by region in bitmap */ - int nbitsinlong; /* num bits of region in each spanned long */ - unsigned long mask; /* bitmask for one long of region */ - int i; /* scans bitmap by longs */ - int ret = 0; /* return value */ - - /* - * Either nlongs_reg == 1 (for small orders that fit in one long) - * or (offset == 0 && mask == ~0UL) (for larger multiword orders.) - */ + int nbits_reg; + int index; + int offset; + int nlongs_reg; + int nbitsinlong; + unsigned long mask; + int i; + int ret = 0; + nbits_reg = 1 << order; index = pos / BITS_PER_LONG; offset = pos - (index * BITS_PER_LONG); nlongs_reg = BITS_TO_LONGS(nbits_reg); nbitsinlong = min(nbits_reg, BITS_PER_LONG); - /* - * Can't do "mask = (1UL << nbitsinlong) - 1", as that - * overflows if nbitsinlong == BITS_PER_LONG. - */ mask = (1UL << (nbitsinlong - 1)); mask += mask - 1; mask <<= offset; @@ -430,7 +349,7 @@ if (bitmap[index + i] & mask) goto done; } - ret = 1; /* all bits in region free (zero) */ + ret = 1; break; case REG_OP_ALLOC: @@ -447,24 +366,11 @@ return ret; } -/** - * bitmap_find_free_region - find a contiguous aligned mem region - * @bitmap: array of unsigned longs corresponding to the bitmap - * @bits: number of bits in the bitmap - * @order: region size (log base 2 of number of bits) to find - * - * Find a region of free (zero) bits in a @bitmap of @bits bits and - * allocate them (set them to one). Only consider regions of length - * a power (@order) of two, aligned to that power of two, which - * makes the search algorithm much faster. - * - * Return the bit offset in bitmap of the allocated region, - * or -errno on failure. - */ static inline int bitmap_find_free_region(unsigned long *bitmap, int bits, int order) { - int pos, end; /* scans bitmap by regions of size order */ + int pos; + int end; for (pos = 0 ; (end = pos + (1 << order)) <= bits; pos = end) { if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) @@ -475,18 +381,6 @@ return -ENOMEM; } -/** - * bitmap_allocate_region - allocate bitmap region - * @bitmap: array of unsigned longs corresponding to the bitmap - * @pos: beginning of bit region to allocate - * @order: region size (log base 2 of number of bits) to allocate - * - * Allocate (set bits in) a specified region of a bitmap. - * - * Return 0 on success, or %-EBUSY if specified region wasn't - * free (not all bits were zero). - */ - static inline int bitmap_allocate_region(unsigned long *bitmap, int pos, int order) { @@ -496,25 +390,13 @@ return 0; } -/** - * bitmap_release_region - release allocated bitmap region - * @bitmap: array of unsigned longs corresponding to the bitmap - * @pos: beginning of bit region to release - * @order: region size (log base 2 of number of bits) to release - * - * This is the complement to __bitmap_find_free_region() and releases - * the found region (by clearing it in the bitmap). - * - * No return value. - */ static inline void bitmap_release_region(unsigned long *bitmap, int pos, int order) { __reg_op(bitmap, pos, order, REG_OP_RELEASE); } - -#define for_each_set_bit(bit, addr, size) \ +#define for_each_set_bit(bit, addr, size) \ for ((bit) = find_first_bit((addr), (size)); \ (bit) < (size); \ (bit) = find_next_bit((addr), (size), (bit) + 1)) Index: sys/ofed/include/linux/compiler.h =================================================================== --- sys/ofed/include/linux/compiler.h +++ sys/ofed/include/linux/compiler.h @@ -2,7 +2,8 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. + * Copyright (c) 2015 François Tigeot * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -62,5 +63,12 @@ #define typeof(x) __typeof(x) #define uninitialized_var(x) x = x +#define __read_mostly __attribute__((__section__(".data.read_mostly"))) +#define __always_unused __unused +#define __must_check __result_use_check + +#define __printf(a,b) __printflike(a,b) + +#define barrier() __asm__ __volatile__("": : :"memory") #endif /* _LINUX_COMPILER_H_ */ Index: sys/ofed/include/linux/delay.h =================================================================== --- sys/ofed/include/linux/delay.h +++ sys/ofed/include/linux/delay.h @@ -2,7 +2,8 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. + * Copyright (c) 2014 François Tigeot * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,6 +32,7 @@ #define _LINUX_DELAY_H_ #include +#include static inline void linux_msleep(int ms) @@ -41,4 +43,26 @@ #undef msleep #define msleep linux_msleep +#define udelay(t) DELAY(t) + +static inline void +mdelay(unsigned long msecs) +{ + int loops = msecs; + while (loops--) + DELAY(1000); +} + +static inline void +ndelay(unsigned long x) +{ + DELAY(howmany(x, 1000)); +} + +static inline void +usleep_range(unsigned long min, unsigned long max) +{ + DELAY(min); +} + #endif /* _LINUX_DELAY_H_ */ Index: sys/ofed/include/linux/dma-mapping.h =================================================================== --- sys/ofed/include/linux/dma-mapping.h +++ sys/ofed/include/linux/dma-mapping.h @@ -87,7 +87,7 @@ int is_phys; }; -#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL << (n)) - 1)) +#define DMA_BIT_MASK(n) ((2ULL << ((n) - 1)) - 1ULL) static inline int dma_supported(struct device *dev, u64 mask) Index: sys/ofed/include/linux/err.h =================================================================== --- sys/ofed/include/linux/err.h +++ sys/ofed/include/linux/err.h @@ -2,7 +2,7 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -52,6 +52,12 @@ return IS_ERR_VALUE((unsigned long)ptr); } +static inline long +IS_ERR_OR_NULL(const void *ptr) +{ + return !ptr || IS_ERR_VALUE((unsigned long)ptr); +} + static inline void * ERR_CAST(void *ptr) { Index: sys/ofed/include/linux/errno.h =================================================================== --- sys/ofed/include/linux/errno.h +++ sys/ofed/include/linux/errno.h @@ -32,6 +32,8 @@ #include +#define ECHRNG EDOM +#define ETIME ETIMEDOUT #define ECOMM ESTALE #define ENODATA ECONNREFUSED #define ENOIOCTLCMD ENOIOCTL Index: sys/ofed/include/linux/io-mapping.h =================================================================== --- sys/ofed/include/linux/io-mapping.h +++ sys/ofed/include/linux/io-mapping.h @@ -2,7 +2,8 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. + * Copyright (c) 2015 François Tigeot * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -33,46 +34,64 @@ #include #include -struct io_mapping; +#include + +struct io_mapping { + resource_size_t base; + unsigned long size; + unsigned long prot; + caddr_t vaddr; +}; static inline struct io_mapping * io_mapping_create_wc(resource_size_t base, unsigned long size) { + struct io_mapping *map; - return ioremap_wc(base, size); + map = malloc(sizeof(struct io_mapping), M_KMALLOC, M_WAITOK); + map->base = base; + map->size = size; + map->prot = VM_MEMATTR_WRITE_COMBINING; + + map->vaddr = pmap_mapdev_attr(base, size, + VM_MEMATTR_WRITE_COMBINING); + if (map->vaddr == NULL) { + free(map, M_KMALLOC); + return (NULL); + } + return (map); } static inline void io_mapping_free(struct io_mapping *mapping) { - - iounmap(mapping); + /* Default memory attribute is write-back */ + pmap_mapdev_attr(mapping->base, mapping->size, VM_MEMATTR_WRITE_BACK); + free(mapping, M_KMALLOC); } static inline void * io_mapping_map_atomic_wc(struct io_mapping *mapping, unsigned long offset) { - - return (((char *)mapping) + offset); -} - -static inline void -io_mapping_unmap_atomic(void *vaddr) -{ - + return ((void *)(mapping->vaddr + offset)); } static inline void * io_mapping_map_wc(struct io_mapping *mapping, unsigned long offset) { + return (io_mapping_map_atomic_wc(mapping, offset)); +} - return (((char *) mapping) + offset); +static inline void +io_mapping_unmap_atomic(void *vaddr) +{ + /* NOP */ } static inline void io_mapping_unmap(void *vaddr) { - + /* NOP */ } #endif /* _LINUX_IO_MAPPING_H_ */ Index: sys/ofed/include/linux/io.h =================================================================== --- sys/ofed/include/linux/io.h +++ sys/ofed/include/linux/io.h @@ -2,7 +2,7 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -60,7 +60,7 @@ /* * XXX This is all x86 specific. It should be bus space access. */ -#define mmiowb() +#define mmiowb() do { } while (0) #undef writel static inline void @@ -104,13 +104,10 @@ *(volatile uint32_t *)addr = htobe32(v); } -void *_ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr); -#define ioremap_nocache(addr, size) \ - _ioremap_attr((addr), (size), VM_MEMATTR_UNCACHEABLE) -#define ioremap_wc(addr, size) \ - _ioremap_attr((addr), (size), VM_MEMATTR_WRITE_COMBINING) -#define ioremap ioremap_nocache -void iounmap(void *addr); +void *ioremap(resource_size_t offset, unsigned long size); +void *ioremap_nocache(resource_size_t offset, unsigned long size); +void *ioremap_wc(resource_size_t offset, unsigned long size); +void iounmap(void *addr, unsigned long size); #define memset_io(a, b, c) memset((a), (b), (c)) #define memcpy_fromio(a, b, c) memcpy((a), (b), (c)) Index: sys/ofed/include/linux/jiffies.h =================================================================== --- sys/ofed/include/linux/jiffies.h +++ sys/ofed/include/linux/jiffies.h @@ -31,9 +31,11 @@ #include #include +#include #include #include +#include static inline int msecs_to_jiffies(int msec) @@ -46,13 +48,49 @@ } #define jiffies ticks +#define jiffies_64 ticks #define jiffies_to_msecs(x) (((int64_t)(x)) * 1000 / hz) +#define MAX_JIFFY_OFFSET ((INT_MAX >> 1) - 1) + #define time_after(a, b) ((int)((b) - (a)) < 0) #define time_before(a, b) time_after(b,a) #define time_after_eq(a, b) ((int)((a) - (b)) >= 0) #define time_before_eq(a, b) time_after_eq(b, a) +#define time_in_range(a,b,c) \ + (time_after_eq(a,b) && time_before_eq(a,c)) #define HZ hz +static inline int +timespec_to_jiffies(const struct timespec *ts) +{ + u64 result; + + result = ((u64)hz * ts->tv_sec) + + (((u64)hz * ts->tv_nsec + NSEC_PER_SEC - 1) / NSEC_PER_SEC); + if (result > MAX_JIFFY_OFFSET) + result = MAX_JIFFY_OFFSET; + + return ((int)result); +} + +static inline int +usecs_to_jiffies(const unsigned int u) +{ + u64 result; + + result = ((u64)u * hz + 1000000 - 1) / 1000000; + if (result > MAX_JIFFY_OFFSET) + result = MAX_JIFFY_OFFSET; + + return ((int)result); +} + +static inline u64 +get_jiffies_64(void) +{ + return ((u64)(unsigned)ticks); +} + #endif /* _LINUX_JIFFIES_H_ */ Index: sys/ofed/include/linux/kernel.h =================================================================== --- sys/ofed/include/linux/kernel.h +++ sys/ofed/include/linux/kernel.h @@ -2,7 +2,8 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. + * Copyright (c) 2014-2015 François Tigeot * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -70,6 +71,7 @@ #undef PTR_ALIGN #define PTR_ALIGN(p, a) ((__typeof(p))ALIGN((uintptr_t)(p), (a))) #define DIV_ROUND_UP howmany +#define DIV_ROUND_UP_ULL(x, n) DIV_ROUND_UP((unsigned long long)(x), (n)) #define FIELD_SIZEOF(t, f) sizeof(((t *)0)->f) #define printk(X...) printf(X) @@ -90,9 +92,6 @@ ({ if (0) log(LOG_DEBUG, pr_fmt(fmt), ##__VA_ARGS__); 0; }) #endif -#define udelay(t) DELAY(t) -#define usleep_range(min,max) DELAY(min) - #ifndef pr_fmt #define pr_fmt(fmt) fmt #endif @@ -164,9 +163,16 @@ #define min(x, y) ((x) < (y) ? (x) : (y)) #define max(x, y) ((x) > (y) ? (x) : (y)) + +#define min3(a, b, c) min(a, min(b,c)) +#define max3(a, b, c) max(a, max(b,c)) + #define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y)) #define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y)) +#define clamp_t(type, _x, min, max) min_t(type, max_t(type, _x, min), max) +#define clamp(x, lo, hi) min( max(x,lo), hi) + /* * This looks more complex than it should be. But we need to * get the type for the ~ right in round_down (it needs to be @@ -184,4 +190,28 @@ int event; } pm_message_t; +/* Swap values of a and b */ +#define swap(a, b) do { \ + typeof(a) _swap_tmp = a; \ + a = b; \ + b = _swap_tmp; \ +} while (0) + +#define DIV_ROUND_CLOSEST(x, divisor) (((x) + ((divisor) / 2)) / (divisor)) + +static inline uintmax_t +mult_frac(uintmax_t x, uintmax_t multiplier, uintmax_t divisor) +{ + uintmax_t q = (x / divisor); + uintmax_t r = (x % divisor); + + return ((q * multiplier) + ((r * multiplier) / divisor)); +} + +static inline int64_t +abs64(int64_t x) +{ + return (x < 0 ? -x : x); +} + #endif /* _LINUX_KERNEL_H_ */ Index: sys/ofed/include/linux/kref.h =================================================================== --- sys/ofed/include/linux/kref.h +++ sys/ofed/include/linux/kref.h @@ -2,7 +2,8 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. + * Copyright (c) 2013 François Tigeot * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -32,33 +33,57 @@ #include #include +#include + struct kref { - volatile u_int count; + atomic_t refcount; }; static inline void kref_init(struct kref *kref) { - refcount_init(&kref->count, 1); + refcount_init(&kref->refcount.counter, 1); } static inline void kref_get(struct kref *kref) { - refcount_acquire(&kref->count); + refcount_acquire(&kref->refcount.counter); } static inline int kref_put(struct kref *kref, void (*rel)(struct kref *kref)) { - if (refcount_release(&kref->count)) { + if (refcount_release(&kref->refcount.counter)) { rel(kref); return 1; } return 0; } +static inline int +kref_sub(struct kref *kref, unsigned int count, + void (*rel)(struct kref *kref)) +{ + while (count--) { + if (refcount_release(&kref->refcount.counter)) { + rel(kref); + return 1; + } + } + return 0; +} + +/* + * kref_get_unless_zero: Increment refcount for object unless it is zero. + */ +static inline int __must_check +kref_get_unless_zero(struct kref *kref) +{ + return atomic_add_unless(&kref->refcount, 1, 0); +} + #endif /* _LINUX_KREF_H_ */ Index: sys/ofed/include/linux/ktime.h =================================================================== --- sys/ofed/include/linux/ktime.h +++ sys/ofed/include/linux/ktime.h @@ -1,5 +1,6 @@ /*- - * Copyright (c) 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2014-2015 Mellanox Technologies, Ltd. + * Copyright (c) 2015 François Tigeot * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -27,274 +28,107 @@ #ifndef _LINUX_KTIME_H #define _LINUX_KTIME_H -#include #include +#include #include +#define ktime_get_ts(x) getnanouptime(x) -/* Get the monotonic time in timespec format: */ -#define ktime_get_ts getnanouptime - -#define NSEC_PER_USEC 1000L -#define NSEC_PER_SEC 1000000000L - -/* - * ktime_t: - * - * On 64-bit CPUs a single 64-bit variable is used to store the hrtimers - * internal representation of time values in scalar nanoseconds. The - * design plays out best on 64-bit CPUs, where most conversions are - * NOPs and most arithmetic ktime_t operations are plain arithmetic - * operations. - * - * On 32-bit CPUs an optimized representation of the timespec structure - * is used to avoid expensive conversions from and to timespecs. The - * endian-aware order of the tv struct members is chosen to allow - * mathematical operations on the tv64 member of the union too, which - * for certain operations produces better code. - * - * For architectures with efficient support for 64/32-bit conversions the - * plain scalar nanosecond based representation can be selected by the - * config switch CONFIG_KTIME_SCALAR. - */ +/* time values in nanoseconds */ union ktime { - s64 tv64; -#if BITS_PER_LONG != 64 && !defined(CONFIG_KTIME_SCALAR) - struct { -# ifdef __BIG_ENDIAN - s32 sec, nsec; -# else - s32 nsec, sec; -# endif - } tv; -#endif + int64_t tv64; }; -typedef union ktime ktime_t; /* Kill this */ +typedef union ktime ktime_t; #define KTIME_MAX ((s64)~((u64)1 << 63)) #define KTIME_SEC_MAX (KTIME_MAX / NSEC_PER_SEC) -/* - * ktime_t definitions when using the 64-bit scalar representation: - */ - -#if (BITS_PER_LONG == 64) || defined(CONFIG_KTIME_SCALAR) - -/** - * ktime_set - Set a ktime_t variable from a seconds/nanoseconds value - * @secs: seconds to set - * @nsecs: nanoseconds to set - * - * Return the ktime_t representation of the value - */ -static inline ktime_t ktime_set(const long secs, const unsigned long nsecs) -{ -#if (BITS_PER_LONG == 64) - if (unlikely(secs >= KTIME_SEC_MAX)) - return (ktime_t){ .tv64 = KTIME_MAX }; -#endif - return (ktime_t) { .tv64 = (s64)secs * NSEC_PER_SEC + (s64)nsecs }; -} - -/* Subtract two ktime_t variables. rem = lhs -rhs: */ -#define ktime_sub(lhs, rhs) \ - ({ (ktime_t){ .tv64 = (lhs).tv64 - (rhs).tv64 }; }) - -/* Add two ktime_t variables. res = lhs + rhs: */ -#define ktime_add(lhs, rhs) \ - ({ (ktime_t){ .tv64 = (lhs).tv64 + (rhs).tv64 }; }) - -/* - * Add a ktime_t variable and a scalar nanosecond value. - * res = kt + nsval: - */ -#define ktime_add_ns(kt, nsval) \ - ({ (ktime_t){ .tv64 = (kt).tv64 + (nsval) }; }) - -/* - * Subtract a scalar nanosecod from a ktime_t variable - * res = kt - nsval: - */ -#define ktime_sub_ns(kt, nsval) \ - ({ (ktime_t){ .tv64 = (kt).tv64 - (nsval) }; }) - -/* convert a timespec to ktime_t format: */ -static inline ktime_t timespec_to_ktime(struct timespec ts) +static inline int64_t +ktime_to_ns(ktime_t kt) { - return ktime_set(ts.tv_sec, ts.tv_nsec); + return kt.tv64; } -/* convert a timeval to ktime_t format: */ -static inline ktime_t timeval_to_ktime(struct timeval tv) +static inline struct timeval +ktime_to_timeval(ktime_t kt) { - return ktime_set(tv.tv_sec, tv.tv_usec * NSEC_PER_USEC); + return ns_to_timeval(kt.tv64); } -/* Map the ktime_t to timespec conversion to ns_to_timespec function */ -#define ktime_to_timespec(kt) ns_to_timespec((kt).tv64) - -/* Map the ktime_t to timeval conversion to ns_to_timeval function */ -#define ktime_to_timeval(kt) ns_to_timeval((kt).tv64) - -/* Convert ktime_t to nanoseconds - NOP in the scalar storage format: */ -#define ktime_to_ns(kt) ((kt).tv64) - -#else /* !((BITS_PER_LONG == 64) || defined(CONFIG_KTIME_SCALAR)) */ - -/* - * Helper macros/inlines to get the ktime_t math right in the timespec - * representation. The macros are sometimes ugly - their actual use is - * pretty okay-ish, given the circumstances. We do all this for - * performance reasons. The pure scalar nsec_t based code was nice and - * simple, but created too many 64-bit / 32-bit conversions and divisions. - * - * Be especially aware that negative values are represented in a way - * that the tv.sec field is negative and the tv.nsec field is greater - * or equal to zero but less than nanoseconds per second. This is the - * same representation which is used by timespecs. - * - * tv.sec < 0 and 0 >= tv.nsec < NSEC_PER_SEC - */ - -/* Set a ktime_t variable to a value in sec/nsec representation: */ -static inline ktime_t ktime_set(const long secs, const unsigned long nsecs) -{ - return (ktime_t) { .tv = { .sec = secs, .nsec = nsecs } }; -} - -/** - * ktime_sub - subtract two ktime_t variables - * @lhs: minuend - * @rhs: subtrahend - * - * Returns the remainder of the subtraction - */ -static inline ktime_t ktime_sub(const ktime_t lhs, const ktime_t rhs) +static inline ktime_t +ktime_add_ns(ktime_t kt, int64_t ns) { ktime_t res; - res.tv64 = lhs.tv64 - rhs.tv64; - if (res.tv.nsec < 0) - res.tv.nsec += NSEC_PER_SEC; - - return res; + res.tv64 = kt.tv64 + ns; + return kt; } -/** - * ktime_add - add two ktime_t variables - * @add1: addend1 - * @add2: addend2 - * - * Returns the sum of @add1 and @add2. - */ -static inline ktime_t ktime_add(const ktime_t add1, const ktime_t add2) +static inline ktime_t +ktime_sub_ns(ktime_t kt, int64_t ns) { ktime_t res; - res.tv64 = add1.tv64 + add2.tv64; - /* - * performance trick: the (u32) -NSEC gives 0x00000000Fxxxxxxx - * so we subtract NSEC_PER_SEC and add 1 to the upper 32 bit. - * - * it's equivalent to: - * tv.nsec -= NSEC_PER_SEC - * tv.sec ++; - */ - if (res.tv.nsec >= NSEC_PER_SEC) - res.tv64 += (u32)-NSEC_PER_SEC; - - return res; -} - -/** - * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable - * @kt: addend - * @nsec: the scalar nsec value to add - * - * Returns the sum of @kt and @nsec in ktime_t format - */ -extern ktime_t ktime_add_ns(const ktime_t kt, u64 nsec); - -/** - * ktime_sub_ns - Subtract a scalar nanoseconds value from a ktime_t variable - * @kt: minuend - * @nsec: the scalar nsec value to subtract - * - * Returns the subtraction of @nsec from @kt in ktime_t format - */ -extern ktime_t ktime_sub_ns(const ktime_t kt, u64 nsec); + res.tv64 = kt.tv64 - ns; + return kt; +} -/** - * timespec_to_ktime - convert a timespec to ktime_t format - * @ts: the timespec variable to convert - * - * Returns a ktime_t variable with the converted timespec value - */ -static inline ktime_t timespec_to_ktime(const struct timespec ts) +static inline ktime_t +ktime_set(const long secs, const unsigned long nsecs) { - return (ktime_t) { .tv = { .sec = (s32)ts.tv_sec, - .nsec = (s32)ts.tv_nsec } }; + ktime_t retval = { (s64)secs * NSEC_PER_SEC + (s64)nsecs }; + return (retval); } -/** - * timeval_to_ktime - convert a timeval to ktime_t format - * @tv: the timeval variable to convert - * - * Returns a ktime_t variable with the converted timeval value - */ -static inline ktime_t timeval_to_ktime(const struct timeval tv) +static inline ktime_t +ktime_sub(ktime_t lhs, ktime_t rhs) { - return (ktime_t) { .tv = { .sec = (s32)tv.tv_sec, - .nsec = (s32)(tv.tv_usec * - NSEC_PER_USEC) } }; + lhs.tv64 -= rhs.tv64; + return (lhs); } -/** - * ktime_to_timespec - convert a ktime_t variable to timespec format - * @kt: the ktime_t variable to convert - * - * Returns the timespec representation of the ktime value - */ -static inline struct timespec ktime_to_timespec(const ktime_t kt) +static inline ktime_t +ktime_add(ktime_t lhs, ktime_t rhs) { - return (struct timespec) { .tv_sec = (time_t) kt.tv.sec, - .tv_nsec = (long) kt.tv.nsec }; + lhs.tv64 += rhs.tv64; + return (lhs); } -/** - * ktime_to_timeval - convert a ktime_t variable to timeval format - * @kt: the ktime_t variable to convert - * - * Returns the timeval representation of the ktime value - */ -static inline struct timeval ktime_to_timeval(const ktime_t kt) +static inline ktime_t +timespec_to_ktime(struct timespec ts) { - return (struct timeval) { - .tv_sec = (time_t) kt.tv.sec, - .tv_usec = (suseconds_t) (kt.tv.nsec / NSEC_PER_USEC) }; + return (ktime_set(ts.tv_sec, ts.tv_nsec)); } -/** - * ktime_to_ns - convert a ktime_t variable to scalar nanoseconds - * @kt: the ktime_t variable to convert - * - * Returns the scalar nanoseconds representation of @kt - */ -static inline s64 ktime_to_ns(const ktime_t kt) +static inline ktime_t +timeval_to_ktime(struct timeval tv) { - return (s64) kt.tv.sec * NSEC_PER_SEC + kt.tv.nsec; + return (ktime_set(tv.tv_sec, tv.tv_usec * NSEC_PER_USEC)); } -#endif /* !((BITS_PER_LONG == 64) || defined(CONFIG_KTIME_SCALAR)) */ +#define ktime_to_timespec(kt) ns_to_timespec((kt).tv64) +#define ktime_to_timeval(kt) ns_to_timeval((kt).tv64) +#define ktime_to_ns(kt) ((kt).tv64) -static inline s64 ktime_get_ns(void) +static inline s64 +ktime_get_ns(void) { struct timespec ts; ktime_t kt; + ktime_get_ts(&ts); kt = timespec_to_ktime(ts); return (ktime_to_ns(kt)); } +static inline ktime_t +ktime_get(void) +{ + struct timespec ts; + + ktime_get_ts(&ts); + return (timespec_to_ktime(ts)); +} + #endif /* _LINUX_KTIME_H */ Index: sys/ofed/include/linux/linux_compat.c =================================================================== --- sys/ofed/include/linux/linux_compat.c +++ sys/ofed/include/linux/linux_compat.c @@ -606,81 +606,30 @@ .fo_sendfile = invfo_sendfile, }; -/* - * Hash of vmmap addresses. This is infrequently accessed and does not - * need to be particularly large. This is done because we must store the - * caller's idea of the map size to properly unmap. - */ -struct vmmap { - LIST_ENTRY(vmmap) vm_next; - void *vm_addr; - unsigned long vm_size; -}; - -struct vmmaphd { - struct vmmap *lh_first; -}; -#define VMMAP_HASH_SIZE 64 -#define VMMAP_HASH_MASK (VMMAP_HASH_SIZE - 1) -#define VM_HASH(addr) ((uintptr_t)(addr) >> PAGE_SHIFT) & VMMAP_HASH_MASK -static struct vmmaphd vmmaphead[VMMAP_HASH_SIZE]; -static struct mtx vmmaplock; - -static void -vmmap_add(void *addr, unsigned long size) +void * +ioremap(resource_size_t offset, unsigned long size) { - struct vmmap *vmmap; - - vmmap = kmalloc(sizeof(*vmmap), GFP_KERNEL); - mtx_lock(&vmmaplock); - vmmap->vm_size = size; - vmmap->vm_addr = addr; - LIST_INSERT_HEAD(&vmmaphead[VM_HASH(addr)], vmmap, vm_next); - mtx_unlock(&vmmaplock); + return (pmap_mapdev_attr(offset, size, VM_MEMATTR_UNCACHEABLE)); } -static struct vmmap * -vmmap_remove(void *addr) +void * +ioremap_nocache(resource_size_t offset, unsigned long size) { - struct vmmap *vmmap; - - mtx_lock(&vmmaplock); - LIST_FOREACH(vmmap, &vmmaphead[VM_HASH(addr)], vm_next) - if (vmmap->vm_addr == addr) - break; - if (vmmap) - LIST_REMOVE(vmmap, vm_next); - mtx_unlock(&vmmaplock); - - return (vmmap); + return (pmap_mapdev_attr(offset, size, VM_MEMATTR_UNCACHEABLE)); } void * -_ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr) +ioremap_wc(resource_size_t offset, unsigned long size) { - void *addr; - - addr = pmap_mapdev_attr(phys_addr, size, attr); - if (addr == NULL) - return (NULL); - vmmap_add(addr, size); - - return (addr); + return (pmap_mapdev_attr(offset, size, VM_MEMATTR_WRITE_COMBINING)); } void -iounmap(void *addr) +iounmap(void *addr, unsigned long size) { - struct vmmap *vmmap; - - vmmap = vmmap_remove(addr); - if (vmmap == NULL) - return; - pmap_unmapdev((vm_offset_t)addr, vmmap->vm_size); - kfree(vmmap); + pmap_unmapdev((vm_offset_t)addr, size); } - void * vmap(struct page **pages, unsigned int count, unsigned long flags, int prot) { @@ -691,23 +640,19 @@ off = kva_alloc(size); if (off == 0) return (NULL); - vmmap_add((void *)off, size); pmap_qenter(off, pages, count); return ((void *)off); } void -vunmap(void *addr) +vunmap(void *addr, unsigned int count) { - struct vmmap *vmmap; + size_t size; - vmmap = vmmap_remove(addr); - if (vmmap == NULL) - return; - pmap_qremove((vm_offset_t)addr, vmmap->vm_size / PAGE_SIZE); - kva_free((vm_offset_t)addr, vmmap->vm_size); - kfree(vmmap); + size = count * PAGE_SIZE; + pmap_qremove((vm_offset_t)addr, count); + kva_free((vm_offset_t)addr, size); } char * @@ -910,7 +855,6 @@ linux_compat_init(void *arg) { struct sysctl_oid *rootoid; - int i; rootoid = SYSCTL_ADD_ROOT_NODE(NULL, OID_AUTO, "sys", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "sys"); @@ -929,9 +873,6 @@ INIT_LIST_HEAD(&pci_drivers); INIT_LIST_HEAD(&pci_devices); spin_lock_init(&pci_lock); - mtx_init(&vmmaplock, "IO Map lock", NULL, MTX_DEF); - for (i = 0; i < VMMAP_HASH_SIZE; i++) - LIST_INIT(&vmmaphead[i]); } SYSINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_init, NULL); Index: sys/ofed/include/linux/list.h =================================================================== --- sys/ofed/include/linux/list.h +++ sys/ofed/include/linux/list.h @@ -95,6 +95,15 @@ entry->prev->next = entry->next; } +static inline void list_replace(struct list_head *old, + struct list_head *new) +{ + new->next = old->next; + new->next->prev = new; + new->prev = old->prev; + new->prev->next = new; +} + static inline void _list_add(struct list_head *new, struct list_head *prev, struct list_head *next) @@ -119,6 +128,9 @@ #define list_first_entry(ptr, type, member) \ list_entry((ptr)->next, type, member) +#define list_next_entry(ptr, member) \ + list_entry(((ptr)->member.next), typeof(*(ptr)), member) + #define list_for_each(p, head) \ for (p = (head)->next; p != (head); p = p->next) @@ -134,6 +146,15 @@ n = list_entry(p->field.next, typeof(*p), field); &p->field != (h);\ p = n, n = list_entry(n->field.next, typeof(*n), field)) +#define list_for_each_entry_continue(p, h, field) \ + for (p = list_next_entry((p), field); &p->field != (h); \ + p = list_next_entry((p), field)) + +#define list_for_each_entry_safe_from(pos, n, head, member) \ + for (n = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.next, typeof(*n), member)) + #define list_for_each_entry_reverse(p, h, field) \ for (p = list_entry((h)->prev, typeof(*p), field); &p->field != (h); \ p = list_entry(p->field.prev, typeof(*p), field)) @@ -381,10 +402,14 @@ #define hlist_for_each_safe(p, n, head) \ for (p = (head)->first; p && ({ n = p->next; 1; }); p = n) -#define hlist_for_each_entry(tp, p, head, field) \ - for (p = (head)->first; \ - p ? (tp = hlist_entry(p, typeof(*tp), field)): NULL; p = p->next) - +#define hlist_entry_safe(ptr, type, member) \ + ((ptr) ? hlist_entry(ptr, type, member) : NULL) + +#define hlist_for_each_entry(pos, head, member) \ + for (pos = hlist_entry_safe((head)->first, typeof(*(pos)), member);\ + pos; \ + pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member)) + #define hlist_for_each_entry_continue(tp, p, field) \ for (p = (p)->next; \ p ? (tp = hlist_entry(p, typeof(*tp), field)): NULL; p = p->next) @@ -398,4 +423,8 @@ tpos = hlist_entry((pos), typeof(*(tpos)), member); 1;}); \ pos = (n)) +#define hlist_add_head_rcu(n, h) hlist_add_head(n, h) + +#define hlist_del_init_rcu(n) hlist_del_init(n) + #endif /* _LINUX_LIST_H_ */ Index: sys/ofed/include/linux/math64.h =================================================================== --- sys/ofed/include/linux/math64.h +++ sys/ofed/include/linux/math64.h @@ -32,7 +32,7 @@ #include #if BITS_PER_LONG == 64 - + # define do_div(n, base) ({ \ uint32_t __base = (base); \ uint32_t __rem; \ @@ -53,6 +53,15 @@ return dividend / divisor; } +static inline u64 div64_u64(u64 dividend, u64 divisor) +{ + return dividend / divisor; +} + +static inline s64 div64_s64(s64 dividend, s64 divisor) +{ + return dividend / divisor; +} #elif BITS_PER_LONG == 32 @@ -88,7 +97,7 @@ *n = res; return rem; } - + # define do_div(n, base) ({ \ uint32_t __base = (base); \ uint32_t __rem; \ @@ -108,7 +117,6 @@ return dividend; } #endif - #endif /* BITS_PER_LONG */ Index: sys/ofed/include/linux/mm.h =================================================================== --- sys/ofed/include/linux/mm.h +++ sys/ofed/include/linux/mm.h @@ -2,7 +2,9 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. + * Copyright (c) 2015 François Tigeot + * Copyright (c) 2015 Matthew Dillon * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -84,4 +86,28 @@ return (0); } +static inline unsigned long +vma_pages(struct vm_area_struct *vma) +{ + unsigned long size; + + size = vma->vm_end - vma->vm_start; + + return size >> PAGE_SHIFT; +} + +#define offset_in_page(off) ((off) & PAGE_MASK) + +static inline void +set_page_dirty(struct vm_page *page) +{ + vm_page_dirty(page); +} + +static inline void +get_page(struct vm_page *page) +{ + vm_page_hold(page); +} + #endif /* _LINUX_MM_H_ */ Index: sys/ofed/include/linux/moduleparam.h =================================================================== --- sys/ofed/include/linux/moduleparam.h +++ sys/ofed/include/linux/moduleparam.h @@ -48,7 +48,7 @@ u16 flags; param_set_fn set; param_get_fn get; - union { + union { void *arg; struct kparam_string *str; struct kparam_array *arr; @@ -83,8 +83,8 @@ SYSINIT(name##_param_sysinit, SI_SUB_DRIVERS, SI_ORDER_FIRST, \ param_sysinit, &__param_##name); -#define module_param_string(name, string, len, perm) - +#define module_param_string(name, string, len, perm) + #define module_param_named(name, var, type, mode) \ module_param_call(name, param_set_##type, param_get_##type, &var, mode) @@ -118,7 +118,7 @@ return 0; } -static inline int +static inline int param_get_short(char *buffer, struct kernel_param *kp) { @@ -126,14 +126,14 @@ } -static inline int +static inline int param_set_ushort(const char *val, struct kernel_param *kp) { return 0; } -static inline int +static inline int param_get_ushort(char *buffer, struct kernel_param *kp) { @@ -141,14 +141,14 @@ } -static inline int +static inline int param_set_int(const char *val, struct kernel_param *kp) { return 0; } -static inline int +static inline int param_get_int(char *buffer, struct kernel_param *kp) { @@ -156,14 +156,14 @@ } -static inline int +static inline int param_set_uint(const char *val, struct kernel_param *kp) { return 0; } -static inline int +static inline int param_get_uint(char *buffer, struct kernel_param *kp) { @@ -171,14 +171,14 @@ } -static inline int +static inline int param_set_long(const char *val, struct kernel_param *kp) { return 0; } -static inline int +static inline int param_get_long(char *buffer, struct kernel_param *kp) { @@ -186,14 +186,14 @@ } -static inline int +static inline int param_set_ulong(const char *val, struct kernel_param *kp) { return 0; } -static inline int +static inline int param_get_ulong(char *buffer, struct kernel_param *kp) { @@ -201,14 +201,14 @@ } -static inline int +static inline int param_set_charp(const char *val, struct kernel_param *kp) { return 0; } -static inline int +static inline int param_get_charp(char *buffer, struct kernel_param *kp) { @@ -216,14 +216,14 @@ } -static inline int +static inline int param_set_bool(const char *val, struct kernel_param *kp) { return 0; } -static inline int +static inline int param_get_bool(char *buffer, struct kernel_param *kp) { Index: sys/ofed/include/linux/scatterlist.h =================================================================== --- sys/ofed/include/linux/scatterlist.h +++ sys/ofed/include/linux/scatterlist.h @@ -3,6 +3,7 @@ * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2015 Matthew Dillon * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -64,6 +65,12 @@ unsigned int orig_nents; /* original size of list */ }; +struct sg_page_iter { + struct scatterlist *sg; + unsigned int sg_pgoffset; /* page index */ + unsigned int maxents; +}; + /* * Maximum number of entries that will be allocated in one piece, if * a list larger than this is required then chaining will be utilized. @@ -326,6 +333,66 @@ return ret; } +/* + * Iterate pages in sg list. + */ +static inline void +_sg_iter_next(struct sg_page_iter *iter) +{ + struct scatterlist *sg; + unsigned int pgcount; + + sg = iter->sg; + pgcount = (sg->offset + sg->length + PAGE_MASK) >> PAGE_SHIFT; + + ++iter->sg_pgoffset; + while (iter->sg_pgoffset >= pgcount) { + iter->sg_pgoffset -= pgcount; + sg = sg_next(sg); + --iter->maxents; + if (sg == NULL || iter->maxents == 0) + break; + pgcount = (sg->offset + sg->length + PAGE_MASK) >> PAGE_SHIFT; + } + iter->sg = sg; +} + +/* + * NOTE: pgoffset is really a page index, not a byte offset. + */ +static inline void +_sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, + unsigned int nents, unsigned long pgoffset) +{ + if (nents) { + /* + * Nominal case. Note subtract 1 from starting page index + * for initial _sg_iter_next() call. + */ + iter->sg = sgl; + iter->sg_pgoffset = pgoffset - 1; + iter->maxents = nents; + _sg_iter_next(iter); + } else { + /* + * Degenerate case + */ + iter->sg = NULL; + iter->sg_pgoffset = 0; + iter->maxents = 0; + } +} + +static inline dma_addr_t +sg_page_iter_dma_address(struct sg_page_iter *spi) +{ + return spi->sg->address + (spi->sg_pgoffset << PAGE_SHIFT); +} + +#define for_each_sg_page(sgl, iter, nents, pgoffset) \ + for (_sg_iter_init(sgl, iter, nents, pgoffset); \ + (iter)->sg; _sg_iter_next(iter)) + #define for_each_sg(sglist, sg, sgmax, _itr) \ for (_itr = 0, sg = (sglist); _itr < (sgmax); _itr++, sg = sg_next(sg)) Index: sys/ofed/include/linux/sched.h =================================================================== --- sys/ofed/include/linux/sched.h +++ sys/ofed/include/linux/sched.h @@ -107,4 +107,15 @@ #define sched_yield() sched_relinquish(curthread) +static inline long +schedule_timeout(signed long timeout) +{ + if (timeout < 0) + return 0; + + pause("lstim", timeout); + + return 0; +} + #endif /* _LINUX_SCHED_H_ */ Index: sys/ofed/include/linux/time.h =================================================================== --- sys/ofed/include/linux/time.h +++ sys/ofed/include/linux/time.h @@ -0,0 +1,131 @@ +/* $FreeBSD$ */ +/* + * Copyright (c) 2014-2015 François Tigeot + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LINUX_TIME_H_ +#define _LINUX_TIME_H_ + +#define NSEC_PER_USEC 1000L +#define NSEC_PER_SEC 1000000000L + +#include +#include + +static inline struct timeval +ns_to_timeval(const int64_t nsec) +{ + struct timeval tv; + long rem; + + if (nsec == 0) { + tv.tv_sec = 0; + tv.tv_usec = 0; + return (tv); + } + + tv.tv_sec = nsec / NSEC_PER_SEC; + rem = nsec % NSEC_PER_SEC; + if (rem < 0) { + tv.tv_sec--; + rem += NSEC_PER_SEC; + } + tv.tv_usec = rem / 1000; + return (tv); +} + +static inline int64_t +timeval_to_ns(const struct timeval *tv) +{ + return ((int64_t)tv->tv_sec * NSEC_PER_SEC) + + tv->tv_usec * NSEC_PER_USEC; +} + +#define getrawmonotonic(ts) nanouptime(ts) + +static inline struct timespec +timespec_sub(struct timespec lhs, struct timespec rhs) +{ + struct timespec ts; + + ts.tv_sec = lhs.tv_sec; + ts.tv_nsec = lhs.tv_nsec; + timespecsub(&ts, &rhs); + + return ts; +} + +static inline void +set_normalized_timespec(struct timespec *ts, time_t sec, int64_t nsec) +{ + /* XXX: this doesn't actually normalize anything */ + ts->tv_sec = sec; + ts->tv_nsec = nsec; +} + +static inline int64_t +timespec_to_ns(const struct timespec *ts) +{ + return ((ts->tv_sec * NSEC_PER_SEC) + ts->tv_nsec); +} + +static inline struct timespec +ns_to_timespec(const int64_t nsec) +{ + struct timespec ts; + int32_t rem; + + if (nsec == 0) { + ts.tv_sec = 0; + ts.tv_nsec = 0; + return (ts); + } + + ts.tv_sec = nsec / NSEC_PER_SEC; + rem = nsec % NSEC_PER_SEC; + if (rem < 0) { + ts.tv_sec--; + rem += NSEC_PER_SEC; + } + ts.tv_nsec = rem; + return (ts); +} + +static inline int +timespec_valid(const struct timespec *ts) +{ + if (ts->tv_sec < 0 || ts->tv_sec > 100000000 || + ts->tv_nsec < 0 || ts->tv_nsec >= 1000000000) + return (0); + return (1); +} + +static inline unsigned long +get_seconds(void) +{ + return time_uptime; +} + +#endif /* _LINUX_TIME_H_ */ Index: sys/ofed/include/linux/types.h =================================================================== --- sys/ofed/include/linux/types.h +++ sys/ofed/include/linux/types.h @@ -36,8 +36,6 @@ #include #include -#define __read_mostly __attribute__((__section__(".data.read_mostly"))) - #ifndef __bitwise__ #ifdef __CHECKER__ #define __bitwise__ __attribute__((bitwise)) Index: sys/ofed/include/linux/uaccess.h =================================================================== --- sys/ofed/include/linux/uaccess.h +++ sys/ofed/include/linux/uaccess.h @@ -2,7 +2,8 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. + * Copyright (c) 2015 François Tigeot * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -32,4 +33,14 @@ #define get_user(_x, _p) -copyin((_p), &(_x), sizeof(*(_p))) #define put_user(_x, _p) -copyout(&(_x), (_p), sizeof(*(_p))) +static inline void pagefault_disable(void) +{ + curthread_pflags_set(TDP_NOFAULTING | TDP_RESETSPUR); +} + +static inline void pagefault_enable(void) +{ + curthread_pflags_restore(~(TDP_NOFAULTING | TDP_RESETSPUR)); +} + #endif /* _LINUX_UACCESS_H_ */ Index: sys/ofed/include/linux/vmalloc.h =================================================================== --- sys/ofed/include/linux/vmalloc.h +++ sys/ofed/include/linux/vmalloc.h @@ -37,6 +37,6 @@ void *vmap(struct page **pages, unsigned int count, unsigned long flags, int prot); -void vunmap(void *addr); +void vunmap(void *addr, unsigned int count); #endif /* _LINUX_VMALLOC_H_ */ Index: sys/ofed/include/linux/wait.h =================================================================== --- sys/ofed/include/linux/wait.h +++ sys/ofed/include/linux/wait.h @@ -39,15 +39,18 @@ #include #include -struct __wait_queue_head { +typedef struct { +} wait_queue_t; + +typedef struct { unsigned int wchan; -}; -typedef struct __wait_queue_head wait_queue_head_t; +} wait_queue_head_t; -#define init_waitqueue_head(x) +#define init_waitqueue_head(x) \ + do { } while (0) static inline void -__wake_up(struct __wait_queue_head *q, int all) +__wake_up(wait_queue_head_t *q, int all) { int wakeup_swapper; void *c; @@ -108,6 +111,23 @@ -_error; \ }) -#define DEFINE_WAIT(x) +static inline int +waitqueue_active(wait_queue_head_t *q) +{ + return 0; /* XXX: not really implemented */ +} + +#define DEFINE_WAIT(name) \ + wait_queue_t name = {} + +static inline void +prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state) +{ +} + +static inline void +finish_wait(wait_queue_head_t *q, wait_queue_t *wait) +{ +} #endif /* _LINUX_WAIT_H_ */ Index: sys/ofed/include/linux/workqueue.h =================================================================== --- sys/ofed/include/linux/workqueue.h +++ sys/ofed/include/linux/workqueue.h @@ -91,11 +91,11 @@ #define flush_scheduled_work() flush_taskqueue(taskqueue_thread) -static inline int queue_work (struct workqueue_struct *q, struct work_struct *work) +static inline int queue_work(struct workqueue_struct *q, struct work_struct *work) { (work)->taskqueue = (q)->taskqueue; /* Return opposite val to align with Linux logic */ - return !taskqueue_enqueue((q)->taskqueue, &(work)->work_task); + return !taskqueue_enqueue((q)->taskqueue, &(work)->work_task); } static inline void @@ -151,6 +151,12 @@ #define create_workqueue(name) \ _create_workqueue_common(name, MAXCPU) +#define alloc_ordered_workqueue(name, flags) \ + _create_workqueue_common(name, 1) + +#define alloc_workqueue(name, flags, max_active) \ + _create_workqueue_common(name, max_active) + static inline void destroy_workqueue(struct workqueue_struct *wq) {