diff --git a/sys/compat/linuxkpi/common/include/linux/dma-mapping.h b/sys/compat/linuxkpi/common/include/linux/dma-mapping.h index 01395cc7a0d5..e910545e04fc 100644 --- a/sys/compat/linuxkpi/common/include/linux/dma-mapping.h +++ b/sys/compat/linuxkpi/common/include/linux/dma-mapping.h @@ -1,383 +1,383 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _LINUXKPI_LINUX_DMA_MAPPING_H_ #define _LINUXKPI_LINUX_DMA_MAPPING_H_ #include #include #include #include #include #include #include #include #include #include #include #include #include #include enum dma_data_direction { DMA_BIDIRECTIONAL = 0, DMA_TO_DEVICE = 1, DMA_FROM_DEVICE = 2, DMA_NONE = 3, }; struct dma_map_ops { void* (*alloc_coherent)(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp); void (*free_coherent)(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle); dma_addr_t (*map_page)(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs); void (*unmap_page)(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir, unsigned long attrs); int (*map_sg)(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, unsigned long attrs); void (*unmap_sg)(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, unsigned long attrs); void (*sync_single_for_cpu)(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir); void (*sync_single_for_device)(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir); void (*sync_single_range_for_cpu)(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, enum dma_data_direction dir); void (*sync_single_range_for_device)(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, enum dma_data_direction dir); void (*sync_sg_for_cpu)(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir); void (*sync_sg_for_device)(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir); int (*mapping_error)(struct device *dev, dma_addr_t dma_addr); int (*dma_supported)(struct device *dev, u64 mask); int is_phys; }; #define DMA_BIT_MASK(n) ((2ULL << ((n) - 1)) - 1ULL) int linux_dma_tag_init(struct device *, u64); int linux_dma_tag_init_coherent(struct device *, u64); void *linux_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag); void *linuxkpi_dmam_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag); dma_addr_t linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len); void linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t size); int linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction dir __unused, unsigned long attrs __unused); void linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, int nents __unused, enum dma_data_direction dir __unused, unsigned long attrs __unused); void linuxkpi_dma_sync(struct device *, dma_addr_t, size_t, bus_dmasync_op_t); static inline int dma_supported(struct device *dev, u64 dma_mask) { /* XXX busdma takes care of this elsewhere. */ return (1); } static inline int dma_set_mask(struct device *dev, u64 dma_mask) { if (!dev->dma_priv || !dma_supported(dev, dma_mask)) return -EIO; return (linux_dma_tag_init(dev, dma_mask)); } static inline int dma_set_coherent_mask(struct device *dev, u64 dma_mask) { if (!dev->dma_priv || !dma_supported(dev, dma_mask)) return -EIO; return (linux_dma_tag_init_coherent(dev, dma_mask)); } static inline int dma_set_mask_and_coherent(struct device *dev, u64 dma_mask) { int r; r = dma_set_mask(dev, dma_mask); if (r == 0) dma_set_coherent_mask(dev, dma_mask); return (r); } static inline void * dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag) { return (linux_dma_alloc_coherent(dev, size, dma_handle, flag)); } static inline void * dma_zalloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag) { return (dma_alloc_coherent(dev, size, dma_handle, flag | __GFP_ZERO)); } static inline void * dmam_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag) { return (linuxkpi_dmam_alloc_coherent(dev, size, dma_handle, flag)); } static inline void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr) { linux_dma_unmap(dev, dma_addr, size); kmem_free(cpu_addr, size); } static inline dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, size_t offset, size_t size, enum dma_data_direction dir, unsigned long attrs) { - return (linux_dma_map_phys(dev, VM_PAGE_TO_PHYS(page) + offset, size)); + return (linux_dma_map_phys(dev, page_to_phys(page) + offset, size)); } /* linux_dma_(un)map_sg_attrs does not support attrs yet */ #define dma_map_sg_attrs(dev, sgl, nents, dir, attrs) \ linux_dma_map_sg_attrs(dev, sgl, nents, dir, 0) #define dma_unmap_sg_attrs(dev, sg, nents, dir, attrs) \ linux_dma_unmap_sg_attrs(dev, sg, nents, dir, 0) static inline dma_addr_t dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction) { - return (linux_dma_map_phys(dev, VM_PAGE_TO_PHYS(page) + offset, size)); + return (linux_dma_map_phys(dev, page_to_phys(page) + offset, size)); } static inline void dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, enum dma_data_direction direction) { linux_dma_unmap(dev, dma_address, size); } static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma, size_t size, enum dma_data_direction direction) { bus_dmasync_op_t op; switch (direction) { case DMA_BIDIRECTIONAL: op = BUS_DMASYNC_POSTREAD; linuxkpi_dma_sync(dev, dma, size, op); op = BUS_DMASYNC_PREREAD; break; case DMA_TO_DEVICE: op = BUS_DMASYNC_POSTWRITE; break; case DMA_FROM_DEVICE: op = BUS_DMASYNC_POSTREAD; break; default: return; } linuxkpi_dma_sync(dev, dma, size, op); } static inline void dma_sync_single(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { dma_sync_single_for_cpu(dev, addr, size, dir); } static inline void dma_sync_single_for_device(struct device *dev, dma_addr_t dma, size_t size, enum dma_data_direction direction) { bus_dmasync_op_t op; switch (direction) { case DMA_BIDIRECTIONAL: op = BUS_DMASYNC_PREWRITE; break; case DMA_TO_DEVICE: op = BUS_DMASYNC_PREREAD; break; case DMA_FROM_DEVICE: op = BUS_DMASYNC_PREWRITE; break; default: return; } linuxkpi_dma_sync(dev, dma, size, op); } static inline void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction direction) { } static inline void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction direction) { } static inline void dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, int direction) { } static inline void dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, int direction) { } #define DMA_MAPPING_ERROR (~(dma_addr_t)0) static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) { if (dma_addr == 0 || dma_addr == DMA_MAPPING_ERROR) return (-ENOMEM); return (0); } static inline unsigned int dma_set_max_seg_size(struct device *dev, unsigned int size) { return (0); } static inline dma_addr_t _dma_map_single_attrs(struct device *dev, void *ptr, size_t size, enum dma_data_direction direction, unsigned long attrs __unused) { dma_addr_t dma; dma = linux_dma_map_phys(dev, vtophys(ptr), size); if (!dma_mapping_error(dev, dma)) dma_sync_single_for_device(dev, dma, size, direction); return (dma); } static inline void _dma_unmap_single_attrs(struct device *dev, dma_addr_t dma, size_t size, enum dma_data_direction direction, unsigned long attrs __unused) { dma_sync_single_for_cpu(dev, dma, size, direction); linux_dma_unmap(dev, dma, size); } static inline size_t dma_max_mapping_size(struct device *dev) { return (SCATTERLIST_MAX_SEGMENT); } #define dma_map_single_attrs(dev, ptr, size, dir, attrs) \ _dma_map_single_attrs(dev, ptr, size, dir, 0) #define dma_unmap_single_attrs(dev, dma_addr, size, dir, attrs) \ _dma_unmap_single_attrs(dev, dma_addr, size, dir, 0) #define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, 0) #define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, 0) #define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, 0) #define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, 0) #define DEFINE_DMA_UNMAP_ADDR(name) dma_addr_t name #define DEFINE_DMA_UNMAP_LEN(name) __u32 name #define dma_unmap_addr(p, name) ((p)->name) #define dma_unmap_addr_set(p, name, v) (((p)->name) = (v)) #define dma_unmap_len(p, name) ((p)->name) #define dma_unmap_len_set(p, name, v) (((p)->name) = (v)) extern int uma_align_cache; #define dma_get_cache_alignment() uma_align_cache static inline int dma_map_sgtable(struct device *dev, struct sg_table *sgt, enum dma_data_direction dir, unsigned long attrs) { int nents; nents = dma_map_sg_attrs(dev, sgt->sgl, sgt->nents, dir, attrs); if (nents < 0) return (nents); sgt->nents = nents; return (0); } static inline void dma_unmap_sgtable(struct device *dev, struct sg_table *sgt, enum dma_data_direction dir, unsigned long attrs) { dma_unmap_sg_attrs(dev, sgt->sgl, sgt->nents, dir, attrs); } #endif /* _LINUXKPI_LINUX_DMA_MAPPING_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/highmem.h b/sys/compat/linuxkpi/common/include/linux/highmem.h index a3f9af82400e..f770bef6b3b7 100644 --- a/sys/compat/linuxkpi/common/include/linux/highmem.h +++ b/sys/compat/linuxkpi/common/include/linux/highmem.h @@ -1,135 +1,135 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io) * Copyright (c) 2017 Mellanox Technologies, Ltd. * Copyright (c) 2021 Vladimir Kondratyev * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _LINUXKPI_LINUX_HIGHMEM_H_ #define _LINUXKPI_LINUX_HIGHMEM_H_ #include #include #include #include #include #include #include #include #include #include #define PageHighMem(p) (0) static inline struct page * kmap_to_page(void *addr) { return (virt_to_page(addr)); } static inline void * kmap(struct page *page) { struct sf_buf *sf; if (PMAP_HAS_DMAP) { - return ((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page))); + return ((void *)PHYS_TO_DMAP(page_to_phys(page))); } else { sched_pin(); sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); if (sf == NULL) { sched_unpin(); return (NULL); } return ((void *)sf_buf_kva(sf)); } } static inline void * kmap_atomic_prot(struct page *page, pgprot_t prot) { vm_memattr_t attr = pgprot2cachemode(prot); if (attr != VM_MEMATTR_DEFAULT) { vm_page_lock(page); page->flags |= PG_FICTITIOUS; vm_page_unlock(page); pmap_page_set_memattr(page, attr); } return (kmap(page)); } static inline void * kmap_atomic(struct page *page) { return (kmap_atomic_prot(page, VM_PROT_ALL)); } static inline void * kmap_local_page_prot(struct page *page, pgprot_t prot) { return (kmap_atomic_prot(page, prot)); } static inline void kunmap(struct page *page) { struct sf_buf *sf; if (!PMAP_HAS_DMAP) { /* lookup SF buffer in list */ sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); /* double-free */ sf_buf_free(sf); sf_buf_free(sf); sched_unpin(); } } static inline void kunmap_atomic(void *vaddr) { if (!PMAP_HAS_DMAP) kunmap(virt_to_page(vaddr)); } static inline void kunmap_local(void *addr) { kunmap_atomic(addr); } #endif /* _LINUXKPI_LINUX_HIGHMEM_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/scatterlist.h b/sys/compat/linuxkpi/common/include/linux/scatterlist.h index b448262f3497..703e0a59e03c 100644 --- a/sys/compat/linuxkpi/common/include/linux/scatterlist.h +++ b/sys/compat/linuxkpi/common/include/linux/scatterlist.h @@ -1,670 +1,670 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. * Copyright (c) 2015 Matthew Dillon * Copyright (c) 2016 Matthew Macy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _LINUXKPI_LINUX_SCATTERLIST_H_ #define _LINUXKPI_LINUX_SCATTERLIST_H_ #include #include #include #include #include struct bus_dmamap; struct scatterlist { unsigned long page_link; #define SG_PAGE_LINK_CHAIN 0x1UL #define SG_PAGE_LINK_LAST 0x2UL #define SG_PAGE_LINK_MASK 0x3UL unsigned int offset; unsigned int length; dma_addr_t dma_address; struct bus_dmamap *dma_map; /* FreeBSD specific */ }; CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0); struct sg_table { struct scatterlist *sgl; unsigned int nents; unsigned int orig_nents; }; struct sg_page_iter { struct scatterlist *sg; unsigned int sg_pgoffset; unsigned int maxents; struct { unsigned int nents; int pg_advance; } internal; }; struct sg_dma_page_iter { struct sg_page_iter base; }; #define SCATTERLIST_MAX_SEGMENT (-1U & ~(PAGE_SIZE - 1)) #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) #define SG_MAGIC 0x87654321UL #define SG_CHAIN SG_PAGE_LINK_CHAIN #define SG_END SG_PAGE_LINK_LAST #define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) #define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) #define sg_chain_ptr(sg) \ ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK)) #define sg_dma_address(sg) (sg)->dma_address #define sg_dma_len(sg) (sg)->length #define for_each_sg_page(sgl, iter, nents, pgoffset) \ for (_sg_iter_init(sgl, iter, nents, pgoffset); \ (iter)->sg; _sg_iter_next(iter)) #define for_each_sg_dma_page(sgl, iter, nents, pgoffset) \ for_each_sg_page(sgl, &(iter)->base, nents, pgoffset) #define for_each_sg(sglist, sg, sgmax, iter) \ for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg)) #define for_each_sgtable_sg(sgt, sg, i) \ for_each_sg((sgt)->sgl, sg, (sgt)->orig_nents, i) #define for_each_sgtable_page(sgt, iter, pgoffset) \ for_each_sg_page((sgt)->sgl, iter, (sgt)->orig_nents, pgoffset) #define for_each_sgtable_dma_sg(sgt, sg, iter) \ for_each_sg((sgt)->sgl, sg, (sgt)->nents, iter) #define for_each_sgtable_dma_page(sgt, iter, pgoffset) \ for_each_sg_dma_page((sgt)->sgl, iter, (sgt)->nents, pgoffset) typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t); typedef void (sg_free_fn) (struct scatterlist *, unsigned int); static inline void sg_assign_page(struct scatterlist *sg, struct page *page) { unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK; sg->page_link = page_link | (unsigned long)page; } static inline void sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, unsigned int offset) { sg_assign_page(sg, page); sg->offset = offset; sg->length = len; } static inline struct page * sg_page(struct scatterlist *sg) { return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK)); } static inline void sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) { sg_set_page(sg, virt_to_page(buf), buflen, ((uintptr_t)buf) & (PAGE_SIZE - 1)); } static inline struct scatterlist * sg_next(struct scatterlist *sg) { if (sg_is_last(sg)) return (NULL); sg++; if (sg_is_chain(sg)) sg = sg_chain_ptr(sg); return (sg); } static inline vm_paddr_t sg_phys(struct scatterlist *sg) { - return (VM_PAGE_TO_PHYS(sg_page(sg)) + sg->offset); + return (page_to_phys(sg_page(sg)) + sg->offset); } static inline void * sg_virt(struct scatterlist *sg) { return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset)); } static inline void sg_chain(struct scatterlist *prv, unsigned int prv_nents, struct scatterlist *sgl) { struct scatterlist *sg = &prv[prv_nents - 1]; sg->offset = 0; sg->length = 0; sg->page_link = ((unsigned long)sgl | SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST; } static inline void sg_mark_end(struct scatterlist *sg) { sg->page_link |= SG_PAGE_LINK_LAST; sg->page_link &= ~SG_PAGE_LINK_CHAIN; } static inline void sg_init_table(struct scatterlist *sg, unsigned int nents) { bzero(sg, sizeof(*sg) * nents); sg_mark_end(&sg[nents - 1]); } static inline void sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen) { sg_init_table(sg, 1); sg_set_buf(sg, buf, buflen); } static struct scatterlist * sg_kmalloc(unsigned int nents, gfp_t gfp_mask) { if (nents == SG_MAX_SINGLE_ALLOC) { return ((void *)__get_free_page(gfp_mask)); } else return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask)); } static inline void sg_kfree(struct scatterlist *sg, unsigned int nents) { if (nents == SG_MAX_SINGLE_ALLOC) { free_page((unsigned long)sg); } else kfree(sg); } static inline void __sg_free_table(struct sg_table *table, unsigned int max_ents, bool skip_first_chunk, sg_free_fn * free_fn) { struct scatterlist *sgl, *next; if (unlikely(!table->sgl)) return; sgl = table->sgl; while (table->orig_nents) { unsigned int alloc_size = table->orig_nents; unsigned int sg_size; if (alloc_size > max_ents) { next = sg_chain_ptr(&sgl[max_ents - 1]); alloc_size = max_ents; sg_size = alloc_size - 1; } else { sg_size = alloc_size; next = NULL; } table->orig_nents -= sg_size; if (skip_first_chunk) skip_first_chunk = 0; else free_fn(sgl, alloc_size); sgl = next; } table->sgl = NULL; } static inline void sg_free_table(struct sg_table *table) { __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); } static inline int __sg_alloc_table(struct sg_table *table, unsigned int nents, unsigned int max_ents, struct scatterlist *first_chunk, gfp_t gfp_mask, sg_alloc_fn *alloc_fn) { struct scatterlist *sg, *prv; unsigned int left; memset(table, 0, sizeof(*table)); if (nents == 0) return (-EINVAL); left = nents; prv = NULL; do { unsigned int sg_size; unsigned int alloc_size = left; if (alloc_size > max_ents) { alloc_size = max_ents; sg_size = alloc_size - 1; } else sg_size = alloc_size; left -= sg_size; if (first_chunk) { sg = first_chunk; first_chunk = NULL; } else { sg = alloc_fn(alloc_size, gfp_mask); } if (unlikely(!sg)) { if (prv) table->nents = ++table->orig_nents; return (-ENOMEM); } sg_init_table(sg, alloc_size); table->nents = table->orig_nents += sg_size; if (prv) sg_chain(prv, max_ents, sg); else table->sgl = sg; if (!left) sg_mark_end(&sg[sg_size - 1]); prv = sg; } while (left); return (0); } static inline int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) { int ret; ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, NULL, gfp_mask, sg_kmalloc); if (unlikely(ret)) __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); return (ret); } #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 static inline struct scatterlist * __sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, unsigned int count, unsigned long off, unsigned long size, unsigned int max_segment, struct scatterlist *prv, unsigned int left_pages, gfp_t gfp_mask) #else static inline int __sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, unsigned int count, unsigned long off, unsigned long size, unsigned int max_segment, gfp_t gfp_mask) #endif { unsigned int i, segs, cur, len; int rc; struct scatterlist *s; #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 if (prv != NULL) { panic( "Support for prv != NULL not implemented in " "__sg_alloc_table_from_pages()"); } #endif if (__predict_false(!max_segment || offset_in_page(max_segment))) #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 return (ERR_PTR(-EINVAL)); #else return (-EINVAL); #endif len = 0; for (segs = i = 1; i < count; ++i) { len += PAGE_SIZE; if (len >= max_segment || page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { ++segs; len = 0; } } if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 return (ERR_PTR(rc)); #else return (rc); #endif cur = 0; for (i = 0, s = sgt->sgl; i < sgt->orig_nents; i++) { unsigned long seg_size; unsigned int j; len = 0; for (j = cur + 1; j < count; ++j) { len += PAGE_SIZE; if (len >= max_segment || page_to_pfn(pages[j]) != page_to_pfn(pages[j - 1]) + 1) break; } seg_size = ((j - cur) << PAGE_SHIFT) - off; sg_set_page(s, pages[cur], MIN(size, seg_size), off); size -= seg_size; off = 0; cur = j; s = sg_next(s); } KASSERT(s != NULL, ("s is NULL after loop in __sg_alloc_table_from_pages()")); #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 if (left_pages == 0) sg_mark_end(s); return (s); #else return (0); #endif } static inline int sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, unsigned int count, unsigned long off, unsigned long size, gfp_t gfp_mask) { #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 return (PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, count, off, size, SCATTERLIST_MAX_SEGMENT, NULL, 0, gfp_mask))); #else return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, SCATTERLIST_MAX_SEGMENT, gfp_mask)); #endif } static inline int sg_alloc_table_from_pages_segment(struct sg_table *sgt, struct page **pages, unsigned int count, unsigned int off, unsigned long size, unsigned int max_segment, gfp_t gfp_mask) { #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300 return (PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, count, off, size, max_segment, NULL, 0, gfp_mask))); #else return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, max_segment, gfp_mask)); #endif } static inline int sg_nents(struct scatterlist *sg) { int nents; for (nents = 0; sg; sg = sg_next(sg)) nents++; return (nents); } static inline void __sg_page_iter_start(struct sg_page_iter *piter, struct scatterlist *sglist, unsigned int nents, unsigned long pgoffset) { piter->internal.pg_advance = 0; piter->internal.nents = nents; piter->sg = sglist; piter->sg_pgoffset = pgoffset; } static inline void _sg_iter_next(struct sg_page_iter *iter) { struct scatterlist *sg; unsigned int pgcount; sg = iter->sg; pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; ++iter->sg_pgoffset; while (iter->sg_pgoffset >= pgcount) { iter->sg_pgoffset -= pgcount; sg = sg_next(sg); --iter->maxents; if (sg == NULL || iter->maxents == 0) break; pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; } iter->sg = sg; } static inline int sg_page_count(struct scatterlist *sg) { return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT); } #define sg_dma_page_count(sg) \ sg_page_count(sg) static inline bool __sg_page_iter_next(struct sg_page_iter *piter) { unsigned int pgcount; if (piter->internal.nents == 0) return (0); if (piter->sg == NULL) return (0); piter->sg_pgoffset += piter->internal.pg_advance; piter->internal.pg_advance = 1; while (1) { pgcount = sg_page_count(piter->sg); if (likely(piter->sg_pgoffset < pgcount)) break; piter->sg_pgoffset -= pgcount; piter->sg = sg_next(piter->sg); if (--piter->internal.nents == 0) return (0); if (piter->sg == NULL) return (0); } return (1); } #define __sg_page_iter_dma_next(itr) \ __sg_page_iter_next(&(itr)->base) static inline void _sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, unsigned int nents, unsigned long pgoffset) { if (nents) { iter->sg = sgl; iter->sg_pgoffset = pgoffset - 1; iter->maxents = nents; _sg_iter_next(iter); } else { iter->sg = NULL; iter->sg_pgoffset = 0; iter->maxents = 0; } } /* * sg_page_iter_dma_address() is implemented as a macro because it * needs to accept two different and identical structure types. This * allows both old and new code to co-exist. The compile time assert * adds some safety, that the structure sizes match. */ #define sg_page_iter_dma_address(spi) ({ \ struct sg_page_iter *__spi = (void *)(spi); \ dma_addr_t __dma_address; \ CTASSERT(sizeof(*(spi)) == sizeof(*__spi)); \ __dma_address = __spi->sg->dma_address + \ (__spi->sg_pgoffset << PAGE_SHIFT); \ __dma_address; \ }) static inline struct page * sg_page_iter_page(struct sg_page_iter *piter) { return (nth_page(sg_page(piter->sg), piter->sg_pgoffset)); } static __inline size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents, const void *buf, size_t buflen, off_t skip) { struct sg_page_iter piter; struct page *page; struct sf_buf *sf; size_t len, copied; char *p, *b; if (buflen == 0) return (0); b = __DECONST(char *, buf); copied = 0; sched_pin(); for_each_sg_page(sgl, &piter, nents, 0) { /* Skip to the start. */ if (piter.sg->length <= skip) { skip -= piter.sg->length; continue; } /* See how much to copy. */ KASSERT(((piter.sg->length - skip) != 0 && (buflen != 0)), ("%s: sg len %u - skip %ju || buflen %zu is 0\n", __func__, piter.sg->length, (uintmax_t)skip, buflen)); len = min(piter.sg->length - skip, buflen); page = sg_page_iter_page(&piter); sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); if (sf == NULL) break; p = (char *)sf_buf_kva(sf) + piter.sg_pgoffset + skip; memcpy(p, b, len); sf_buf_free(sf); /* We copied so nothing more to skip. */ skip = 0; copied += len; /* Either we exactly filled the page, or we are done. */ buflen -= len; if (buflen == 0) break; b += len; } sched_unpin(); return (copied); } static inline size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents, const void *buf, size_t buflen) { return (sg_pcopy_from_buffer(sgl, nents, buf, buflen, 0)); } static inline size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents, void *buf, size_t buflen, off_t offset) { struct sg_page_iter iter; struct scatterlist *sg; struct page *page; struct sf_buf *sf; char *vaddr; size_t total = 0; size_t len; if (!PMAP_HAS_DMAP) sched_pin(); for_each_sg_page(sgl, &iter, nents, 0) { sg = iter.sg; if (offset >= sg->length) { offset -= sg->length; continue; } len = ulmin(buflen, sg->length - offset); if (len == 0) break; page = sg_page_iter_page(&iter); if (!PMAP_HAS_DMAP) { sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); if (sf == NULL) break; vaddr = (char *)sf_buf_kva(sf); } else - vaddr = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page)); + vaddr = (char *)PHYS_TO_DMAP(page_to_phys(page)); memcpy(buf, vaddr + sg->offset + offset, len); if (!PMAP_HAS_DMAP) sf_buf_free(sf); /* start at beginning of next page */ offset = 0; /* advance buffer */ buf = (char *)buf + len; buflen -= len; total += len; } if (!PMAP_HAS_DMAP) sched_unpin(); return (total); } #endif /* _LINUXKPI_LINUX_SCATTERLIST_H_ */ diff --git a/sys/compat/linuxkpi/common/src/linux_page.c b/sys/compat/linuxkpi/common/src/linux_page.c index 0cb724ebf6c4..cc45a77eadbe 100644 --- a/sys/compat/linuxkpi/common/src/linux_page.c +++ b/sys/compat/linuxkpi/common/src/linux_page.c @@ -1,538 +1,538 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io) * Copyright (c) 2017 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef __i386__ DEFINE_IDR(mtrr_idr); static MALLOC_DEFINE(M_LKMTRR, "idr", "Linux MTRR compat"); extern int pat_works; #endif void si_meminfo(struct sysinfo *si) { si->totalram = physmem; si->freeram = vm_free_count(); si->totalhigh = 0; si->freehigh = 0; si->mem_unit = PAGE_SIZE; } void * linux_page_address(struct page *page) { if (page->object != kernel_object) { return (PMAP_HAS_DMAP ? - ((void *)(uintptr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page))) : + ((void *)(uintptr_t)PHYS_TO_DMAP(page_to_phys(page))) : NULL); } return ((void *)(uintptr_t)(VM_MIN_KERNEL_ADDRESS + IDX_TO_OFF(page->pindex))); } struct page * linux_alloc_pages(gfp_t flags, unsigned int order) { struct page *page; if (PMAP_HAS_DMAP) { unsigned long npages = 1UL << order; int req = VM_ALLOC_WIRED; if ((flags & M_ZERO) != 0) req |= VM_ALLOC_ZERO; if (order == 0 && (flags & GFP_DMA32) == 0) { page = vm_page_alloc_noobj(req); if (page == NULL) return (NULL); } else { vm_paddr_t pmax = (flags & GFP_DMA32) ? BUS_SPACE_MAXADDR_32BIT : BUS_SPACE_MAXADDR; retry: page = vm_page_alloc_noobj_contig(req, npages, 0, pmax, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); if (page == NULL) { if (flags & M_WAITOK) { if (!vm_page_reclaim_contig(req, npages, 0, pmax, PAGE_SIZE, 0)) { vm_wait(NULL); } flags &= ~M_WAITOK; goto retry; } return (NULL); } } } else { vm_offset_t vaddr; vaddr = linux_alloc_kmem(flags, order); if (vaddr == 0) return (NULL); page = PHYS_TO_VM_PAGE(vtophys((void *)vaddr)); KASSERT(vaddr == (vm_offset_t)page_address(page), ("Page address mismatch")); } return (page); } void linux_free_pages(struct page *page, unsigned int order) { if (PMAP_HAS_DMAP) { unsigned long npages = 1UL << order; unsigned long x; for (x = 0; x != npages; x++) { vm_page_t pgo = page + x; if (vm_page_unwire_noq(pgo)) vm_page_free(pgo); } } else { vm_offset_t vaddr; vaddr = (vm_offset_t)page_address(page); linux_free_kmem(vaddr, order); } } vm_offset_t linux_alloc_kmem(gfp_t flags, unsigned int order) { size_t size = ((size_t)PAGE_SIZE) << order; void *addr; if ((flags & GFP_DMA32) == 0) { addr = kmem_malloc(size, flags & GFP_NATIVE_MASK); } else { addr = kmem_alloc_contig(size, flags & GFP_NATIVE_MASK, 0, BUS_SPACE_MAXADDR_32BIT, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); } return ((vm_offset_t)addr); } void linux_free_kmem(vm_offset_t addr, unsigned int order) { size_t size = ((size_t)PAGE_SIZE) << order; kmem_free((void *)addr, size); } static int linux_get_user_pages_internal(vm_map_t map, unsigned long start, int nr_pages, int write, struct page **pages) { vm_prot_t prot; size_t len; int count; prot = write ? (VM_PROT_READ | VM_PROT_WRITE) : VM_PROT_READ; len = ptoa((vm_offset_t)nr_pages); count = vm_fault_quick_hold_pages(map, start, len, prot, pages, nr_pages); return (count == -1 ? -EFAULT : nr_pages); } int __get_user_pages_fast(unsigned long start, int nr_pages, int write, struct page **pages) { vm_map_t map; vm_page_t *mp; vm_offset_t va; vm_offset_t end; vm_prot_t prot; int count; if (nr_pages == 0 || in_interrupt()) return (0); MPASS(pages != NULL); map = &curthread->td_proc->p_vmspace->vm_map; end = start + ptoa((vm_offset_t)nr_pages); if (!vm_map_range_valid(map, start, end)) return (-EINVAL); prot = write ? (VM_PROT_READ | VM_PROT_WRITE) : VM_PROT_READ; for (count = 0, mp = pages, va = start; va < end; mp++, va += PAGE_SIZE, count++) { *mp = pmap_extract_and_hold(map->pmap, va, prot); if (*mp == NULL) break; if ((prot & VM_PROT_WRITE) != 0 && (*mp)->dirty != VM_PAGE_BITS_ALL) { /* * Explicitly dirty the physical page. Otherwise, the * caller's changes may go unnoticed because they are * performed through an unmanaged mapping or by a DMA * operation. * * The object lock is not held here. * See vm_page_clear_dirty_mask(). */ vm_page_dirty(*mp); } } return (count); } long get_user_pages_remote(struct task_struct *task, struct mm_struct *mm, unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, struct vm_area_struct **vmas) { vm_map_t map; map = &task->task_thread->td_proc->p_vmspace->vm_map; return (linux_get_user_pages_internal(map, start, nr_pages, !!(gup_flags & FOLL_WRITE), pages)); } long get_user_pages(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, struct vm_area_struct **vmas) { vm_map_t map; map = &curthread->td_proc->p_vmspace->vm_map; return (linux_get_user_pages_internal(map, start, nr_pages, !!(gup_flags & FOLL_WRITE), pages)); } int is_vmalloc_addr(const void *addr) { return (vtoslab((vm_offset_t)addr & ~UMA_SLAB_MASK) != NULL); } vm_fault_t lkpi_vmf_insert_pfn_prot_locked(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t prot) { vm_object_t vm_obj = vma->vm_obj; vm_object_t tmp_obj; vm_page_t page; vm_pindex_t pindex; VM_OBJECT_ASSERT_WLOCKED(vm_obj); pindex = OFF_TO_IDX(addr - vma->vm_start); if (vma->vm_pfn_count == 0) vma->vm_pfn_first = pindex; MPASS(pindex <= OFF_TO_IDX(vma->vm_end)); retry: page = vm_page_grab(vm_obj, pindex, VM_ALLOC_NOCREAT); if (page == NULL) { page = PHYS_TO_VM_PAGE(IDX_TO_OFF(pfn)); if (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) goto retry; if (page->object != NULL) { tmp_obj = page->object; vm_page_xunbusy(page); VM_OBJECT_WUNLOCK(vm_obj); VM_OBJECT_WLOCK(tmp_obj); if (page->object == tmp_obj && vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) { KASSERT(page->object == tmp_obj, ("page has changed identity")); KASSERT((page->oflags & VPO_UNMANAGED) == 0, ("page does not belong to shmem")); vm_pager_page_unswapped(page); if (pmap_page_is_mapped(page)) { vm_page_xunbusy(page); VM_OBJECT_WUNLOCK(tmp_obj); printf("%s: page rename failed: page " "is mapped\n", __func__); VM_OBJECT_WLOCK(vm_obj); return (VM_FAULT_NOPAGE); } vm_page_remove(page); } VM_OBJECT_WUNLOCK(tmp_obj); VM_OBJECT_WLOCK(vm_obj); goto retry; } if (vm_page_insert(page, vm_obj, pindex)) { vm_page_xunbusy(page); return (VM_FAULT_OOM); } vm_page_valid(page); } pmap_page_set_memattr(page, pgprot2cachemode(prot)); vma->vm_pfn_count++; return (VM_FAULT_NOPAGE); } int lkpi_remap_pfn_range(struct vm_area_struct *vma, unsigned long start_addr, unsigned long start_pfn, unsigned long size, pgprot_t prot) { vm_object_t vm_obj; unsigned long addr, pfn; int err = 0; vm_obj = vma->vm_obj; VM_OBJECT_WLOCK(vm_obj); for (addr = start_addr, pfn = start_pfn; addr < start_addr + size; addr += PAGE_SIZE) { vm_fault_t ret; retry: ret = lkpi_vmf_insert_pfn_prot_locked(vma, addr, pfn, prot); if ((ret & VM_FAULT_OOM) != 0) { VM_OBJECT_WUNLOCK(vm_obj); vm_wait(NULL); VM_OBJECT_WLOCK(vm_obj); goto retry; } if ((ret & VM_FAULT_ERROR) != 0) { err = -EFAULT; break; } pfn++; } VM_OBJECT_WUNLOCK(vm_obj); if (unlikely(err)) { zap_vma_ptes(vma, start_addr, (pfn - start_pfn) << PAGE_SHIFT); return (err); } return (0); } int lkpi_io_mapping_map_user(struct io_mapping *iomap, struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size) { pgprot_t prot; int ret; prot = cachemode2protval(iomap->attr); ret = lkpi_remap_pfn_range(vma, addr, pfn, size, prot); return (ret); } /* * Although FreeBSD version of unmap_mapping_range has semantics and types of * parameters compatible with Linux version, the values passed in are different * @obj should match to vm_private_data field of vm_area_struct returned by * mmap file operation handler, see linux_file_mmap_single() sources * @holelen should match to size of area to be munmapped. */ void lkpi_unmap_mapping_range(void *obj, loff_t const holebegin __unused, loff_t const holelen, int even_cows __unused) { vm_object_t devobj; vm_page_t page; int i, page_count; devobj = cdev_pager_lookup(obj); if (devobj != NULL) { page_count = OFF_TO_IDX(holelen); VM_OBJECT_WLOCK(devobj); retry: for (i = 0; i < page_count; i++) { page = vm_page_lookup(devobj, i); if (page == NULL) continue; if (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) goto retry; cdev_pager_free_page(devobj, page); } VM_OBJECT_WUNLOCK(devobj); vm_object_deallocate(devobj); } } int lkpi_arch_phys_wc_add(unsigned long base, unsigned long size) { #ifdef __i386__ struct mem_range_desc *mrdesc; int error, id, act; /* If PAT is available, do nothing */ if (pat_works) return (0); mrdesc = malloc(sizeof(*mrdesc), M_LKMTRR, M_WAITOK); mrdesc->mr_base = base; mrdesc->mr_len = size; mrdesc->mr_flags = MDF_WRITECOMBINE; strlcpy(mrdesc->mr_owner, "drm", sizeof(mrdesc->mr_owner)); act = MEMRANGE_SET_UPDATE; error = mem_range_attr_set(mrdesc, &act); if (error == 0) { error = idr_get_new(&mtrr_idr, mrdesc, &id); MPASS(idr_find(&mtrr_idr, id) == mrdesc); if (error != 0) { act = MEMRANGE_SET_REMOVE; mem_range_attr_set(mrdesc, &act); } } if (error != 0) { free(mrdesc, M_LKMTRR); pr_warn( "Failed to add WC MTRR for [%p-%p]: %d; " "performance may suffer\n", (void *)base, (void *)(base + size - 1), error); } else pr_warn("Successfully added WC MTRR for [%p-%p]\n", (void *)base, (void *)(base + size - 1)); return (error != 0 ? -error : id + __MTRR_ID_BASE); #else return (0); #endif } void lkpi_arch_phys_wc_del(int reg) { #ifdef __i386__ struct mem_range_desc *mrdesc; int act; /* Check if arch_phys_wc_add() failed. */ if (reg < __MTRR_ID_BASE) return; mrdesc = idr_find(&mtrr_idr, reg - __MTRR_ID_BASE); MPASS(mrdesc != NULL); idr_remove(&mtrr_idr, reg - __MTRR_ID_BASE); act = MEMRANGE_SET_REMOVE; mem_range_attr_set(mrdesc, &act); free(mrdesc, M_LKMTRR); #endif } /* * This is a highly simplified version of the Linux page_frag_cache. * We only support up-to 1 single page as fragment size and we will * always return a full page. This may be wasteful on small objects * but the only known consumer (mt76) is either asking for a half-page * or a full page. If this was to become a problem we can implement * a more elaborate version. */ void * linuxkpi_page_frag_alloc(struct page_frag_cache *pfc, size_t fragsz, gfp_t gfp) { vm_page_t pages; if (fragsz == 0) return (NULL); KASSERT(fragsz <= PAGE_SIZE, ("%s: fragsz %zu > PAGE_SIZE not yet " "supported", __func__, fragsz)); pages = alloc_pages(gfp, flsl(howmany(fragsz, PAGE_SIZE) - 1)); if (pages == NULL) return (NULL); pfc->va = linux_page_address(pages); /* Passed in as "count" to __page_frag_cache_drain(). Unused by us. */ pfc->pagecnt_bias = 0; return (pfc->va); } void linuxkpi_page_frag_free(void *addr) { vm_page_t page; page = PHYS_TO_VM_PAGE(vtophys(addr)); linux_free_pages(page, 0); } void linuxkpi__page_frag_cache_drain(struct page *page, size_t count __unused) { linux_free_pages(page, 0); }