diff --git a/sys/compat/linuxkpi/common/include/linux/gfp.h b/sys/compat/linuxkpi/common/include/linux/gfp.h --- a/sys/compat/linuxkpi/common/include/linux/gfp.h +++ b/sys/compat/linuxkpi/common/include/linux/gfp.h @@ -134,6 +134,8 @@ return (linux_alloc_pages(GFP_ATOMIC, order)); } +struct folio *folio_alloc(gfp_t gfp, unsigned int order); + /* * Page management for mapped pages: */ diff --git a/sys/compat/linuxkpi/common/include/linux/mm.h b/sys/compat/linuxkpi/common/include/linux/mm.h --- a/sys/compat/linuxkpi/common/include/linux/mm.h +++ b/sys/compat/linuxkpi/common/include/linux/mm.h @@ -161,6 +161,14 @@ return (virt_to_page(p)); } +static inline struct folio * +virt_to_folio(const void *p) +{ + struct page *page = virt_to_page(p); + + return (page_folio(page)); +} + /* * Compute log2 of the power of two rounded up count of pages * needed for size bytes. @@ -282,8 +290,30 @@ __free_page(page); } -void linux_release_pages(struct page **pages, int nr); -#define release_pages(pages, nr) linux_release_pages((pages), (nr)) +static inline void +folio_get(struct folio *folio) +{ + get_page(&folio->page); +} + +static inline void +folio_put(struct folio *folio) +{ + put_page(&folio->page); +} + +/* + * Linux uses the following "transparent" union so that `release_pages()` + * accepts both a list of `struct page` or a list of `struct folio`. This + * relies on the fact that a `struct folio` can be cast to a `struct page`. + */ +typedef union { + struct page **pages; + struct folio **folios; +} release_pages_arg __attribute__ ((__transparent_union__)); + +void linux_release_pages(release_pages_arg arg, int nr); +#define release_pages(arg, nr) linux_release_pages((arg), (nr)) extern long lkpi_get_user_pages(unsigned long start, unsigned long nr_pages, @@ -416,4 +446,34 @@ return (false); } +static inline unsigned long +folio_pfn(struct folio *folio) +{ + return (page_to_pfn(&folio->page)); +} + +static inline long +folio_nr_pages(struct folio *folio) +{ + return (1); +} + +static inline size_t +folio_size(struct folio *folio) +{ + return (PAGE_SIZE); +} + +static inline void +folio_mark_dirty(struct folio *folio) +{ + set_page_dirty(&folio->page); +} + +static inline void * +folio_address(const struct folio *folio) +{ + return (page_address(&folio->page)); +} + #endif /* _LINUXKPI_LINUX_MM_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/mm_types.h b/sys/compat/linuxkpi/common/include/linux/mm_types.h --- a/sys/compat/linuxkpi/common/include/linux/mm_types.h +++ b/sys/compat/linuxkpi/common/include/linux/mm_types.h @@ -79,4 +79,15 @@ extern struct mm_struct *linux_get_task_mm(struct task_struct *); #define get_task_mm(task) linux_get_task_mm(task) +struct folio { + /* + * The page member must be at the beginning because `page_folio(p)` + * casts from a `struct page` to a `struct folio`. + * + * `release_pages()` also relies on this to be able to accept either a + * list of `struct page` or a list of `struct folio`. + */ + struct page page; +}; + #endif /* _LINUXKPI_LINUX_MM_TYPES_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/page-flags.h b/sys/compat/linuxkpi/common/include/linux/page-flags.h --- a/sys/compat/linuxkpi/common/include/linux/page-flags.h +++ b/sys/compat/linuxkpi/common/include/linux/page-flags.h @@ -29,6 +29,13 @@ #ifndef _LINUXKPI_LINUX_PAGEFLAGS_H_ #define _LINUXKPI_LINUX_PAGEFLAGS_H_ +#include + #define PageHighMem(p) (0) +#define page_folio(p) \ + (_Generic((p), \ + const struct page *: (const struct folio *)(p), \ + struct page *: (struct folio *)(p))) + #endif /* _LINUXKPI_LINUX_PAGEFLAGS_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/pagemap.h b/sys/compat/linuxkpi/common/include/linux/pagemap.h --- a/sys/compat/linuxkpi/common/include/linux/pagemap.h +++ b/sys/compat/linuxkpi/common/include/linux/pagemap.h @@ -33,6 +33,8 @@ #include #include +struct folio_batch; + #define invalidate_mapping_pages(...) \ linux_invalidate_mapping_pages(__VA_ARGS__) diff --git a/sys/compat/linuxkpi/common/include/linux/pagevec.h b/sys/compat/linuxkpi/common/include/linux/pagevec.h --- a/sys/compat/linuxkpi/common/include/linux/pagevec.h +++ b/sys/compat/linuxkpi/common/include/linux/pagevec.h @@ -66,4 +66,72 @@ { } +/* + * struct folio + * + * On Linux, `struct folio` replaces `struct page`. To manage a list of folios, + * there is `struct folio_batch` on top of this, which replaces `struct + * pagevec` above. + * + * Here is the original description when `struct folio` was added to the Linux + * kernel: + * "A struct folio is a new abstraction to replace the venerable struct page. + * A function which takes a struct folio argument declares that it will + * operate on the entire (possibly compound) page, not just PAGE_SIZE bytes. + * In return, the caller guarantees that the pointer it is passing does not + * point to a tail page. No change to generated code." + */ + +struct folio; + +struct folio_batch { + uint8_t nr; + struct folio *folios[PAGEVEC_SIZE]; +}; + +static inline void +folio_batch_init(struct folio_batch *fbatch) +{ + fbatch->nr = 0; +} + +static inline void +folio_batch_reinit(struct folio_batch *fbatch) +{ + fbatch->nr = 0; +} + +static inline unsigned int +folio_batch_count(struct folio_batch *fbatch) +{ + return (fbatch->nr); +} + +static inline unsigned int +folio_batch_space(struct folio_batch *fbatch) +{ + return (PAGEVEC_SIZE - fbatch->nr); +} + +static inline unsigned int +folio_batch_add(struct folio_batch *fbatch, struct folio *folio) +{ + KASSERT( + fbatch->nr < PAGEVEC_SIZE, + ("struct folio_batch %p is full", fbatch)); + + fbatch->folios[fbatch->nr++] = folio; + + return (folio_batch_space(fbatch)); +} + +void __folio_batch_release(struct folio_batch *fbatch); + +static inline void +folio_batch_release(struct folio_batch *fbatch) +{ + if (folio_batch_count(fbatch)) + __folio_batch_release(fbatch); +} + #endif /* _LINUXKPI_LINUX_PAGEVEC_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/scatterlist.h b/sys/compat/linuxkpi/common/include/linux/scatterlist.h --- a/sys/compat/linuxkpi/common/include/linux/scatterlist.h +++ b/sys/compat/linuxkpi/common/include/linux/scatterlist.h @@ -674,4 +674,11 @@ return (total); } +static inline void +sg_set_folio(struct scatterlist *sg, struct folio *folio, size_t len, + size_t offset) +{ + sg_set_page(sg, &folio->page, len, offset); +} + #endif /* _LINUXKPI_LINUX_SCATTERLIST_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/shmem_fs.h b/sys/compat/linuxkpi/common/include/linux/shmem_fs.h --- a/sys/compat/linuxkpi/common/include/linux/shmem_fs.h +++ b/sys/compat/linuxkpi/common/include/linux/shmem_fs.h @@ -54,4 +54,14 @@ #define shmem_truncate_range(...) \ linux_shmem_truncate_range(__VA_ARGS__) +static inline struct folio * +shmem_read_folio_gfp(vm_object_t obj, int pindex, gfp_t gfp) +{ + struct page *page; + + page = shmem_read_mapping_page_gfp(obj, pindex, gfp); + + return (page_folio(page)); +} + #endif /* _LINUXKPI_LINUX_SHMEM_FS_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/swap.h b/sys/compat/linuxkpi/common/include/linux/swap.h --- a/sys/compat/linuxkpi/common/include/linux/swap.h +++ b/sys/compat/linuxkpi/common/include/linux/swap.h @@ -37,6 +37,9 @@ #include #include +#include +#include + static inline long get_nr_swap_pages(void) { @@ -54,4 +57,15 @@ return (curproc == pageproc); } +static inline void +folio_mark_accessed(struct folio *folio) +{ + mark_page_accessed(&folio->page); +} + +static inline void +check_move_unevictable_folios(struct folio_batch *fbatch) +{ +} + #endif diff --git a/sys/compat/linuxkpi/common/src/linux_folio.c b/sys/compat/linuxkpi/common/src/linux_folio.c new file mode 100644 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_folio.c @@ -0,0 +1,58 @@ +/*- + * Copyright (c) 2024-2025 The FreeBSD Foundation + * Copyright (c) 2024-2025 Jean-Sébastien Pédron + * + * This software was developed by Jean-Sébastien Pédron under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +struct folio * +folio_alloc(gfp_t gfp, unsigned int order) +{ + struct page *page; + struct folio *folio; + + /* + * Allocated pages are wired already. There is no need to increase a + * refcount here. + */ + page = alloc_pages(gfp | __GFP_COMP, order); + folio = (struct folio *)page; + + return (folio); +} + +void +__folio_batch_release(struct folio_batch *fbatch) +{ + release_pages(fbatch->folios, folio_batch_count(fbatch)); + + folio_batch_reinit(fbatch); +} diff --git a/sys/compat/linuxkpi/common/src/linux_page.c b/sys/compat/linuxkpi/common/src/linux_page.c --- a/sys/compat/linuxkpi/common/src/linux_page.c +++ b/sys/compat/linuxkpi/common/src/linux_page.c @@ -194,12 +194,14 @@ } void -linux_release_pages(struct page **pages, int nr) +linux_release_pages(release_pages_arg arg, int nr) { int i; + CTASSERT(offsetof(struct folio, page) == 0); + for (i = 0; i < nr; i++) - __free_page(pages[i]); + __free_page(arg.pages[i]); } vm_offset_t diff --git a/sys/conf/files b/sys/conf/files --- a/sys/conf/files +++ b/sys/conf/files @@ -4590,6 +4590,8 @@ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_firmware.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" +compat/linuxkpi/common/src/linux_folio.c optional compat_linuxkpi \ + compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_fpu.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_hrtimer.c optional compat_linuxkpi \ diff --git a/sys/modules/linuxkpi/Makefile b/sys/modules/linuxkpi/Makefile --- a/sys/modules/linuxkpi/Makefile +++ b/sys/modules/linuxkpi/Makefile @@ -7,6 +7,7 @@ linux_dmi.c \ linux_domain.c \ linux_firmware.c \ + linux_folio.c \ linux_fpu.c \ linux_hrtimer.c \ linux_idr.c \