Page MenuHomeFreeBSD
Files F160727379

D55957.diff
No OneTemporary
Actions

D55957.diff
View Options

diff --git a/sys/arm64/arm64/mte.c b/sys/arm64/arm64/mte.c
--- a/sys/arm64/arm64/mte.c
+++ b/sys/arm64/arm64/mte.c
@@ -111,6 +111,62 @@
page->md.pv_flags |= PV_MTE_TAGGED;
}
+void
+mte_save_tags(vm_page_t m, void *tags)
+{
+ char *va;
+ size_t block_size;
+ uint64_t tag_word, tmp;
+
+ block_size = mte_block_size();
+ va = PHYS_TO_DMAP(m->phys_addr);
+ for (int i = 0; i < PAGE_SIZE / 32; i += sizeof(uint64_t)) {
+ tag_word = 0;
+ __asm __volatile(
+ ".arch_extension memtag \n"
+ "1: \n"
+ /* Load the tag granule */
+ "ldgm %1, [%2] \n"
+ /* Store the parts in tag_word */
+ "orr %0, %0, %1 \n"
+ /* Next tag granule */
+ "add %2, %2, %3 \n"
+ /* Check if we are on the last granule for this word */
+ "tst %2, 0xff \n"
+ "b.ne 1b \n"
+ ".arch_extension nomemtag" :
+ "+r" (tag_word), "=r"(tmp), "+r" (va) : "r" (block_size));
+
+ *(uint64_t *)((uintptr_t)tags + i) = tag_word;
+ }
+}
+
+void
+mte_load_tags(vm_page_t m, const void *tags)
+{
+ char *va;
+ size_t block_size;
+ uint64_t tag_word;
+
+ block_size = mte_block_size();
+ va = PHYS_TO_DMAP(m->phys_addr);
+ for (int i = 0; i < PAGE_SIZE / 32; i += sizeof(uint64_t)) {
+ tag_word = *(uint64_t *)((uintptr_t)tags + i);
+ __asm __volatile(
+ ".arch_extension memtag \n"
+ "1: \n"
+ /* Store the tag granule */
+ "stgm %1, [%0] \n"
+ /* Next tag granule */
+ "add %0, %0, %2 \n"
+ /* Check if we are on the last granule for this word */
+ "tst %0, 0xff \n"
+ "b.ne 1b \n"
+ ".arch_extension nomemtag" :
+ "+r" (va) : "r" (tag_word), "r" (block_size));
+ }
+}
+
/**
* Copy the allocation tags from given target to destination page. This is called
* on a copy-on-write and anything that causes a pmap_copy_page call.
diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h
--- a/sys/arm64/include/cpu.h
+++ b/sys/arm64/include/cpu.h
@@ -285,6 +285,8 @@
void mte_thread0(struct thread *);
void mte_sync_tags(vm_page_t page);
+void mte_save_tags(vm_page_t, void *);
+void mte_load_tags(vm_page_t, const void *);
void mte_copy_tags(vm_page_t, vm_page_t, char *, char *);
/* Functions to read the sanitised view of the special registers */
diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c
--- a/sys/vm/swap_pager.c
+++ b/sys/vm/swap_pager.c
@@ -142,6 +142,9 @@
struct swblk {
vm_pindex_t p;
daddr_t d[SWAP_META_PAGES];
+#ifdef __aarch64__
+ uint8_t *t[SWAP_META_PAGES];
+#endif
};
/*
@@ -427,6 +430,10 @@
static uma_zone_t swblk_zone;
static uma_zone_t swpctrie_zone;
+#ifdef __aarch64__
+static uma_zone_t swtag_zone;
+#endif
+
/*
* pagerops for OBJT_SWAP - "swap pager". Some ops are also global procedure
* calls hooked from other parts of the VM system and do not appear here.
@@ -770,6 +777,25 @@
if (!uma_zone_reserve_kva(swpctrie_zone, n))
printf("Cannot reserve swap pctrie zone, "
"reduce kern.maxswzone.\n");
+
+#ifdef __aarch64__
+ swtag_zone = uma_zcreate("swtag", PAGE_SIZE / 32, NULL, NULL,
+ NULL, NULL, (PAGE_SIZE / 32) - 1, 0);
+ /* XXX: Should it be n * SWAP_META_PAGES ? */
+ if (!uma_zone_reserve_kva(swtag_zone, n))
+ printf("Cannot reserve swap pctrie zone, "
+ "reduce kern.maxswzone.\n");
+#endif
+}
+
+static void
+swap_swblk_free(struct swblk *sb)
+{
+#ifdef __aarch64__
+ for (int i = 0; i < SWAP_META_PAGES; i++)
+ uma_zfree(swtag_zone, sb->t[i]);
+#endif
+ uma_zfree(swblk_zone, sb);
}
bool
@@ -1540,6 +1566,42 @@
return (r);
}
+static void
+swp_pager_meta_arch_put_page(vm_page_t m)
+{
+#if defined(__aarch64__)
+ struct swblk *sb;
+ vm_pindex_t modpi;
+
+ sb = swblk_lookup(m->object, m->pindex);
+ modpi = m->pindex % SWAP_META_PAGES;
+ if ((m->md.pv_flags & PV_MTE_TAGGED) != 0) {
+ if (sb->t[modpi] == NULL)
+ sb->t[modpi] = uma_zalloc(swtag_zone, M_WAITOK);
+ mte_save_tags(m, sb->t[modpi]);
+ } else if (sb->t[modpi] != NULL) {
+ uma_zfree(swtag_zone, sb->t[modpi]);
+ sb->t[modpi] = NULL;
+ }
+#endif
+}
+
+static void
+swp_pager_meta_arch_get_page(vm_page_t m)
+{
+#if defined(__aarch64__)
+ struct swblk *sb;
+ vm_pindex_t modpi;
+
+ sb = swblk_lookup(m->object, m->pindex);
+ modpi = m->pindex % SWAP_META_PAGES;
+ if (sb->t[modpi] != NULL) {
+ mte_load_tags(m, sb->t[modpi]);
+ dmb(ishst); /* Ensure the tags are loaded */
+ }
+#endif
+}
+
/*
* swap_pager_putpages:
*
@@ -1625,6 +1687,11 @@
}
VM_OBJECT_WUNLOCK(object);
+ for (j = 0; j < n; ++j) {
+ mreq = ma[i + j];
+ swp_pager_meta_arch_put_page(mreq);
+ }
+
bp = uma_zalloc(swwbuf_zone, M_WAITOK);
MPASS((bp->b_flags & B_MAXPHYS) != 0);
if (async)
@@ -1804,6 +1871,7 @@
KASSERT(m->dirty == 0,
("swp_pager_async_iodone: page %p is dirty", m));
+ swp_pager_meta_arch_get_page(m);
vm_page_valid(m);
if (i < bp->b_pgbefore ||
i >= bp->b_npages - bp->b_pgafter)
@@ -2013,7 +2081,7 @@
}
if (sb_empty) {
swblk_iter_remove(&blks);
- uma_zfree(swblk_zone, sb);
+ swap_swblk_free(sb);
}
/*
@@ -2137,7 +2205,7 @@
if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
swblk_lookup_remove(object, sb);
- uma_zfree(swblk_zone, sb);
+ swap_swblk_free(sb);
}
}
@@ -2174,8 +2242,12 @@
pageproc ? M_USE_RESERVE : 0));
if (sb != NULL) {
sb->p = rounddown(pindex, SWAP_META_PAGES);
- for (i = 0; i < SWAP_META_PAGES; i++)
+ for (i = 0; i < SWAP_META_PAGES; i++) {
sb->d[i] = SWAPBLK_NONE;
+#ifdef __aarch64__
+ sb->t[i] = NULL;
+#endif
+ }
if (atomic_cmpset_int(&swblk_zone_exhausted,
1, 0))
printf("swblk zone ok\n");
@@ -2212,7 +2284,7 @@
break;
}
if (nowait_noreplace) {
- uma_zfree(swblk_zone, sb);
+ swap_swblk_free(sb);
return (swapblk);
}
VM_OBJECT_WUNLOCK(object);
@@ -2228,7 +2300,7 @@
VM_OBJECT_WLOCK(object);
sb1 = swblk_iter_reinit(blks, object, pindex);
if (sb1 != NULL) {
- uma_zfree(swblk_zone, sb);
+ swap_swblk_free(sb);
sb = sb1;
goto allocated;
}
@@ -2250,7 +2322,7 @@
if (swapblk == SWAPBLK_NONE &&
swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
swblk_iter_remove(blks);
- uma_zfree(swblk_zone, sb);
+ swap_swblk_free(sb);
}
}
return (prev_swapblk);
@@ -2312,7 +2384,7 @@
if (swp_pager_swblk_empty(sb, 0, start) &&
swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
swblk_iter_remove(&srcblks);
- uma_zfree(swblk_zone, sb);
+ swap_swblk_free(sb);
}
if (d_mask != 0) {
/* Finish block transfer, with the lock released. */
@@ -2381,7 +2453,7 @@
if (swp_pager_swblk_empty(sb, 0, start) &&
swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
swblk_iter_remove(&blks);
- uma_zfree(swblk_zone, sb);
+ swap_swblk_free(sb);
}
}
swp_pager_freeswapspace(&range);
@@ -2399,7 +2471,7 @@
if (sb->d[i] != SWAPBLK_NONE)
swp_pager_update_freerange(range, sb->d[i]);
}
- uma_zfree(swblk_zone, sb);
+ swap_swblk_free(sb);
}
/*

File Metadata

Mime Type
text/plain
Expires
Sun, Jun 28, 6:11 AM (19 h, 28 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
34425537
Default Alt Text
D55957.diff (6 KB)

Event Timeline