Page MenuHomeFreeBSD
Files F167650509

D15933.id44176.diff
No OneTemporary
Actions

D15933.id44176.diff
View Options

Index: sys/amd64/include/counter.h
===================================================================
--- sys/amd64/include/counter.h
+++ sys/amd64/include/counter.h
@@ -45,7 +45,7 @@
counter_u64_read_one(uint64_t *p, int cpu)
{
- return (*(uint64_t *)((char *)p + sizeof(struct pcpu) * cpu));
+ return (*(uint64_t *)((char *)p + UMA_PCPU_ZONE_SIZE * cpu));
}
static inline uint64_t
@@ -65,7 +65,7 @@
counter_u64_zero_one_cpu(void *arg)
{
- *((uint64_t *)((char *)arg + sizeof(struct pcpu) *
+ *((uint64_t *)((char *)arg + UMA_PCPU_ZONE_SIZE *
PCPU_GET(cpuid))) = 0;
}
Index: sys/arm/include/counter.h
===================================================================
--- sys/arm/include/counter.h
+++ sys/arm/include/counter.h
@@ -47,7 +47,7 @@
counter_u64_read_one(uint64_t *p, int cpu)
{
- return (atomic_load_64((uint64_t *)((char *)p + sizeof(struct pcpu) *
+ return (atomic_load_64((uint64_t *)((char *)p + UMA_PCPU_ZONE_SIZE *
cpu)));
}
@@ -68,7 +68,7 @@
counter_u64_zero_one_cpu(void *arg)
{
- atomic_store_64((uint64_t *)((char *)arg + sizeof(struct pcpu) *
+ atomic_store_64((uint64_t *)((char *)arg + UMA_PCPU_ZONE_SIZE *
PCPU_GET(cpuid)), 0);
}
Index: sys/arm64/include/counter.h
===================================================================
--- sys/arm64/include/counter.h
+++ sys/arm64/include/counter.h
@@ -44,7 +44,7 @@
counter_u64_read_one(uint64_t *p, int cpu)
{
- return (*(uint64_t *)((char *)p + sizeof(struct pcpu) * cpu));
+ return (*(uint64_t *)((char *)p + UMA_PCPU_ZONE_SIZE * cpu));
}
static inline uint64_t
@@ -64,7 +64,7 @@
counter_u64_zero_one_cpu(void *arg)
{
- *((uint64_t *)((char *)arg + sizeof(struct pcpu) *
+ *((uint64_t *)((char *)arg + UMA_PCPU_ZONE_SIZE *
PCPU_GET(cpuid))) = 0;
}
Index: sys/i386/include/counter.h
===================================================================
--- sys/i386/include/counter.h
+++ sys/i386/include/counter.h
@@ -104,13 +104,13 @@
critical_enter();
CPU_FOREACH(i) {
res += *(uint64_t *)((char *)p +
- sizeof(struct pcpu) * i);
+ UMA_PCPU_ZONE_SIZE * i);
}
critical_exit();
} else {
CPU_FOREACH(i)
res += counter_u64_read_one_8b((uint64_t *)((char *)p +
- sizeof(struct pcpu) * i));
+ UMA_PCPU_ZONE_SIZE * i));
}
return (res);
}
@@ -137,7 +137,7 @@
{
uint64_t *p;
- p = (uint64_t *)((char *)arg + sizeof(struct pcpu) * PCPU_GET(cpuid));
+ p = (uint64_t *)((char *)arg + UMA_PCPU_ZONE_SIZE * PCPU_GET(cpuid));
counter_u64_zero_one_8b(p);
}
@@ -149,7 +149,7 @@
if ((cpu_feature & CPUID_CX8) == 0) {
critical_enter();
CPU_FOREACH(i)
- *(uint64_t *)((char *)c + sizeof(struct pcpu) * i) = 0;
+ *(uint64_t *)((char *)c + UMA_PCPU_ZONE_SIZE * i) = 0;
critical_exit();
} else {
smp_rendezvous(smp_no_rendezvous_barrier,
Index: sys/mips/include/counter.h
===================================================================
--- sys/mips/include/counter.h
+++ sys/mips/include/counter.h
@@ -47,7 +47,7 @@
counter_u64_read_one(uint64_t *p, int cpu)
{
- return (*(uint64_t *)((char *)p + sizeof(struct pcpu) * cpu));
+ return (*(uint64_t *)((char *)p + UMA_PCPU_ZONE_SIZE * cpu));
}
static inline uint64_t
@@ -68,7 +68,7 @@
counter_u64_zero_one_cpu(void *arg)
{
- *((uint64_t *)((char *)arg + sizeof(struct pcpu) *
+ *((uint64_t *)((char *)arg + UMA_PCPU_ZONE_SIZE *
PCPU_GET(cpuid))) = 0;
}
Index: sys/powerpc/include/counter.h
===================================================================
--- sys/powerpc/include/counter.h
+++ sys/powerpc/include/counter.h
@@ -50,7 +50,7 @@
counter_u64_read_one(uint64_t *p, int cpu)
{
- return (*(uint64_t *)((char *)p + sizeof(struct pcpu) * cpu));
+ return (*(uint64_t *)((char *)p + UMA_PCPU_ZONE_SIZE * cpu));
}
static inline uint64_t
@@ -70,7 +70,7 @@
counter_u64_zero_one_cpu(void *arg)
{
- *((uint64_t *)((char *)arg + sizeof(struct pcpu) *
+ *((uint64_t *)((char *)arg + UMA_PCPU_ZONE_SIZE *
PCPU_GET(cpuid))) = 0;
}
@@ -113,7 +113,7 @@
counter_u64_read_one(uint64_t *p, int cpu)
{
- return (*(uint64_t *)((char *)p + sizeof(struct pcpu) * cpu));
+ return (*(uint64_t *)((char *)p + UMA_PCPU_ZONE_SIZE * cpu));
}
static inline uint64_t
@@ -134,7 +134,7 @@
counter_u64_zero_one_cpu(void *arg)
{
- *((uint64_t *)((char *)arg + sizeof(struct pcpu) *
+ *((uint64_t *)((char *)arg + UMA_PCPU_ZONE_SIZE *
PCPU_GET(cpuid))) = 0;
}
Index: sys/riscv/include/counter.h
===================================================================
--- sys/riscv/include/counter.h
+++ sys/riscv/include/counter.h
@@ -46,7 +46,7 @@
counter_u64_read_one(uint64_t *p, int cpu)
{
- return (*(uint64_t *)((char *)p + sizeof(struct pcpu) * cpu));
+ return (*(uint64_t *)((char *)p + UMA_PCPU_ZONE_SIZE * cpu));
}
static inline uint64_t
@@ -67,7 +67,7 @@
counter_u64_zero_one_cpu(void *arg)
{
- *((uint64_t *)((char *)arg + sizeof(struct pcpu) *
+ *((uint64_t *)((char *)arg + UMA_PCPU_ZONE_SIZE *
PCPU_GET(cpuid))) = 0;
}
Index: sys/sparc64/include/counter.h
===================================================================
--- sys/sparc64/include/counter.h
+++ sys/sparc64/include/counter.h
@@ -47,7 +47,7 @@
counter_u64_read_one(uint64_t *p, int cpu)
{
- return (*(uint64_t *)((char *)p + sizeof(struct pcpu) * cpu));
+ return (*(uint64_t *)((char *)p + UMA_PCPU_ZONE_SIZE * cpu));
}
static inline uint64_t
@@ -68,7 +68,7 @@
counter_u64_zero_one_cpu(void *arg)
{
- *((uint64_t *)((char *)arg + sizeof(struct pcpu) *
+ *((uint64_t *)((char *)arg + UMA_PCPU_ZONE_SIZE *
PCPU_GET(cpuid))) = 0;
}
Index: sys/sys/pcpu.h
===================================================================
--- sys/sys/pcpu.h
+++ sys/sys/pcpu.h
@@ -208,14 +208,16 @@
zpcpu_get(void *base)
{
- return ((char *)(base) + sizeof(struct pcpu) * curcpu);
+ /* UMA_PCPU_ZONE_SIZE == PAGE_SIZE */
+ return ((char *)(base) + PAGE_SIZE * curcpu);
}
static inline void *
zpcpu_get_cpu(void *base, int cpu)
{
- return ((char *)(base) + sizeof(struct pcpu) * cpu);
+ /* UMA_PCPU_ZONE_SIZE == PAGE_SIZE */
+ return ((char *)(base) + PAGE_SIZE * cpu);
}
/*
Index: sys/vm/uma.h
===================================================================
--- sys/vm/uma.h
+++ sys/vm/uma.h
@@ -44,6 +44,8 @@
/* User visible parameters */
#define UMA_SMALLEST_UNIT (PAGE_SIZE / 256) /* Smallest item allocated */
+#define UMA_PCPU_ZONE_SIZE PAGE_SIZE
+
/* Types and type defs */
struct uma_zone;
@@ -279,8 +281,7 @@
* mini-dumps.
*/
#define UMA_ZONE_PCPU 0x8000 /*
- * Allocates mp_maxid + 1 slabs sized to
- * sizeof(struct pcpu).
+ * Allocates mp_maxid + 1 slabs of PAGE_SIZE
*/
#define UMA_ZONE_NUMA 0x10000 /*
* NUMA aware Zone. Implements a best
Index: sys/vm/uma_core.c
===================================================================
--- sys/vm/uma_core.c
+++ sys/vm/uma_core.c
@@ -229,8 +229,10 @@
static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
+static void *pcpu_page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
static void page_free(void *, vm_size_t, uint8_t);
+static void pcpu_page_free(void *, vm_size_t, uint8_t);
static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int);
static void cache_drain(uma_zone_t);
static void bucket_drain(uma_zone_t, uma_bucket_t);
@@ -1172,6 +1174,54 @@
return (p);
}
+static void *
+pcpu_page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
+ int wait)
+{
+ TAILQ_HEAD(, vm_page) alloctail;
+ vm_offset_t addr, zkva;
+ struct pcpu *pc;
+ int cpu;
+ vm_page_t p, p_next;
+
+ TAILQ_INIT(&alloctail);
+ MPASS(bytes == (mp_maxid+1)*PAGE_SIZE);
+ *pflag = UMA_SLAB_KERNEL;
+
+ for (cpu = 0; cpu <= mp_maxid; cpu++) {
+ if (CPU_ABSENT(cpu)) {
+ p = vm_page_alloc_domain(NULL, 0, 0, VM_ALLOC_INTERRUPT |
+ VM_ALLOC_WIRED | VM_ALLOC_NOOBJ |
+ ((wait & M_WAITOK) != 0 ? VM_ALLOC_WAITOK :
+ VM_ALLOC_NOWAIT));
+
+ } else {
+ pc = pcpu_find(cpu);
+ p = vm_page_alloc_domain(NULL, 0, pc->pc_domain, VM_ALLOC_INTERRUPT |
+ VM_ALLOC_WIRED | VM_ALLOC_NOOBJ |
+ ((wait & M_WAITOK) != 0 ? VM_ALLOC_WAITOK :
+ VM_ALLOC_NOWAIT));
+ }
+ if (__predict_false(p == NULL))
+ goto fail;
+ TAILQ_INSERT_TAIL(&alloctail, p, listq);
+ }
+ if ((addr = kva_alloc(bytes)) == 0)
+ goto fail;
+ zkva = addr;
+ TAILQ_FOREACH(p, &alloctail, listq) {
+ pmap_qenter(zkva, &p, 1);
+ zkva += PAGE_SIZE;
+ }
+ return ((void*)addr);
+ fail:
+ TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) {
+ vm_page_unwire(p, PQ_NONE);
+ vm_page_free(p);
+ }
+ return (NULL);
+}
+
/*
* Allocates a number of pages from within an object
*
@@ -1257,6 +1307,37 @@
kmem_free(vmem, (vm_offset_t)mem, size);
}
+/*
+ * Frees pcpu zone allocations
+ *
+ * Arguments:
+ * mem A pointer to the memory to be freed
+ * size The size of the memory being freed
+ * flags The original p->us_flags field
+ *
+ * Returns:
+ * Nothing
+ */
+static void
+pcpu_page_free(void *mem, vm_size_t size, uint8_t flags)
+{
+ vm_offset_t sva, curva;
+ vm_paddr_t paddr;
+ vm_page_t m;
+
+ MPASS(size == (mp_maxid+1)*PAGE_SIZE);
+ sva = (vm_offset_t)mem;
+ for (curva = sva; curva < sva + size; curva += PAGE_SIZE) {
+ paddr = pmap_kextract(curva);
+ m = PHYS_TO_VM_PAGE(paddr);
+ vm_page_unwire(m, PQ_NONE);
+ vm_page_free(m);
+ }
+ pmap_qremove(sva, size >> PAGE_SHIFT);
+ kva_free(sva, size);
+}
+
+
/*
* Zero fill initializer
*
@@ -1290,9 +1371,8 @@
if (keg->uk_flags & UMA_ZONE_PCPU) {
u_int ncpus = (mp_maxid + 1) ? (mp_maxid + 1) : MAXCPU;
- slabsize = sizeof(struct pcpu);
- keg->uk_ppera = howmany(ncpus * sizeof(struct pcpu),
- PAGE_SIZE);
+ slabsize = PAGE_SIZE;
+ keg->uk_ppera = ncpus;
} else {
slabsize = UMA_SLAB_SIZE;
keg->uk_ppera = 1;
@@ -1311,7 +1391,7 @@
keg->uk_rsize = rsize;
KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 ||
- keg->uk_rsize < sizeof(struct pcpu),
+ keg->uk_rsize < UMA_PCPU_ZONE_SIZE,
("%s: size %u too large", __func__, keg->uk_rsize));
if (keg->uk_flags & UMA_ZONE_OFFPAGE)
@@ -1529,6 +1609,8 @@
else if (keg->uk_ppera == 1)
keg->uk_allocf = uma_small_alloc;
#endif
+ else if (keg->uk_flags & UMA_ZONE_PCPU)
+ keg->uk_allocf = pcpu_page_alloc;
else
keg->uk_allocf = page_alloc;
#ifdef UMA_MD_SMALL_ALLOC
@@ -1536,6 +1618,9 @@
keg->uk_freef = uma_small_free;
else
#endif
+ if (keg->uk_flags & UMA_ZONE_PCPU)
+ keg->uk_freef = pcpu_page_free;
+ else
keg->uk_freef = page_free;
/*

File Metadata

Mime Type
text/plain
Expires
Mon, Aug 24, 12:45 PM (18 h, 20 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
37184258
Default Alt Text
D15933.id44176.diff (10 KB)

Event Timeline