Index: sys/kern/subr_smr.c =================================================================== --- sys/kern/subr_smr.c +++ sys/kern/subr_smr.c @@ -146,6 +146,14 @@ static uma_zone_t smr_shared_zone; static uma_zone_t smr_zone; +enum { + BOOT_COLD = 1, + BOOT_ZONE = 2, + BOOT_PCPU = 3, +} booted = BOOT_COLD; + +static smr_shared_t smr_init_list; + #ifndef INVARIANTS #define SMR_SEQ_INIT 1 /* All valid sequence numbers are odd. */ #define SMR_SEQ_INCR 2 @@ -582,30 +590,51 @@ return (success); } +static void +smr_init_cpu(smr_shared_t s, int cpu, int limit, int flags) +{ + smr_t c; + + c = zpcpu_get_cpu(s->s_cpu, cpu); + c->c_seq = SMR_SEQ_INVALID; + c->c_shared = s; + c->c_deferred = 0; + c->c_limit = limit; + c->c_flags = flags; +} + smr_t smr_create(const char *name, int limit, int flags) { - smr_t smr, c; + smr_t smr; smr_shared_t s; int i; + KASSERT(booted >= BOOT_ZONE, + ("%s: SMR is not yet initialized", __func__)); + s = uma_zalloc(smr_shared_zone, M_WAITOK); smr = uma_zalloc_pcpu(smr_zone, M_WAITOK); s->s_name = name; s->s_rd_seq = s->s_wr.seq = SMR_SEQ_INIT; s->s_wr.ticks = ticks; + s->s_cpu = smr; - /* Initialize all CPUS, not just those running. */ - for (i = 0; i <= mp_maxid; i++) { - c = zpcpu_get_cpu(smr, i); - c->c_seq = SMR_SEQ_INVALID; - c->c_shared = s; - c->c_deferred = 0; - c->c_limit = limit; - c->c_flags = flags; + smr_init_cpu(s, 0, limit, flags); + if (booted < BOOT_PCPU) { + /* + * Per-CPU memory for APs might not be accessible at this point. + * Defer initialization of the rest of the CPU sections. + */ + s->s_next = smr_init_list; + smr_init_list = s; + } else { + /* Initialize all CPUs, not just those running. */ + for (i = 1; i <= mp_maxid; i++) + smr_init_cpu(s, i, limit, flags); + atomic_thread_fence_seq_cst(); } - atomic_thread_fence_seq_cst(); return (smr); } @@ -622,12 +651,33 @@ /* * Initialize the UMA slab zone. */ -void -smr_init(void) +static void +smr_init(void *arg __unused) { smr_shared_zone = uma_zcreate("SMR SHARED", sizeof(struct smr_shared), NULL, NULL, NULL, NULL, (CACHE_LINE_SIZE * 2) - 1, 0); smr_zone = uma_zcreate("SMR CPU", sizeof(struct smr), NULL, NULL, NULL, NULL, (CACHE_LINE_SIZE * 2) - 1, UMA_ZONE_PCPU); + booted = BOOT_ZONE; +} +SYSINIT(smr_init, SI_SUB_SMR, SI_ORDER_FIRST, smr_init, NULL); + +static void +smr_init2(void *arg __unused) +{ + smr_shared_t s; + smr_t c0; + int i; + + /* + * Complete initialization of per-CPU sections, started in smr_create(). + */ + for (s = smr_init_list; s != NULL; s = s->s_next) { + c0 = zpcpu_get_cpu(s->s_cpu, 0); + for (i = 1; i <= mp_maxid; i++) + smr_init_cpu(s, i, c0->c_limit, c0->c_flags); + } + booted = BOOT_PCPU; } +SYSINIT(smr_init2, SI_SUB_CPU, SI_ORDER_ANY, smr_init2, NULL); Index: sys/sys/kernel.h =================================================================== --- sys/sys/kernel.h +++ sys/sys/kernel.h @@ -92,6 +92,7 @@ SI_SUB_TUNABLES = 0x0700000, /* establish tunable values */ SI_SUB_COPYRIGHT = 0x0800001, /* first use of console*/ SI_SUB_VM = 0x1000000, /* virtual memory system init */ + SI_SUB_SMR = 0x1010000, /* SMR is initialized */ SI_SUB_COUNTER = 0x1100000, /* counter(9) is initialized */ SI_SUB_KMEM = 0x1800000, /* kernel memory*/ SI_SUB_HYPERVISOR = 0x1A40000, /* Index: sys/sys/smr.h =================================================================== --- sys/sys/smr.h +++ sys/sys/smr.h @@ -65,9 +65,11 @@ uint64_t _pair; }; struct smr_shared { - const char *s_name; /* Name for debugging/reporting. */ union s_wr s_wr; /* Write sequence */ smr_seq_t s_rd_seq; /* Minimum observed read sequence. */ + smr_t s_cpu; /* Per-CPU sections. */ + struct smr_shared *s_next; /* Deferred init list entry. */ + const char *s_name; /* Name for debugging/reporting. */ }; typedef struct smr_shared *smr_shared_t; @@ -255,7 +257,4 @@ smr_wait(smr, smr_advance(smr)); } -/* Only at startup. */ -void smr_init(void); - #endif /* _SYS_SMR_H_ */ Index: sys/vm/uma_core.c =================================================================== --- sys/vm/uma_core.c +++ sys/vm/uma_core.c @@ -2228,6 +2228,9 @@ keg->uk_flags |= UMA_ZONE_ROUNDROBIN; #endif + KASSERT(booted >= BOOT_KVA || (keg->uk_flags & UMA_ZONE_PCPU) == 0, + ("%s: early pcpu keg %s", __func__, keg->uk_name)); + /* * If we haven't booted yet we need allocations to go through the * startup cache until the vm is ready. @@ -2307,10 +2310,7 @@ if (keg->uk_allocf == startup_alloc) { /* Switch to the real allocator. */ - if (keg->uk_flags & UMA_ZONE_PCPU) - keg->uk_allocf = pcpu_page_alloc; - else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && - keg->uk_ppera > 1) + if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && keg->uk_ppera > 1) keg->uk_allocf = contig_alloc; else keg->uk_allocf = page_alloc; @@ -2869,7 +2869,6 @@ NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); bucket_init(); - smr_init(); } #ifndef UMA_MD_SMALL_ALLOC @@ -3312,7 +3311,7 @@ #ifdef UMA_ZALLOC_DEBUG void *item; - KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, + KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0 || !smp_started, ("uma_zalloc_arg: called with non-SMR zone.\n")); if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN) return (item); @@ -4655,7 +4654,8 @@ uma_zone_set_smr(uma_zone_t zone, smr_t smr) { - ZONE_ASSERT_COLD(zone); + if (smp_started) + ZONE_ASSERT_COLD(zone); KASSERT(smr != NULL, ("Got NULL smr")); KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, Index: sys/vm/vm_radix.c =================================================================== --- sys/vm/vm_radix.c +++ sys/vm/vm_radix.c @@ -375,10 +375,20 @@ vm_radix_node_zone = uma_zcreate("RADIX NODE", sizeof(struct vm_radix_node), NULL, NULL, NULL, NULL, - VM_RADIX_PAD, UMA_ZONE_VM | UMA_ZONE_SMR | UMA_ZONE_ZINIT); - vm_radix_smr = uma_zone_get_smr(vm_radix_node_zone); + VM_RADIX_PAD, UMA_ZONE_VM | UMA_ZONE_ZINIT); } +/* + * Attach SMR structures to the radix node zone. + */ +static void +vm_radix_zinit2(void *arg __unused) +{ + vm_radix_smr = smr_create("RADIX NODE", 0, 0); + uma_zone_set_smr(vm_radix_node_zone, vm_radix_smr); +} +SYSINIT(vm_radix_zinit2, SI_SUB_SMR, SI_ORDER_ANY, vm_radix_zinit2, NULL); + /* * Inserts the key-value pair into the trie. * Panics if the key already exists.