Index: sys/dev/hwpmc/hwpmc_logging.c =================================================================== --- sys/dev/hwpmc/hwpmc_logging.c +++ sys/dev/hwpmc/hwpmc_logging.c @@ -50,7 +50,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -116,8 +118,10 @@ /* reserve LEN bytes of space and initialize the entry header */ #define _PMCLOG_RESERVE(PO,TYPE,LEN,ACTION) do { \ uint32_t *_le; \ - int _len = roundup((LEN), sizeof(uint32_t)); \ + int _len = roundup((LEN), sizeof(uint32_t)); \ + spinlock_enter(); \ if ((_le = pmclog_reserve((PO), _len)) == NULL) { \ + spinlock_exit(); \ ACTION; \ } \ *_le = _PMCLOG_TO_HEADER(TYPE,_len); \ @@ -139,7 +143,13 @@ #define PMCLOG_EMITNULLSTRING(L) do { bzero(_le, (L)); } while (0) #define PMCLOG_DESPATCH(PO) \ - pmclog_release((PO)); \ + pmclog_release((PO)); \ + spinlock_exit(); \ + } while (0) + +#define PMCLOG_DESPATCH_SYNC(PO) \ + pmclog_schedule_io((PO)); \ + spinlock_exit(); \ } while (0) @@ -191,6 +201,7 @@ static void pmclog_release(struct pmc_owner *po); static uint32_t *pmclog_reserve(struct pmc_owner *po, int length); static void pmclog_schedule_io(struct pmc_owner *po); +static void pmclog_schedule_all(struct pmc_owner *po); static void pmclog_stop_kthread(struct pmc_owner *po); /* @@ -206,9 +217,7 @@ { struct pmclog_buffer *plb; - mtx_assert(&po->po_mtx, MA_OWNED); - - KASSERT(po->po_curbuf == NULL, + KASSERT(po->po_curbuf[curcpu] == NULL, ("[pmclog,%d] po=%p current buffer still valid", __LINE__, po)); mtx_lock_spin(&pmc_bufferlist_mtx); @@ -227,7 +236,7 @@ plb->plb_base, plb->plb_fence)); #endif - po->po_curbuf = plb; + po->po_curbuf[curcpu] = plb; /* update stats */ atomic_add_int(&pmc_stats.pm_buffer_requests, 1); @@ -460,18 +469,21 @@ static void pmclog_release(struct pmc_owner *po) { - KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base, + struct pmclog_buffer *plb; + + spinlock_enter(); + plb = po->po_curbuf[curcpu]; + KASSERT(plb->plb_ptr >= plb->plb_base, ("[pmclog,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__, - po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base)); - KASSERT(po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence, + po, plb->plb_ptr, plb->plb_base)); + KASSERT(plb->plb_ptr <= plb->plb_fence, ("[pmclog,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__, - po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_fence)); + po, plb->plb_ptr, plb->plb_fence)); /* schedule an I/O if we've filled a buffer */ - if (po->po_curbuf->plb_ptr >= po->po_curbuf->plb_fence) + if (plb->plb_ptr >= plb->plb_fence) pmclog_schedule_io(po); - - mtx_unlock_spin(&po->po_mtx); + spinlock_exit(); PMCDBG1(LOG,REL,1, "po=%p", po); } @@ -492,36 +504,32 @@ uintptr_t newptr, oldptr; uint32_t *lh; struct timespec ts; + struct pmclog_buffer *plb, **pplb; PMCDBG2(LOG,ALL,1, "po=%p len=%d", po, length); KASSERT(length % sizeof(uint32_t) == 0, ("[pmclog,%d] length not a multiple of word size", __LINE__)); - mtx_lock_spin(&po->po_mtx); - /* No more data when shutdown in progress. */ - if (po->po_flags & PMC_PO_SHUTDOWN) { - mtx_unlock_spin(&po->po_mtx); + if (po->po_flags & PMC_PO_SHUTDOWN) return (NULL); - } - if (po->po_curbuf == NULL) - if (pmclog_get_buffer(po) != 0) { - mtx_unlock_spin(&po->po_mtx); - return (NULL); - } + pplb = &po->po_curbuf[curcpu]; + if (*pplb == NULL && pmclog_get_buffer(po) != 0) + goto fail; - KASSERT(po->po_curbuf != NULL, + KASSERT(*pplb != NULL, ("[pmclog,%d] po=%p no current buffer", __LINE__, po)); - KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base && - po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence, + plb = *pplb; + KASSERT(plb->plb_ptr >= plb->plb_base && + plb->plb_ptr <= plb->plb_fence, ("[pmclog,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p", - __LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base, - po->po_curbuf->plb_fence)); + __LINE__, po, plb->plb_ptr, plb->plb_base, + plb->plb_fence)); - oldptr = (uintptr_t) po->po_curbuf->plb_ptr; + oldptr = (uintptr_t) plb->plb_ptr; newptr = oldptr + length; KASSERT(oldptr != (uintptr_t) NULL, @@ -531,8 +539,8 @@ * If we have space in the current buffer, return a pointer to * available space with the PO structure locked. */ - if (newptr <= (uintptr_t) po->po_curbuf->plb_fence) { - po->po_curbuf->plb_ptr = (char *) newptr; + if (newptr <= (uintptr_t) plb->plb_fence) { + plb->plb_ptr = (char *) newptr; goto done; } @@ -542,24 +550,23 @@ */ pmclog_schedule_io(po); - if (pmclog_get_buffer(po) != 0) { - mtx_unlock_spin(&po->po_mtx); - return (NULL); - } + if (pmclog_get_buffer(po) != 0) + goto fail; - KASSERT(po->po_curbuf != NULL, + plb = *pplb; + KASSERT(plb != NULL, ("[pmclog,%d] po=%p no current buffer", __LINE__, po)); - KASSERT(po->po_curbuf->plb_ptr != NULL, + KASSERT(plb->plb_ptr != NULL, ("[pmclog,%d] null return from pmc_get_log_buffer", __LINE__)); - KASSERT(po->po_curbuf->plb_ptr == po->po_curbuf->plb_base && - po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence, + KASSERT(plb->plb_ptr == plb->plb_base && + plb->plb_ptr <= plb->plb_fence, ("[pmclog,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p", - __LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base, - po->po_curbuf->plb_fence)); + __LINE__, po, plb->plb_ptr, plb->plb_base, + plb->plb_fence)); - oldptr = (uintptr_t) po->po_curbuf->plb_ptr; + oldptr = (uintptr_t) plb->plb_ptr; done: lh = (uint32_t *) oldptr; @@ -568,6 +575,8 @@ *lh++ = ts.tv_sec & 0xFFFFFFFF; *lh++ = ts.tv_nsec & 0xFFFFFFF; return ((uint32_t *) oldptr); + fail: + return (NULL); } /* @@ -579,26 +588,28 @@ static void pmclog_schedule_io(struct pmc_owner *po) { - KASSERT(po->po_curbuf != NULL, - ("[pmclog,%d] schedule_io with null buffer po=%p", __LINE__, po)); + struct pmclog_buffer *plb; - KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base, + plb = po->po_curbuf[curcpu]; + po->po_curbuf[curcpu] = NULL; + KASSERT(plb != NULL, + ("[pmclog,%d] schedule_io with null buffer po=%p", __LINE__, po)); + KASSERT(plb->plb_ptr >= plb->plb_base, ("[pmclog,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__, - po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base)); - KASSERT(po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence, + po, plb->plb_ptr, plb->plb_base)); + KASSERT(plb->plb_ptr <= plb->plb_fence, ("[pmclog,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__, - po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_fence)); + po, plb->plb_ptr, plb->plb_fence)); PMCDBG1(LOG,SIO, 1, "po=%p", po); - mtx_assert(&po->po_mtx, MA_OWNED); - /* * Add the current buffer to the tail of the buffer list and * wakeup the helper. */ - TAILQ_INSERT_TAIL(&po->po_logbuffers, po->po_curbuf, plb_next); - po->po_curbuf = NULL; + mtx_lock_spin(&po->po_mtx); + TAILQ_INSERT_TAIL(&po->po_logbuffers, plb, plb_next); + mtx_unlock_spin(&po->po_mtx); wakeup_one(po); } @@ -673,7 +684,7 @@ sizeof(struct pmclog_initialize)); PMCLOG_EMIT32(PMC_VERSION); PMCLOG_EMIT32(md->pmd_cputype); - PMCLOG_DESPATCH(po); + PMCLOG_DESPATCH_SYNC(po); return (0); @@ -726,14 +737,21 @@ TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next); mtx_unlock_spin(&pmc_bufferlist_mtx); } - - /* return the 'current' buffer to the global pool */ - if ((lb = po->po_curbuf) != NULL) { - PMCLOG_INIT_BUFFER_DESCRIPTOR(lb); - mtx_lock_spin(&pmc_bufferlist_mtx); - TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next); - mtx_unlock_spin(&pmc_bufferlist_mtx); + for (int i = 0; i < mp_ncpus; i++) { + thread_lock(curthread); + sched_bind(curthread, i); + thread_unlock(curthread); + /* return the 'current' buffer to the global pool */ + if ((lb = po->po_curbuf[curcpu]) != NULL) { + PMCLOG_INIT_BUFFER_DESCRIPTOR(lb); + mtx_lock_spin(&pmc_bufferlist_mtx); + TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next); + mtx_unlock_spin(&pmc_bufferlist_mtx); + } } + thread_lock(curthread); + sched_unbind(curthread); + thread_unlock(curthread); /* drop a reference to the fd */ if (po->po_file != NULL) { @@ -754,7 +772,6 @@ pmclog_flush(struct pmc_owner *po) { int error; - struct pmclog_buffer *lb; PMCDBG1(LOG,FLS,1, "po=%p", po); @@ -776,23 +793,43 @@ goto error; } - /* - * Schedule the current buffer if any and not empty. - */ - mtx_lock_spin(&po->po_mtx); - lb = po->po_curbuf; - if (lb && lb->plb_ptr != lb->plb_base) { - pmclog_schedule_io(po); - } else - error = ENOBUFS; - mtx_unlock_spin(&po->po_mtx); - + pmclog_schedule_all(po); error: mtx_unlock(&pmc_kthread_mtx); return (error); } +static void +pmclog_schedule_one_cond(void *arg) +{ + struct pmc_owner *po = arg; + struct pmclog_buffer *plb; + + spinlock_enter(); + plb = po->po_curbuf[curcpu]; + if (plb && plb->plb_ptr != plb->plb_base) + pmclog_schedule_io(po); + spinlock_exit(); +} + +static void +pmclog_schedule_all(struct pmc_owner *po) +{ + /* + * Schedule the current buffer if any and not empty. + */ + for (int i = 0; i < mp_ncpus; i++) { + thread_lock(curthread); + sched_bind(curthread, i); + thread_unlock(curthread); + pmclog_schedule_one_cond(po); + } + thread_lock(curthread); + sched_unbind(curthread); + thread_unlock(curthread); +} + int pmclog_close(struct pmc_owner *po) { @@ -806,19 +843,14 @@ /* * Schedule the current buffer. */ - mtx_lock_spin(&po->po_mtx); - if (po->po_curbuf) - pmclog_schedule_io(po); - else - wakeup_one(po); - mtx_unlock_spin(&po->po_mtx); + pmclog_schedule_all(po); + wakeup_one(po); /* * Initiate shutdown: no new data queued, * thread will close file on last block. */ po->po_flags |= PMC_PO_SHUTDOWN; - mtx_unlock(&pmc_kthread_mtx); return (0); @@ -851,7 +883,7 @@ pmclog_process_closelog(struct pmc_owner *po) { PMCLOG_RESERVE(po,CLOSELOG,sizeof(struct pmclog_closelog)); - PMCLOG_DESPATCH(po); + PMCLOG_DESPATCH_SYNC(po); } void @@ -915,14 +947,14 @@ else PMCLOG_EMITNULLSTRING(PMC_NAME_MAX); pmc_soft_ev_release(ps); - PMCLOG_DESPATCH(po); + PMCLOG_DESPATCH_SYNC(po); } else { PMCLOG_RESERVE(po, PMCALLOCATE, sizeof(struct pmclog_pmcallocate)); PMCLOG_EMIT32(pm->pm_id); PMCLOG_EMIT32(pm->pm_event); PMCLOG_EMIT32(pm->pm_flags); - PMCLOG_DESPATCH(po); + PMCLOG_DESPATCH_SYNC(po); } } @@ -943,7 +975,7 @@ PMCLOG_EMIT32(pm->pm_id); PMCLOG_EMIT32(pid); PMCLOG_EMITSTRING(path, pathlen); - PMCLOG_DESPATCH(po); + PMCLOG_DESPATCH_SYNC(po); } void @@ -958,7 +990,7 @@ PMCLOG_RESERVE(po, PMCDETACH, sizeof(struct pmclog_pmcdetach)); PMCLOG_EMIT32(pm->pm_id); PMCLOG_EMIT32(pid); - PMCLOG_DESPATCH(po); + PMCLOG_DESPATCH_SYNC(po); } /* Index: sys/sys/pmc.h =================================================================== --- sys/sys/pmc.h +++ sys/sys/pmc.h @@ -625,9 +625,9 @@ #define PMC_HASH_SIZE 1024 #define PMC_MTXPOOL_SIZE 2048 -#define PMC_LOG_BUFFER_SIZE 4 +#define PMC_LOG_BUFFER_SIZE 512 #define PMC_NLOGBUFFERS 1024 -#define PMC_NSAMPLES 1024 +#define PMC_NSAMPLES 4096 #define PMC_CALLCHAIN_DEPTH 32 #define PMC_SYSCTL_NAME_PREFIX "kern." PMC_MODULE_NAME "." @@ -816,11 +816,11 @@ struct proc *po_owner; /* owner proc */ uint32_t po_flags; /* (k) flags PMC_PO_* */ struct proc *po_kthread; /* (k) helper kthread */ - struct pmclog_buffer *po_curbuf; /* current log buffer */ struct file *po_file; /* file reference */ int po_error; /* recorded error */ short po_sscount; /* # SS PMCs owned */ short po_logprocmaps; /* global mappings done */ + struct pmclog_buffer *po_curbuf[MAXCPU]; /* current log buffer */ }; #define PMC_PO_OWNS_LOGFILE 0x00000001 /* has a log file */