Index: sys/dev/atkbdc/atkbd.c =================================================================== --- sys/dev/atkbdc/atkbd.c +++ sys/dev/atkbdc/atkbd.c @@ -475,6 +475,7 @@ evdev_set_id(evdev, BUS_I8042, PS2_KEYBOARD_VENDOR, PS2_KEYBOARD_PRODUCT, 0); evdev_set_methods(evdev, kbd, &atkbd_evdev_methods); + evdev_set_flag(evdev, EVDEV_FLAG_LOCKLESS); evdev_support_event(evdev, EV_SYN); evdev_support_event(evdev, EV_KEY); evdev_support_event(evdev, EV_LED); Index: sys/dev/evdev/cdev.c =================================================================== --- sys/dev/evdev/cdev.c +++ sys/dev/evdev/cdev.c @@ -110,8 +110,12 @@ client->ec_buffer_ready = 0; client->ec_evdev = evdev; - mtx_init(&client->ec_buffer_mtx, "evclient", "evdev", MTX_DEF); - knlist_init_mtx(&client->ec_selp.si_note, &client->ec_buffer_mtx); + if (!bit_test(evdev->ev_flags, EVDEV_FLAG_LOCKLESS)) { + mtx_init(&client->ec_buffer_mtx, "evclient", "evdev", MTX_DEF); + knlist_init_mtx(&client->ec_selp.si_note, + &client->ec_buffer_mtx); + } else + knlist_init_mtx(&client->ec_selp.si_note, evdev->ev_lock); /* Avoid race with evdev_unregister */ EVDEV_LOCK(evdev); @@ -149,11 +153,19 @@ evdev_dispose_client(client->ec_evdev, client); EVDEV_UNLOCK(client->ec_evdev); - knlist_clear(&client->ec_selp.si_note, 0); + /* + * Take kqueue lock explicitly. Otherwise lockless mode will trigger + * knlist_cleardel() 'Mutex owned' assertion as devfs can call + * evdev_dtor() with Giant lock already taken. + */ + EVDEV_CLIENT_LOCKQ(client); + knlist_clear(&client->ec_selp.si_note, 1); + EVDEV_CLIENT_UNLOCKQ(client); seldrain(&client->ec_selp); knlist_destroy(&client->ec_selp.si_note); funsetown(&client->ec_sigio); - mtx_destroy(&client->ec_buffer_mtx); + if (EVDEV_CLIENT_LOCKQ_EXISTS(client)) + mtx_destroy(&client->ec_buffer_mtx); free(client, M_EVDEV); } @@ -202,9 +214,11 @@ (client->ec_buffer_head + 1) % client->ec_buffer_size; remaining--; - EVDEV_CLIENT_UNLOCKQ(client); + if (!EVDEV_CLIENT_LOCKQ_SLEEPABLE(client)) + EVDEV_CLIENT_UNLOCKQ(client); ret = uiomove(&event, sizeof(struct input_event), uio); - EVDEV_CLIENT_LOCKQ(client); + if (!EVDEV_CLIENT_LOCKQ_SLEEPABLE(client)) + EVDEV_CLIENT_LOCKQ(client); } EVDEV_CLIENT_UNLOCKQ(client); @@ -833,7 +847,8 @@ size_t head, tail, count, i; bool last_was_syn = false; - EVDEV_CLIENT_LOCKQ(client); + if (EVDEV_CLIENT_LOCKQ_EXISTS(client)) + EVDEV_CLIENT_LOCKQ(client); i = head = client->ec_buffer_head; tail = client->ec_buffer_tail; @@ -869,5 +884,6 @@ client->ec_buffer_head = i; client->ec_buffer_tail = tail; - EVDEV_CLIENT_UNLOCKQ(client); + if (EVDEV_CLIENT_LOCKQ_EXISTS(client)) + EVDEV_CLIENT_UNLOCKQ(client); } Index: sys/dev/evdev/evdev.h =================================================================== --- sys/dev/evdev/evdev.h +++ sys/dev/evdev/evdev.h @@ -87,6 +87,7 @@ * for MT protocol type B reports */ #define EVDEV_FLAG_MT_AUTOREL 0x02 /* Autorelease MT-slots not listed in * current MT protocol type B report */ +#define EVDEV_FLAG_LOCKLESS 0x03 /* Avoid locks in evdev_push() */ #define EVDEV_FLAG_MAX 0x1F #define EVDEV_FLAG_CNT (EVDEV_FLAG_MAX + 1) Index: sys/dev/evdev/evdev.c =================================================================== --- sys/dev/evdev/evdev.c +++ sys/dev/evdev/evdev.c @@ -68,6 +68,7 @@ int evdev_rcpt_mask = EVDEV_RCPT_SYSMOUSE | EVDEV_RCPT_KBDMUX; int evdev_sysmouse_t_axis = 0; +static int evdev_hz = 20; #ifdef EVDEV_SUPPORT SYSCTL_NODE(_kern, OID_AUTO, evdev, CTLFLAG_RW, 0, "Evdev args"); @@ -76,10 +77,15 @@ "bit2 - mouse hardware, bit3 - keyboard hardware"); SYSCTL_INT(_kern_evdev, OID_AUTO, sysmouse_t_axis, CTLFLAG_RW, &evdev_sysmouse_t_axis, 0, "Extract T-axis from 0-none, 1-ums, 2-psm"); +/* Setting to 0 disables lockless mode and enables notification on interrupt */ +SYSCTL_INT(_kern_evdev, OID_AUTO, poll_rate, CTLFLAG_RW, + &evdev_hz, 0, "Userland notification rate (hz), 0 - disable polling"); #endif static void evdev_start_repeat(struct evdev_dev *, uint16_t); static void evdev_stop_repeat(struct evdev_dev *); +static void evdev_start_notification(struct evdev_dev *); +static void evdev_stop_notification(struct evdev_dev *); static int evdev_check_event(struct evdev_dev *, uint16_t, uint16_t, int32_t); static inline void @@ -209,8 +215,6 @@ if (evdev_event_supported(evdev, EV_REP) && bit_test(evdev->ev_flags, EVDEV_FLAG_SOFTREPEAT)) { - /* Initialize callout */ - callout_init_mtx(&evdev->ev_rep_callout, &evdev->ev_mtx, 0); if (evdev->ev_rep[REP_DELAY] == 0 && evdev->ev_rep[REP_PERIOD] == 0) { @@ -219,6 +223,8 @@ evdev->ev_rep[REP_PERIOD] = 33; } } + callout_init_mtx(&evdev->ev_rep_callout, evdev->ev_lock, 0); + callout_init_mtx(&evdev->ev_note_callout, evdev->ev_lock, 0); /* Initialize multitouch protocol type B states */ if (bit_test(evdev->ev_abs_flags, ABS_MT_SLOT) && @@ -278,9 +284,11 @@ LIST_FOREACH(client, &evdev->ev_clients, ec_link) { evdev_revoke_client(client); evdev_dispose_client(evdev, client); - EVDEV_CLIENT_LOCKQ(client); + if (EVDEV_CLIENT_LOCKQ_EXISTS(client)) + EVDEV_CLIENT_LOCKQ(client); evdev_notify_event(client); - EVDEV_CLIENT_UNLOCKQ(client); + if (EVDEV_CLIENT_LOCKQ_EXISTS(client)) + EVDEV_CLIENT_UNLOCKQ(client); } EVDEV_UNLOCK(evdev); @@ -732,12 +740,19 @@ if (evdev->ev_grabber != NULL && evdev->ev_grabber != client) continue; - EVDEV_CLIENT_LOCKQ(client); + if (EVDEV_CLIENT_LOCKQ_EXISTS(client)) + EVDEV_CLIENT_LOCKQ(client); evdev_client_push(client, type, code, value); - if (type == EV_SYN && code == SYN_REPORT) + /* wakeup, selwakeup and knote take internal locks */ + if (type == EV_SYN && code == SYN_REPORT && !(evdev_hz > 0 && + bit_test(evdev->ev_flags, EVDEV_FLAG_LOCKLESS))) evdev_notify_event(client); - EVDEV_CLIENT_UNLOCKQ(client); + if (EVDEV_CLIENT_LOCKQ_EXISTS(client)) + EVDEV_CLIENT_UNLOCKQ(client); } + if (type == EV_SYN && code == SYN_REPORT && evdev_hz > 0 && + bit_test(evdev->ev_flags, EVDEV_FLAG_LOCKLESS)) + evdev->ev_note = true; evdev->ev_event_count++; } @@ -888,11 +903,14 @@ EVDEV_LOCK_ASSERT(evdev); - if (LIST_EMPTY(&evdev->ev_clients) && evdev->ev_methods != NULL && - evdev->ev_methods->ev_open != NULL) { + if (LIST_EMPTY(&evdev->ev_clients)) { debugf(evdev, "calling ev_open() on device %s", evdev->ev_shortname); - ret = evdev->ev_methods->ev_open(evdev); + if (evdev->ev_methods != NULL && + evdev->ev_methods->ev_open != NULL) + ret = evdev->ev_methods->ev_open(evdev); + if (!ret && bit_test(evdev->ev_flags, EVDEV_FLAG_LOCKLESS)) + evdev_start_notification(evdev); } if (ret == 0) LIST_INSERT_HEAD(&evdev->ev_clients, client, ec_link); @@ -914,6 +932,8 @@ if (evdev_event_supported(evdev, EV_REP) && bit_test(evdev->ev_flags, EVDEV_FLAG_SOFTREPEAT)) evdev_stop_repeat(evdev); + if (bit_test(evdev->ev_flags, EVDEV_FLAG_LOCKLESS)) + evdev_stop_notification(evdev); } evdev_release_client(evdev, client); } @@ -988,4 +1008,45 @@ } } +static void +evdev_notification_callout(void *arg) +{ + struct evdev_dev *evdev = arg; + struct evdev_client *client; + + EVDEV_LOCK_ASSERT(evdev); + + if (evdev->ev_note) { + evdev->ev_note = false; + LIST_FOREACH(client, &evdev->ev_clients, ec_link) { + if (evdev->ev_grabber == NULL || + evdev->ev_grabber == client) + evdev_notify_event(client); + } + } + + evdev_start_notification(evdev); +} + +static void +evdev_start_notification(struct evdev_dev *evdev) +{ + + EVDEV_LOCK_ASSERT(evdev); + + if (evdev_hz > 0) + callout_reset(&evdev->ev_note_callout, + evdev_hz > hz ? 1 : hz / evdev_hz, + evdev_notification_callout, evdev); +} + +static void +evdev_stop_notification(struct evdev_dev *evdev) +{ + + EVDEV_LOCK_ASSERT(evdev); + + callout_stop(&evdev->ev_note_callout); +} + MODULE_VERSION(evdev, 1); Index: sys/dev/evdev/evdev_private.h =================================================================== --- sys/dev/evdev/evdev_private.h +++ sys/dev/evdev/evdev_private.h @@ -92,6 +92,10 @@ struct evdev_client * ev_grabber; size_t ev_report_size; + /* Lockless mode: */ + bool ev_note; + struct callout ev_note_callout; + /* Supported features: */ bitstr_t bit_decl(ev_prop_flags, INPUT_PROP_CNT); bitstr_t bit_decl(ev_type_flags, EV_CNT); @@ -176,15 +180,34 @@ struct input_event ec_buffer[]; }; -#define EVDEV_CLIENT_LOCKQ(client) mtx_lock(&(client)->ec_buffer_mtx) -#define EVDEV_CLIENT_UNLOCKQ(client) mtx_unlock(&(client)->ec_buffer_mtx) -#define EVDEV_CLIENT_LOCKQ_ASSERT(client) \ - mtx_assert(&(client)->ec_buffer_mtx, MA_OWNED) -#define EVDEV_CLIENT_EMPTYQ(client) \ - ((client)->ec_buffer_head == (client)->ec_buffer_ready) -#define EVDEV_CLIENT_SIZEQ(client) \ - (((client)->ec_buffer_ready + (client)->ec_buffer_size - \ - (client)->ec_buffer_head) % (client)->ec_buffer_size) +#define EVDEV_CLIENT_LOCKQ_EXISTS(client) \ + mtx_initialized(&(client)->ec_buffer_mtx) +#define EVDEV_CLIENT_LOCKQ(client) do { \ + if (EVDEV_CLIENT_LOCKQ_EXISTS(client)) \ + mtx_lock(&(client)->ec_buffer_mtx); \ + else \ + EVDEV_LOCK((client)->ec_evdev); \ +} while (0) +#define EVDEV_CLIENT_UNLOCKQ(client) do { \ + if (EVDEV_CLIENT_LOCKQ_EXISTS(client)) \ + mtx_unlock(&(client)->ec_buffer_mtx); \ + else \ + EVDEV_UNLOCK((client)->ec_evdev); \ +} while (0) +#define EVDEV_CLIENT_LOCKQ_ASSERT(client) do { \ + if (EVDEV_CLIENT_LOCKQ_EXISTS(client)) \ + mtx_assert(&(client)->ec_buffer_mtx, MA_OWNED); \ + else \ + EVDEV_LOCK_ASSERT((client)->ec_evdev); \ +} while (0) +#define EVDEV_CLIENT_LOCKQ_SLEEPABLE(client) \ + (!EVDEV_CLIENT_LOCKQ_EXISTS(client) && \ + (client)->ec_evdev->ev_lock == &Giant) +#define EVDEV_CLIENT_EMPTYQ(client) \ + ((client)->ec_buffer_head == (client)->ec_buffer_ready) +#define EVDEV_CLIENT_SIZEQ(client) \ + (((client)->ec_buffer_ready + (client)->ec_buffer_size - \ + (client)->ec_buffer_head) % (client)->ec_buffer_size) /* Input device interface: */ void evdev_send_event(struct evdev_dev *, uint16_t, uint16_t, int32_t); Index: sys/dev/kbdmux/kbdmux.c =================================================================== --- sys/dev/kbdmux/kbdmux.c +++ sys/dev/kbdmux/kbdmux.c @@ -496,6 +496,7 @@ evdev_set_phys(evdev, phys_loc); evdev_set_id(evdev, BUS_VIRTUAL, 0, 0, 0); evdev_set_methods(evdev, kbd, &kbdmux_evdev_methods); + evdev_set_flag(evdev, EVDEV_FLAG_LOCKLESS); evdev_support_event(evdev, EV_SYN); evdev_support_event(evdev, EV_KEY); evdev_support_event(evdev, EV_LED); Index: sys/dev/usb/input/ukbd.c =================================================================== --- sys/dev/usb/input/ukbd.c +++ sys/dev/usb/input/ukbd.c @@ -1345,6 +1345,7 @@ uaa->info.idProduct, 0); evdev_set_serial(evdev, usb_get_serial(uaa->device)); evdev_set_methods(evdev, kbd, &ukbd_evdev_methods); + evdev_set_flag(evdev, EVDEV_FLAG_LOCKLESS); evdev_support_event(evdev, EV_SYN); evdev_support_event(evdev, EV_KEY); if (sc->sc_flags & (UKBD_FLAG_NUMLOCK | UKBD_FLAG_CAPSLOCK |