diff --git a/sys/dev/evdev/cdev.c b/sys/dev/evdev/cdev.c --- a/sys/dev/evdev/cdev.c +++ b/sys/dev/evdev/cdev.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -125,23 +126,20 @@ mtx_init(&client->ec_buffer_mtx, "evclient", "evdev", MTX_DEF); knlist_init_mtx(&client->ec_selp.si_note, &client->ec_buffer_mtx); + ret = EVDEV_LIST_LOCK_SIG(evdev); + if (ret != 0) + goto out; /* Avoid race with evdev_unregister */ - EVDEV_LOCK(evdev); if (dev->si_drv1 == NULL) ret = ENODEV; else ret = evdev_register_client(evdev, client); - - if (ret != 0) - evdev_revoke_client(client); - /* - * Unlock evdev here because non-sleepable lock held - * while calling devfs_set_cdevpriv upsets WITNESS - */ - EVDEV_UNLOCK(evdev); - - if (!ret) + EVDEV_LIST_UNLOCK(evdev); +out: + if (ret == 0) ret = devfs_set_cdevpriv(client, evdev_dtor); + else + client->ec_revoked = true; if (ret != 0) { debugf(client, "cannot register evdev client"); @@ -156,11 +154,13 @@ { struct evdev_client *client = (struct evdev_client *)data; - EVDEV_LOCK(client->ec_evdev); + EVDEV_LIST_LOCK(client->ec_evdev); if (!client->ec_revoked) evdev_dispose_client(client->ec_evdev, client); - EVDEV_UNLOCK(client->ec_evdev); + EVDEV_LIST_UNLOCK(client->ec_evdev); + if (client->ec_evdev->ev_lock_type != EV_LOCK_MTX) + epoch_wait_preempt(client->ec_evdev->ev_epoch); knlist_clear(&client->ec_selp.si_note, 0); seldrain(&client->ec_selp); knlist_destroy(&client->ec_selp.si_note); @@ -547,12 +547,12 @@ if (*(int *)data != 0) return (EINVAL); - EVDEV_LOCK(evdev); + EVDEV_LIST_LOCK(evdev); if (dev->si_drv1 != NULL && !client->ec_revoked) { evdev_dispose_client(evdev, client); evdev_revoke_client(client); } - EVDEV_UNLOCK(evdev); + EVDEV_LIST_UNLOCK(evdev); return (0); case EVIOCSCLOCKID: @@ -717,7 +717,7 @@ evdev_revoke_client(struct evdev_client *client) { - EVDEV_LOCK_ASSERT(client->ec_evdev); + EVDEV_LIST_LOCK_ASSERT(client->ec_evdev); client->ec_revoked = true; } diff --git a/sys/dev/evdev/evdev.h b/sys/dev/evdev/evdev.h --- a/sys/dev/evdev/evdev.h +++ b/sys/dev/evdev/evdev.h @@ -30,6 +30,7 @@ #define _DEV_EVDEV_EVDEV_H #include +#include #include #include #include @@ -110,6 +111,7 @@ const struct evdev_methods *); int evdev_register(struct evdev_dev *); int evdev_register_mtx(struct evdev_dev *, struct mtx *); +int evdev_register_epoch(struct evdev_dev *, epoch_t); int evdev_unregister(struct evdev_dev *); int evdev_push_event(struct evdev_dev *, uint16_t, uint16_t, int32_t); void evdev_support_prop(struct evdev_dev *, uint16_t); diff --git a/sys/dev/evdev/evdev.c b/sys/dev/evdev/evdev.c --- a/sys/dev/evdev/evdev.c +++ b/sys/dev/evdev/evdev.c @@ -31,12 +31,15 @@ #include #include +#include #include +#include #include #include #include #include #include +#include #include #include @@ -295,12 +298,14 @@ evdev->ev_shortname, evdev->ev_name, evdev->ev_serial); /* Initialize internal structures */ - LIST_INIT(&evdev->ev_clients); + CK_SLIST_INIT(&evdev->ev_clients); + sx_init(&evdev->ev_list_lock, "evsx"); if (evdev_event_supported(evdev, EV_REP) && bit_test(evdev->ev_flags, EVDEV_FLAG_SOFTREPEAT)) { /* Initialize callout */ - callout_init_mtx(&evdev->ev_rep_callout, evdev->ev_lock, 0); + callout_init_mtx(&evdev->ev_rep_callout, + evdev->ev_state_lock, 0); if (evdev->ev_rep[REP_DELAY] == 0 && evdev->ev_rep[REP_PERIOD] == 0) { @@ -332,6 +337,8 @@ evdev_sysctl_create(evdev); bail_out: + if (ret != 0) + sx_destroy(&evdev->ev_list_lock); return (ret); } @@ -341,8 +348,9 @@ int ret; evdev->ev_lock_type = EV_LOCK_INTERNAL; - evdev->ev_lock = &evdev->ev_mtx; + evdev->ev_state_lock = &evdev->ev_mtx; mtx_init(&evdev->ev_mtx, "evmtx", NULL, MTX_DEF); + evdev->ev_epoch = global_epoch_preempt; ret = evdev_register_common(evdev); if (ret != 0) @@ -356,10 +364,27 @@ { evdev->ev_lock_type = EV_LOCK_MTX; - evdev->ev_lock = mtx; + evdev->ev_state_lock = mtx; return (evdev_register_common(evdev)); } +int +evdev_register_epoch(struct evdev_dev *evdev, epoch_t epoch) +{ + int ret; + + evdev->ev_lock_type = EV_LOCK_EPOCH; + evdev->ev_state_lock = &evdev->ev_mtx; + mtx_init(&evdev->ev_mtx, "evmtx", NULL, MTX_DEF); + evdev->ev_epoch = epoch; + + ret = evdev_register_common(evdev); + if (ret != 0) + mtx_destroy(&evdev->ev_mtx); + + return (ret); +} + int evdev_unregister(struct evdev_dev *evdev) { @@ -370,22 +395,23 @@ sysctl_ctx_free(&evdev->ev_sysctl_ctx); - EVDEV_LOCK(evdev); + EVDEV_LIST_LOCK(evdev); evdev->ev_cdev->si_drv1 = NULL; /* Wake up sleepers */ - LIST_FOREACH_SAFE(client, &evdev->ev_clients, ec_link, tmp) { + CK_SLIST_FOREACH_SAFE(client, &evdev->ev_clients, ec_link, tmp) { evdev_revoke_client(client); evdev_dispose_client(evdev, client); EVDEV_CLIENT_LOCKQ(client); evdev_notify_event(client); EVDEV_CLIENT_UNLOCKQ(client); } - EVDEV_UNLOCK(evdev); + EVDEV_LIST_UNLOCK(evdev); - /* destroy_dev can sleep so release lock */ + /* release lock to avoid deadlock with evdev_dtor */ ret = evdev_cdev_destroy(evdev); evdev->ev_cdev = NULL; - if (ret == 0 && evdev->ev_lock_type == EV_LOCK_INTERNAL) + sx_destroy(&evdev->ev_list_lock); + if (ret == 0 && evdev->ev_lock_type != EV_LOCK_MTX) mtx_destroy(&evdev->ev_mtx); evdev_free_absinfo(evdev->ev_absinfo); @@ -689,7 +715,7 @@ } else { /* Start/stop callout for evdev repeats */ if (bit_test(evdev->ev_key_states, code) == !*value && - !LIST_EMPTY(&evdev->ev_clients)) { + !CK_SLIST_EMPTY(&evdev->ev_clients)) { if (*value == KEY_EVENT_DOWN) evdev_start_repeat(evdev, code); else @@ -817,6 +843,7 @@ evdev_propagate_event(struct evdev_dev *evdev, uint16_t type, uint16_t code, int32_t value) { + struct epoch_tracker et; struct evdev_client *client; debugf(evdev, "%s pushed event %d/%d/%d", @@ -825,7 +852,9 @@ EVDEV_LOCK_ASSERT(evdev); /* Propagate event through all clients */ - LIST_FOREACH(client, &evdev->ev_clients, ec_link) { + if (evdev->ev_lock_type != EV_LOCK_MTX) + epoch_enter_preempt(evdev->ev_epoch, &et); + CK_SLIST_FOREACH(client, &evdev->ev_clients, ec_link) { if (evdev->ev_grabber != NULL && evdev->ev_grabber != client) continue; @@ -835,6 +864,8 @@ evdev_notify_event(client); EVDEV_CLIENT_UNLOCKQ(client); } + if (evdev->ev_lock_type != EV_LOCK_MTX) + epoch_exit_preempt(evdev->ev_epoch, &et); evdev->ev_event_count++; } @@ -962,10 +993,10 @@ case EV_ABS: case EV_SW: push: - if (evdev->ev_lock_type != EV_LOCK_INTERNAL) + if (evdev->ev_lock_type == EV_LOCK_MTX) EVDEV_LOCK(evdev); ret = evdev_push_event(evdev, type, code, value); - if (evdev->ev_lock_type != EV_LOCK_INTERNAL) + if (evdev->ev_lock_type == EV_LOCK_MTX) EVDEV_UNLOCK(evdev); break; @@ -983,16 +1014,16 @@ debugf(evdev, "adding new client for device %s", evdev->ev_shortname); - EVDEV_LOCK_ASSERT(evdev); + EVDEV_LIST_LOCK_ASSERT(evdev); - if (LIST_EMPTY(&evdev->ev_clients) && evdev->ev_methods != NULL && + if (CK_SLIST_EMPTY(&evdev->ev_clients) && evdev->ev_methods != NULL && evdev->ev_methods->ev_open != NULL) { debugf(evdev, "calling ev_open() on device %s", evdev->ev_shortname); ret = evdev->ev_methods->ev_open(evdev); } if (ret == 0) - LIST_INSERT_HEAD(&evdev->ev_clients, client, ec_link); + CK_SLIST_INSERT_HEAD(&evdev->ev_clients, client, ec_link); return (ret); } @@ -1001,18 +1032,27 @@ { debugf(evdev, "removing client for device %s", evdev->ev_shortname); - EVDEV_LOCK_ASSERT(evdev); + EVDEV_LIST_LOCK_ASSERT(evdev); - LIST_REMOVE(client, ec_link); - if (LIST_EMPTY(&evdev->ev_clients)) { + CK_SLIST_REMOVE(&evdev->ev_clients, client, evdev_client, ec_link); + if (CK_SLIST_EMPTY(&evdev->ev_clients)) { if (evdev->ev_methods != NULL && evdev->ev_methods->ev_close != NULL) (void)evdev->ev_methods->ev_close(evdev); if (evdev_event_supported(evdev, EV_REP) && - bit_test(evdev->ev_flags, EVDEV_FLAG_SOFTREPEAT)) + bit_test(evdev->ev_flags, EVDEV_FLAG_SOFTREPEAT)) { + if (evdev->ev_lock_type != EV_LOCK_MTX) + EVDEV_LOCK(evdev); evdev_stop_repeat(evdev); + if (evdev->ev_lock_type != EV_LOCK_MTX) + EVDEV_UNLOCK(evdev); + } } + if (evdev->ev_lock_type != EV_LOCK_MTX) + EVDEV_LOCK(evdev); evdev_release_client(evdev, client); + if (evdev->ev_lock_type != EV_LOCK_MTX) + EVDEV_UNLOCK(evdev); } int diff --git a/sys/dev/evdev/evdev_private.h b/sys/dev/evdev/evdev_private.h --- a/sys/dev/evdev/evdev_private.h +++ b/sys/dev/evdev/evdev_private.h @@ -31,12 +31,15 @@ #define _DEV_EVDEV_EVDEV_PRIVATE_H #include +#include +#include #include #include #include #include #include #include +#include #include #include @@ -77,8 +80,9 @@ enum evdev_lock_type { - EV_LOCK_INTERNAL = 0, /* Internal evdev mutex */ + EV_LOCK_INTERNAL = 0, /* Internal epoch */ EV_LOCK_MTX, /* Driver`s mutex */ + EV_LOCK_EPOCH, /* External epoch */ }; struct evdev_dev @@ -89,8 +93,10 @@ struct cdev * ev_cdev; int ev_unit; enum evdev_lock_type ev_lock_type; - struct mtx * ev_lock; - struct mtx ev_mtx; + struct mtx * ev_state_lock; /* Evdev state lock */ + struct mtx ev_mtx; /* Evdev internal state lock */ + epoch_t ev_epoch; + struct sx ev_list_lock; /* Evdev client list lock */ struct input_id ev_id; struct evdev_client * ev_grabber; size_t ev_report_size; @@ -139,28 +145,56 @@ struct sysctl_ctx_list ev_sysctl_ctx; LIST_ENTRY(evdev_dev) ev_link; - LIST_HEAD(, evdev_client) ev_clients; + CK_SLIST_HEAD(, evdev_client) ev_clients; }; #define SYSTEM_CONSOLE_LOCK &Giant -#define EVDEV_LOCK(evdev) mtx_lock((evdev)->ev_lock) -#define EVDEV_UNLOCK(evdev) mtx_unlock((evdev)->ev_lock) +#define EVDEV_LOCK(evdev) mtx_lock((evdev)->ev_state_lock) +#define EVDEV_UNLOCK(evdev) mtx_unlock((evdev)->ev_state_lock) #define EVDEV_LOCK_ASSERT(evdev) do { \ - if ((evdev)->ev_lock != SYSTEM_CONSOLE_LOCK) \ - mtx_assert((evdev)->ev_lock, MA_OWNED); \ + if ((evdev)->ev_state_lock != SYSTEM_CONSOLE_LOCK) \ + mtx_assert((evdev)->ev_state_lock, MA_OWNED); \ } while (0) #define EVDEV_ENTER(evdev) do { \ - if ((evdev)->ev_lock_type == EV_LOCK_INTERNAL) \ + if ((evdev)->ev_lock_type != EV_LOCK_MTX) \ EVDEV_LOCK(evdev); \ else \ EVDEV_LOCK_ASSERT(evdev); \ } while (0) #define EVDEV_EXIT(evdev) do { \ - if ((evdev)->ev_lock_type == EV_LOCK_INTERNAL) \ + if ((evdev)->ev_lock_type != EV_LOCK_MTX) \ EVDEV_UNLOCK(evdev); \ } while (0) +#define EVDEV_LIST_LOCK(evdev) do { \ + if ((evdev)->ev_lock_type == EV_LOCK_MTX) \ + EVDEV_LOCK(evdev); \ + else \ + sx_xlock(&(evdev)->ev_list_lock); \ +} while (0) +#define EVDEV_LIST_UNLOCK(evdev) do { \ + if ((evdev)->ev_lock_type == EV_LOCK_MTX) \ + EVDEV_UNLOCK(evdev); \ + else \ + sx_unlock(&(evdev)->ev_list_lock); \ +} while (0) +#define EVDEV_LIST_LOCK_ASSERT(evdev) do { \ + if ((evdev)->ev_lock_type == EV_LOCK_MTX) \ + EVDEV_LOCK_ASSERT(evdev); \ + else \ + sx_assert(&(evdev)->ev_list_lock, MA_OWNED); \ +} while (0) +static inline int +EVDEV_LIST_LOCK_SIG(struct evdev_dev *evdev) +{ + if (evdev->ev_lock_type == EV_LOCK_MTX) { + EVDEV_LOCK(evdev); + return (0); + } + return (sx_xlock_sig(&evdev->ev_list_lock)); +} + struct evdev_client { struct evdev_dev * ec_evdev; @@ -177,7 +211,7 @@ bool ec_blocked; bool ec_selected; - LIST_ENTRY(evdev_client) ec_link; + CK_SLIST_ENTRY(evdev_client) ec_link; struct input_event ec_buffer[]; };