Index: sys/dev/iscsi/iscsi.h =================================================================== --- sys/dev/iscsi/iscsi.h +++ sys/dev/iscsi/iscsi.h @@ -131,7 +131,8 @@ TAILQ_HEAD(, iscsi_session) sc_sessions; struct cv sc_cv; unsigned int sc_last_session_id; - eventhandler_tag sc_shutdown_eh; + eventhandler_tag sc_shutdown_pre_eh; + eventhandler_tag sc_shutdown_post_eh; }; #endif /* !ISCSI_H */ Index: sys/dev/iscsi/iscsi.c =================================================================== --- sys/dev/iscsi/iscsi.c +++ sys/dev/iscsi/iscsi.c @@ -98,6 +98,9 @@ static int fail_on_disconnection = 0; SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN, &fail_on_disconnection, 0, "Destroy CAM SIM on connection failure"); +static int fail_on_shutdown = 1; +SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_shutdown, CTLFLAG_RWTUN, + &fail_on_shutdown, 0, "Fail disconnected sessions on shutdown"); static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator"); static uma_zone_t iscsi_outstanding_zone; @@ -416,9 +419,6 @@ struct iscsi_softc *sc; sc = is->is_softc; - sx_xlock(&sc->sc_lock); - TAILQ_REMOVE(&sc->sc_sessions, is, is_next); - sx_xunlock(&sc->sc_lock); icl_conn_close(is->is_conn); callout_drain(&is->is_callout); @@ -450,6 +450,10 @@ #ifdef ICL_KERNEL_PROXY cv_destroy(&is->is_login_cv); #endif + sx_xlock(&sc->sc_lock); + TAILQ_REMOVE(&sc->sc_sessions, is, is_next); + sx_xunlock(&sc->sc_lock); + ISCSI_SESSION_DEBUG(is, "terminated"); free(is, M_ISCSI); @@ -473,12 +477,7 @@ STAILQ_EMPTY(&is->is_postponed)) cv_wait(&is->is_maintenance_cv, &is->is_lock); - if (is->is_reconnecting) { - ISCSI_SESSION_UNLOCK(is); - iscsi_maintenance_thread_reconnect(is); - continue; - } - + /* Terminate supersedes reconnect. */ if (is->is_terminating) { ISCSI_SESSION_UNLOCK(is); iscsi_maintenance_thread_terminate(is); @@ -486,6 +485,12 @@ return; } + if (is->is_reconnecting) { + ISCSI_SESSION_UNLOCK(is); + iscsi_maintenance_thread_reconnect(is); + continue; + } + iscsi_session_send_postponed(is); ISCSI_SESSION_UNLOCK(is); } @@ -605,6 +610,11 @@ return; out: + if (is->is_terminating) { + ISCSI_SESSION_UNLOCK(is); + return; + } + ISCSI_SESSION_UNLOCK(is); if (reconnect_needed) @@ -2190,7 +2200,6 @@ if (is->is_immediate_data && (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) { len = csio->dxfer_len; - //ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len); if (len > is->is_first_burst_length) { ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_first_burst_length); len = is->is_first_burst_length; @@ -2318,30 +2327,59 @@ } static void -iscsi_shutdown(struct iscsi_softc *sc) +iscsi_terminate_sessions(struct iscsi_softc *sc) +{ + struct iscsi_session *is; + + sx_slock(&sc->sc_lock); + TAILQ_FOREACH(is, &sc->sc_sessions, is_next) + iscsi_session_terminate(is); + while(!TAILQ_EMPTY(&sc->sc_sessions)) { + ISCSI_DEBUG("waiting for sessions to terminate"); + cv_wait(&sc->sc_cv, &sc->sc_lock); + } + ISCSI_DEBUG("all sessions terminated"); + sx_sunlock(&sc->sc_lock); +} + +static void +iscsi_shutdown_pre(struct iscsi_softc *sc) { struct iscsi_session *is; + if (!fail_on_shutdown) + return; + /* - * Trying to reconnect during system shutdown would lead to hang. + * Force fail all disconnected sessions to ensure we don't hang during + * shutdown due to LUNs with mounted filesystems supported by disconnected + * iSCSI sessions which will fail all queued BIOs. */ + ISCSI_DEBUG("forcing failing all disconnected sessions due to shutdown"); + fail_on_disconnection = 1; - /* - * If we have any sessions waiting for reconnection, request - * maintenance thread to fail them immediately instead of waiting - * for reconnect timeout. - */ sx_slock(&sc->sc_lock); TAILQ_FOREACH(is, &sc->sc_sessions, is_next) { ISCSI_SESSION_LOCK(is); - if (is->is_waiting_for_iscsid) + if (!is->is_connected) { + ISCSI_SESSION_DEBUG(is, "force failing disconnected session early"); iscsi_session_reconnect(is); + } ISCSI_SESSION_UNLOCK(is); } sx_sunlock(&sc->sc_lock); } +static void +iscsi_shutdown_post(struct iscsi_softc *sc) +{ + + + ISCSI_DEBUG("removing all sessions due to shutdown"); + iscsi_terminate_sessions(sc); +} + static int iscsi_load(void) { @@ -2364,8 +2402,16 @@ } sc->sc_cdev->si_drv1 = sc; - sc->sc_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_pre_sync, - iscsi_shutdown, sc, SHUTDOWN_PRI_DEFAULT-1); + sc->sc_shutdown_pre_eh = EVENTHANDLER_REGISTER(shutdown_pre_sync, + iscsi_shutdown_pre, sc, SHUTDOWN_PRI_FIRST); + /* + * This needs to get run after filesystem shutdown and before CAM + * shutdown otherwise when rebooting with an iSCSI session that is + * disconnected but has outstanding requests, dashutdown() will hang on + * cam_periph_runccb(). + */ + sc->sc_shutdown_post_eh = EVENTHANDLER_REGISTER(shutdown_post_sync, + iscsi_shutdown_post, sc, SHUTDOWN_PRI_DEFAULT - 1); return (0); } @@ -2373,7 +2419,7 @@ static int iscsi_unload(void) { - struct iscsi_session *is, *tmp; + if (sc->sc_cdev != NULL) { ISCSI_DEBUG("removing device node"); @@ -2381,18 +2427,12 @@ ISCSI_DEBUG("device node removed"); } - if (sc->sc_shutdown_eh != NULL) - EVENTHANDLER_DEREGISTER(shutdown_pre_sync, sc->sc_shutdown_eh); + if (sc->sc_shutdown_pre_eh != NULL) + EVENTHANDLER_DEREGISTER(shutdown_pre_sync, sc->sc_shutdown_pre_eh); + if (sc->sc_shutdown_post_eh != NULL) + EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->sc_shutdown_post_eh); - sx_slock(&sc->sc_lock); - TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) - iscsi_session_terminate(is); - while(!TAILQ_EMPTY(&sc->sc_sessions)) { - ISCSI_DEBUG("waiting for sessions to terminate"); - cv_wait(&sc->sc_cv, &sc->sc_lock); - } - ISCSI_DEBUG("all sessions terminated"); - sx_sunlock(&sc->sc_lock); + iscsi_terminate_sessions(sc); uma_zdestroy(iscsi_outstanding_zone); sx_destroy(&sc->sc_lock); Index: sys/dev/iscsi_initiator/iscsi.c =================================================================== --- sys/dev/iscsi_initiator/iscsi.c +++ sys/dev/iscsi_initiator/iscsi.c @@ -727,7 +727,7 @@ #ifdef DO_EVENTHANDLER if((isc->eh = EVENTHANDLER_REGISTER(shutdown_pre_sync, iscsi_shutdown, - sc, SHUTDOWN_PRI_DEFAULT-1)) == NULL) + sc, SHUTDOWN_PRI_DEFAULT - 1)) == NULL) xdebug("shutdown event registration failed\n"); #endif /*