Index: sys/dev/iscsi/iscsi.c =================================================================== --- sys/dev/iscsi/iscsi.c +++ sys/dev/iscsi/iscsi.c @@ -33,6 +33,7 @@ __FBSDID("$FreeBSD$"); #include +#include #include #include #include @@ -48,6 +49,7 @@ #include #include #include +#include #include #include @@ -1062,6 +1064,63 @@ icl_pdu_free(response); } +static void +iscsi_pdu_get_data_bio(struct icl_pdu *response, size_t pdu_offset, + struct bio *bp, size_t oreceived, size_t data_segment_len) +{ + vm_offset_t vaddr; + size_t page_offset, todo; + boolean_t mapped; + int i; + + MPASS(bp->bio_flags & BIO_UNMAPPED); + if (oreceived < PAGE_SIZE - bp->bio_ma_offset) { + page_offset = bp->bio_ma_offset + oreceived; + oreceived = 0; + i = 0; + } else { + oreceived -= PAGE_SIZE - bp->bio_ma_offset; + for (i = 1; oreceived >= PAGE_SIZE; i++) + oreceived -= PAGE_SIZE; + page_offset = oreceived; + } + + while (data_segment_len > 0) { + todo = MIN(data_segment_len, PAGE_SIZE - page_offset); + + mapped = pmap_map_io_transient(bp->bio_ma + i, &vaddr, 1, + FALSE); + icl_pdu_get_data(response, pdu_offset, (char *)vaddr + + page_offset, todo); + if (__predict_false(mapped)) + pmap_unmap_io_transient(bp->bio_ma + 1, &vaddr, 1, + FALSE); + + page_offset = 0; + pdu_offset += todo; + data_segment_len -= todo; + i++; + } +} + +static void +iscsi_pdu_get_data_csio(struct icl_pdu *response, size_t pdu_offset, + struct ccb_scsiio *csio, size_t oreceived, size_t data_segment_len) +{ + switch (csio->ccb_h.flags & CAM_DATA_MASK) { + case CAM_DATA_BIO: + iscsi_pdu_get_data_bio(response, pdu_offset, + (struct bio *)csio->data_ptr, oreceived, data_segment_len); + break; + case CAM_DATA_VADDR: + icl_pdu_get_data(response, pdu_offset, + csio->data_ptr + oreceived, data_segment_len); + break; + default: + __assert_unreachable(); + } +} + static void iscsi_pdu_handle_data_in(struct icl_pdu *response) { @@ -1140,7 +1199,7 @@ iscsi_outstanding_remove(is, io); ISCSI_SESSION_UNLOCK(is); - icl_pdu_get_data(response, 0, csio->data_ptr + oreceived, data_segment_len); + iscsi_pdu_get_data_csio(response, 0, csio, oreceived, data_segment_len); /* * XXX: Check F. @@ -1191,6 +1250,63 @@ icl_pdu_free(response); } +static int +iscsi_pdu_append_data_bio(struct icl_pdu *request, struct bio *bp, size_t off, + size_t len, int how) +{ + vm_offset_t vaddr; + size_t page_offset, todo; + boolean_t mapped; + int error, i; + + MPASS(bp->bio_flags & BIO_UNMAPPED); + if (off < PAGE_SIZE - bp->bio_ma_offset) { + page_offset = bp->bio_ma_offset + off; + off = 0; + i = 0; + } else { + off -= PAGE_SIZE - bp->bio_ma_offset; + for (i = 1; off >= PAGE_SIZE; i++) + off -= PAGE_SIZE; + page_offset = off; + } + + while (len > 0) { + todo = MIN(len, PAGE_SIZE - page_offset); + + mapped = pmap_map_io_transient(bp->bio_ma + i, &vaddr, 1, + FALSE); + error = icl_pdu_append_data(request, (char *)vaddr + + page_offset, todo, how); + if (__predict_false(mapped)) + pmap_unmap_io_transient(bp->bio_ma + 1, &vaddr, 1, + FALSE); + if (error) + return (error); + + page_offset = 0; + len -= todo; + i++; + } + return (0); +} + +static int +iscsi_pdu_append_data_csio(struct icl_pdu *request, struct ccb_scsiio *csio, + size_t off, size_t len, int how) +{ + switch (csio->ccb_h.flags & CAM_DATA_MASK) { + case CAM_DATA_BIO: + return (iscsi_pdu_append_data_bio(request, + (struct bio *)csio->data_ptr, off, len, how)); + case CAM_DATA_VADDR: + return (icl_pdu_append_data(request, csio->data_ptr + off, len, + how)); + default: + __assert_unreachable(); + } +} + static void iscsi_pdu_handle_r2t(struct icl_pdu *response) { @@ -1285,7 +1401,7 @@ bhsr2t->bhsr2t_target_transfer_tag; bhsdo->bhsdo_datasn = htonl(datasn); bhsdo->bhsdo_buffer_offset = htonl(off); - error = icl_pdu_append_data(request, csio->data_ptr + off, len, + error = iscsi_pdu_append_data_csio(request, csio, off, len, M_NOWAIT); if (error != 0) { ISCSI_SESSION_WARN(is, "failed to allocate memory; " @@ -2392,7 +2508,8 @@ len = is->is_conn->ic_max_send_data_segment_length; } - error = icl_pdu_append_data(request, csio->data_ptr, len, M_NOWAIT); + error = iscsi_pdu_append_data_csio(request, csio, 0, len, + M_NOWAIT); if (error != 0) { iscsi_outstanding_remove(is, io); icl_pdu_free(request);