diff --git a/sys/cam/cam.c b/sys/cam/cam.c index ce7dc81b3495..7d9d8602d009 100644 --- a/sys/cam/cam.c +++ b/sys/cam/cam.c @@ -1,645 +1,665 @@ /*- * Generic utility routines for the Common Access Method layer. * * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1997 Justin T. Gibbs. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #ifdef _KERNEL #include #include #include #include #else /* _KERNEL */ #include #include #include #include #endif /* _KERNEL */ #include #include #include #include #include #ifdef _KERNEL #include +#include #include #include FEATURE(scbus, "SCSI devices support"); #endif static int camstatusentrycomp(const void *key, const void *member); const struct cam_status_entry cam_status_table[] = { { CAM_REQ_INPROG, "CCB request is in progress" }, { CAM_REQ_CMP, "CCB request completed without error" }, { CAM_REQ_ABORTED, "CCB request aborted by the host" }, { CAM_UA_ABORT, "Unable to abort CCB request" }, { CAM_REQ_CMP_ERR, "CCB request completed with an error" }, { CAM_BUSY, "CAM subsystem is busy" }, { CAM_REQ_INVALID, "CCB request was invalid" }, { CAM_PATH_INVALID, "Supplied Path ID is invalid" }, { CAM_DEV_NOT_THERE, "Device Not Present" }, { CAM_UA_TERMIO, "Unable to terminate I/O CCB request" }, { CAM_SEL_TIMEOUT, "Selection Timeout" }, { CAM_CMD_TIMEOUT, "Command timeout" }, { CAM_SCSI_STATUS_ERROR, "SCSI Status Error" }, { CAM_MSG_REJECT_REC, "Message Reject Reveived" }, { CAM_SCSI_BUS_RESET, "SCSI Bus Reset Sent/Received" }, { CAM_UNCOR_PARITY, "Uncorrectable parity/CRC error" }, { CAM_AUTOSENSE_FAIL, "Auto-Sense Retrieval Failed" }, { CAM_NO_HBA, "No HBA Detected" }, { CAM_DATA_RUN_ERR, "Data Overrun error" }, { CAM_UNEXP_BUSFREE, "Unexpected Bus Free" }, { CAM_SEQUENCE_FAIL, "Target Bus Phase Sequence Failure" }, { CAM_CCB_LEN_ERR, "CCB length supplied is inadequate" }, { CAM_PROVIDE_FAIL, "Unable to provide requested capability" }, { CAM_BDR_SENT, "SCSI BDR Message Sent" }, { CAM_REQ_TERMIO, "CCB request terminated by the host" }, { CAM_UNREC_HBA_ERROR, "Unrecoverable Host Bus Adapter Error" }, { CAM_REQ_TOO_BIG, "The request was too large for this host" }, { CAM_REQUEUE_REQ, "Unconditionally Re-queue Request", }, { CAM_ATA_STATUS_ERROR, "ATA Status Error" }, { CAM_SCSI_IT_NEXUS_LOST,"Initiator/Target Nexus Lost" }, { CAM_SMP_STATUS_ERROR, "SMP Status Error" }, { CAM_IDE, "Initiator Detected Error Message Received" }, { CAM_RESRC_UNAVAIL, "Resource Unavailable" }, { CAM_UNACKED_EVENT, "Unacknowledged Event by Host" }, { CAM_MESSAGE_RECV, "Message Received in Host Target Mode" }, { CAM_INVALID_CDB, "Invalid CDB received in Host Target Mode" }, { CAM_LUN_INVALID, "Invalid Lun" }, { CAM_TID_INVALID, "Invalid Target ID" }, { CAM_FUNC_NOTAVAIL, "Function Not Available" }, { CAM_NO_NEXUS, "Nexus Not Established" }, { CAM_IID_INVALID, "Invalid Initiator ID" }, { CAM_CDB_RECVD, "CDB Received" }, { CAM_LUN_ALRDY_ENA, "LUN Already Enabled for Target Mode" }, { CAM_SCSI_BUSY, "SCSI Bus Busy" }, }; #ifdef _KERNEL SYSCTL_NODE(_kern, OID_AUTO, cam, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "CAM Subsystem"); #ifndef CAM_DEFAULT_SORT_IO_QUEUES #define CAM_DEFAULT_SORT_IO_QUEUES 1 #endif int cam_sort_io_queues = CAM_DEFAULT_SORT_IO_QUEUES; SYSCTL_INT(_kern_cam, OID_AUTO, sort_io_queues, CTLFLAG_RWTUN, &cam_sort_io_queues, 0, "Sort IO queues to try and optimise disk access patterns"); #endif void cam_strvis(u_int8_t *dst, const u_int8_t *src, int srclen, int dstlen) { cam_strvis_flag(dst, src, srclen, dstlen, CAM_STRVIS_FLAG_NONASCII_ESC); } void cam_strvis_flag(u_int8_t *dst, const u_int8_t *src, int srclen, int dstlen, uint32_t flags) { struct sbuf sb; sbuf_new(&sb, dst, dstlen, SBUF_FIXEDLEN); cam_strvis_sbuf(&sb, src, srclen, flags); sbuf_finish(&sb); } void cam_strvis_sbuf(struct sbuf *sb, const u_int8_t *src, int srclen, uint32_t flags) { /* Trim leading/trailing spaces, nulls. */ while (srclen > 0 && src[0] == ' ') src++, srclen--; while (srclen > 0 && (src[srclen-1] == ' ' || src[srclen-1] == '\0')) srclen--; while (srclen > 0) { if (*src < 0x20 || *src >= 0x80) { /* SCSI-II Specifies that these should never occur. */ /* non-printable character */ switch (flags & CAM_STRVIS_FLAG_NONASCII_MASK) { case CAM_STRVIS_FLAG_NONASCII_ESC: sbuf_printf(sb, "\\%c%c%c", ((*src & 0300) >> 6) + '0', ((*src & 0070) >> 3) + '0', ((*src & 0007) >> 0) + '0'); break; case CAM_STRVIS_FLAG_NONASCII_RAW: /* * If we run into a NUL, just transform it * into a space. */ if (*src != 0x00) sbuf_putc(sb, *src); else sbuf_putc(sb, ' '); break; case CAM_STRVIS_FLAG_NONASCII_SPC: sbuf_putc(sb, ' '); break; case CAM_STRVIS_FLAG_NONASCII_TRIM: default: break; } } else { /* normal character */ sbuf_putc(sb, *src); } src++; srclen--; } } /* * Compare string with pattern, returning 0 on match. * Short pattern matches trailing blanks in name, * Shell globbing rules apply: * matches 0 or more characters, * ? matchces one character, [...] denotes a set to match one char, * [^...] denotes a complimented set to match one character. * Spaces in str used to match anything in the pattern string * but was removed because it's a bug. No current patterns require * it, as far as I know, but it's impossible to know what drives * returned. * * Each '*' generates recursion, so keep the number of * in check. */ int cam_strmatch(const u_int8_t *str, const u_int8_t *pattern, int str_len) { while (*pattern != '\0' && str_len > 0) { if (*pattern == '*') { pattern++; if (*pattern == '\0') return (0); do { if (cam_strmatch(str, pattern, str_len) == 0) return (0); str++; str_len--; } while (str_len > 0); return (1); } else if (*pattern == '[') { int negate_range, ok; uint8_t pc = UCHAR_MAX; uint8_t sc; ok = 0; sc = *str++; str_len--; pattern++; if ((negate_range = (*pattern == '^')) != 0) pattern++; while ((*pattern != ']') && *pattern != '\0') { if (*pattern == '-') { if (pattern[1] == '\0') /* Bad pattern */ return (1); if (sc >= pc && sc <= pattern[1]) ok = 1; pattern++; } else if (*pattern == sc) ok = 1; pc = *pattern; pattern++; } if (ok == negate_range) return (1); pattern++; } else if (*pattern == '?') { /* * NB: || *str == ' ' of the old code is a bug and was * removed. If you add it back, keep this the last if * before the naked else */ pattern++; str++; str_len--; } else { if (*str != *pattern) return (1); pattern++; str++; str_len--; } } /* '*' is allowed to match nothing, so gobble it */ while (*pattern == '*') pattern++; if ( *pattern != '\0') { /* Pattern not fully consumed. Not a match */ return (1); } /* Eat trailing spaces, which get added by SAT */ while (str_len > 0 && *str == ' ') { str++; str_len--; } return (str_len); } caddr_t cam_quirkmatch(caddr_t target, caddr_t quirk_table, int num_entries, int entry_size, cam_quirkmatch_t *comp_func) { for (; num_entries > 0; num_entries--, quirk_table += entry_size) { if ((*comp_func)(target, quirk_table) == 0) return (quirk_table); } return (NULL); } const struct cam_status_entry* cam_fetch_status_entry(cam_status status) { status &= CAM_STATUS_MASK; return (bsearch(&status, &cam_status_table, nitems(cam_status_table), sizeof(*cam_status_table), camstatusentrycomp)); } static int camstatusentrycomp(const void *key, const void *member) { cam_status status; const struct cam_status_entry *table_entry; status = *(const cam_status *)key; table_entry = (const struct cam_status_entry *)member; return (status - table_entry->status_code); } #ifdef _KERNEL char * cam_error_string(union ccb *ccb, char *str, int str_len, cam_error_string_flags flags, cam_error_proto_flags proto_flags) #else /* !_KERNEL */ char * cam_error_string(struct cam_device *device, union ccb *ccb, char *str, int str_len, cam_error_string_flags flags, cam_error_proto_flags proto_flags) #endif /* _KERNEL/!_KERNEL */ { char path_str[64]; struct sbuf sb; if ((ccb == NULL) || (str == NULL) || (str_len <= 0)) return(NULL); if (flags == CAM_ESF_NONE) return(NULL); switch (ccb->ccb_h.func_code) { case XPT_ATA_IO: switch (proto_flags & CAM_EPF_LEVEL_MASK) { case CAM_EPF_NONE: break; case CAM_EPF_ALL: case CAM_EPF_NORMAL: proto_flags |= CAM_EAF_PRINT_RESULT; /* FALLTHROUGH */ case CAM_EPF_MINIMAL: proto_flags |= CAM_EAF_PRINT_STATUS; /* FALLTHROUGH */ default: break; } break; case XPT_SCSI_IO: switch (proto_flags & CAM_EPF_LEVEL_MASK) { case CAM_EPF_NONE: break; case CAM_EPF_ALL: case CAM_EPF_NORMAL: proto_flags |= CAM_ESF_PRINT_SENSE; /* FALLTHROUGH */ case CAM_EPF_MINIMAL: proto_flags |= CAM_ESF_PRINT_STATUS; /* FALLTHROUGH */ default: break; } break; case XPT_SMP_IO: switch (proto_flags & CAM_EPF_LEVEL_MASK) { case CAM_EPF_NONE: break; case CAM_EPF_ALL: proto_flags |= CAM_ESMF_PRINT_FULL_CMD; /* FALLTHROUGH */ case CAM_EPF_NORMAL: case CAM_EPF_MINIMAL: proto_flags |= CAM_ESMF_PRINT_STATUS; /* FALLTHROUGH */ default: break; } break; default: break; } #ifdef _KERNEL xpt_path_string(ccb->csio.ccb_h.path, path_str, sizeof(path_str)); #else /* !_KERNEL */ cam_path_string(device, path_str, sizeof(path_str)); #endif /* _KERNEL/!_KERNEL */ sbuf_new(&sb, str, str_len, 0); if (flags & CAM_ESF_COMMAND) { sbuf_cat(&sb, path_str); switch (ccb->ccb_h.func_code) { case XPT_ATA_IO: ata_command_sbuf(&ccb->ataio, &sb); break; case XPT_SCSI_IO: #ifdef _KERNEL scsi_command_string(&ccb->csio, &sb); #else /* !_KERNEL */ scsi_command_string(device, &ccb->csio, &sb); #endif /* _KERNEL/!_KERNEL */ break; case XPT_SMP_IO: smp_command_sbuf(&ccb->smpio, &sb, path_str, 79 - strlen(path_str), (proto_flags & CAM_ESMF_PRINT_FULL_CMD) ? 79 : 0); break; case XPT_NVME_IO: case XPT_NVME_ADMIN: nvme_command_sbuf(&ccb->nvmeio, &sb); break; default: sbuf_printf(&sb, "CAM func %#x", ccb->ccb_h.func_code); break; } sbuf_printf(&sb, "\n"); } if (flags & CAM_ESF_CAM_STATUS) { cam_status status; const struct cam_status_entry *entry; sbuf_cat(&sb, path_str); status = ccb->ccb_h.status & CAM_STATUS_MASK; entry = cam_fetch_status_entry(status); if (entry == NULL) sbuf_printf(&sb, "CAM status: Unknown (%#x)\n", ccb->ccb_h.status); else sbuf_printf(&sb, "CAM status: %s\n", entry->status_text); } if (flags & CAM_ESF_PROTO_STATUS) { switch (ccb->ccb_h.func_code) { case XPT_ATA_IO: if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_ATA_STATUS_ERROR) break; if (proto_flags & CAM_EAF_PRINT_STATUS) { sbuf_cat(&sb, path_str); ata_status_sbuf(&ccb->ataio, &sb); sbuf_printf(&sb, "\n"); } if (proto_flags & CAM_EAF_PRINT_RESULT) { sbuf_cat(&sb, path_str); sbuf_printf(&sb, "RES: "); ata_res_sbuf(&ccb->ataio.res, &sb); sbuf_printf(&sb, "\n"); } break; case XPT_SCSI_IO: if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_SCSI_STATUS_ERROR) break; if (proto_flags & CAM_ESF_PRINT_STATUS) { sbuf_cat(&sb, path_str); sbuf_printf(&sb, "SCSI status: %s\n", scsi_status_string(&ccb->csio)); } if ((proto_flags & CAM_ESF_PRINT_SENSE) && (ccb->csio.scsi_status == SCSI_STATUS_CHECK_COND) && (ccb->ccb_h.status & CAM_AUTOSNS_VALID)) { #ifdef _KERNEL scsi_sense_sbuf(&ccb->csio, &sb, SSS_FLAG_NONE); #else /* !_KERNEL */ scsi_sense_sbuf(device, &ccb->csio, &sb, SSS_FLAG_NONE); #endif /* _KERNEL/!_KERNEL */ } break; case XPT_SMP_IO: if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_SMP_STATUS_ERROR) break; if (proto_flags & CAM_ESF_PRINT_STATUS) { sbuf_cat(&sb, path_str); sbuf_printf(&sb, "SMP status: %s (%#x)\n", smp_error_desc(ccb->smpio.smp_response[2]), ccb->smpio.smp_response[2]); } /* There is no SMP equivalent to SCSI sense. */ break; default: break; } } sbuf_finish(&sb); return(sbuf_data(&sb)); } #ifdef _KERNEL void cam_error_print(union ccb *ccb, cam_error_string_flags flags, cam_error_proto_flags proto_flags) { char str[512]; printf("%s", cam_error_string(ccb, str, sizeof(str), flags, proto_flags)); } #else /* !_KERNEL */ void cam_error_print(struct cam_device *device, union ccb *ccb, cam_error_string_flags flags, cam_error_proto_flags proto_flags, FILE *ofile) { char str[512]; if ((device == NULL) || (ccb == NULL) || (ofile == NULL)) return; fprintf(ofile, "%s", cam_error_string(device, ccb, str, sizeof(str), flags, proto_flags)); } #endif /* _KERNEL/!_KERNEL */ /* * Common calculate geometry fuction * * Caller should set ccg->volume_size and block_size. * The extended parameter should be zero if extended translation * should not be used. */ void cam_calc_geometry(struct ccb_calc_geometry *ccg, int extended) { uint32_t size_mb, secs_per_cylinder; if (ccg->block_size == 0) { ccg->ccb_h.status = CAM_REQ_CMP_ERR; return; } size_mb = (1024L * 1024L) / ccg->block_size; if (size_mb == 0) { ccg->ccb_h.status = CAM_REQ_CMP_ERR; return; } size_mb = ccg->volume_size / size_mb; if (size_mb > 1024 && extended) { ccg->heads = 255; ccg->secs_per_track = 63; } else { ccg->heads = 64; ccg->secs_per_track = 32; } secs_per_cylinder = ccg->heads * ccg->secs_per_track; if (secs_per_cylinder == 0) { ccg->ccb_h.status = CAM_REQ_CMP_ERR; return; } ccg->cylinders = ccg->volume_size / secs_per_cylinder; ccg->ccb_h.status = CAM_REQ_CMP; } #ifdef _KERNEL struct memdesc memdesc_ccb(union ccb *ccb) { struct ccb_hdr *ccb_h; void *data_ptr; uint32_t dxfer_len; uint16_t sglist_cnt; ccb_h = &ccb->ccb_h; switch (ccb_h->func_code) { case XPT_SCSI_IO: { struct ccb_scsiio *csio; csio = &ccb->csio; data_ptr = csio->data_ptr; dxfer_len = csio->dxfer_len; sglist_cnt = csio->sglist_cnt; break; } case XPT_CONT_TARGET_IO: { struct ccb_scsiio *ctio; ctio = &ccb->ctio; data_ptr = ctio->data_ptr; dxfer_len = ctio->dxfer_len; sglist_cnt = ctio->sglist_cnt; break; } case XPT_ATA_IO: { struct ccb_ataio *ataio; ataio = &ccb->ataio; data_ptr = ataio->data_ptr; dxfer_len = ataio->dxfer_len; sglist_cnt = 0; break; } case XPT_NVME_IO: case XPT_NVME_ADMIN: { struct ccb_nvmeio *nvmeio; nvmeio = &ccb->nvmeio; data_ptr = nvmeio->data_ptr; dxfer_len = nvmeio->dxfer_len; sglist_cnt = nvmeio->sglist_cnt; break; } default: panic("%s: Unsupported func code %d", __func__, ccb_h->func_code); } switch ((ccb_h->flags & CAM_DATA_MASK)) { case CAM_DATA_VADDR: return (memdesc_vaddr(data_ptr, dxfer_len)); case CAM_DATA_PADDR: return (memdesc_paddr((vm_paddr_t)(uintptr_t)data_ptr, dxfer_len)); case CAM_DATA_SG: return (memdesc_vlist(data_ptr, sglist_cnt)); case CAM_DATA_SG_PADDR: return (memdesc_plist(data_ptr, sglist_cnt)); case CAM_DATA_BIO: return (memdesc_bio(data_ptr)); default: panic("%s: flags 0x%X unimplemented", __func__, ccb_h->flags); } } + +int +bus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb, + bus_dmamap_callback_t *callback, void *callback_arg, + int flags) +{ + struct ccb_hdr *ccb_h; + struct memdesc mem; + + ccb_h = &ccb->ccb_h; + if ((ccb_h->flags & CAM_DIR_MASK) == CAM_DIR_NONE) { + callback(callback_arg, NULL, 0, 0); + return (0); + } + + mem = memdesc_ccb(ccb); + return (bus_dmamap_load_mem(dmat, map, &mem, callback, callback_arg, + flags)); +} #endif diff --git a/sys/kern/subr_bus_dma.c b/sys/kern/subr_bus_dma.c index da7a2ee4cdc9..683b41d0047c 100644 --- a/sys/kern/subr_bus_dma.c +++ b/sys/kern/subr_bus_dma.c @@ -1,725 +1,706 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 EMC Corp. * All rights reserved. * * Copyright (c) 1997, 1998 Justin T. Gibbs. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_bus.h" #include "opt_iommu.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Convenience function for manipulating driver locks from busdma (during * busdma_swi, for example). */ void busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) { struct mtx *dmtx; dmtx = (struct mtx *)arg; switch (op) { case BUS_DMA_LOCK: mtx_lock(dmtx); break; case BUS_DMA_UNLOCK: mtx_unlock(dmtx); break; default: panic("Unknown operation 0x%x for busdma_lock_mutex!", op); } } /* * dflt_lock should never get called. It gets put into the dma tag when * lockfunc == NULL, which is only valid if the maps that are associated * with the tag are meant to never be deferred. * * XXX Should have a way to identify which driver is responsible here. */ void _busdma_dflt_lock(void *arg, bus_dma_lock_op_t op) { panic("driver error: _bus_dma_dflt_lock called"); } /* * Load up data starting at offset within a region specified by a * list of virtual address ranges until either length or the region * are exhausted. */ static int _bus_dmamap_load_vlist(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dma_segment_t *list, int sglist_cnt, struct pmap *pmap, int *nsegs, int flags, size_t offset, size_t length) { int error; error = 0; for (; sglist_cnt > 0 && length != 0; sglist_cnt--, list++) { char *addr; size_t ds_len; KASSERT((offset < list->ds_len), ("Invalid mid-segment offset")); addr = (char *)(uintptr_t)list->ds_addr + offset; ds_len = list->ds_len - offset; offset = 0; if (ds_len > length) ds_len = length; length -= ds_len; KASSERT((ds_len != 0), ("Segment length is zero")); error = _bus_dmamap_load_buffer(dmat, map, addr, ds_len, pmap, flags, NULL, nsegs); if (error) break; } return (error); } /* * Load a list of physical addresses. */ static int _bus_dmamap_load_plist(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dma_segment_t *list, int sglist_cnt, int *nsegs, int flags) { int error; error = 0; for (; sglist_cnt > 0; sglist_cnt--, list++) { error = _bus_dmamap_load_phys(dmat, map, (vm_paddr_t)list->ds_addr, list->ds_len, flags, NULL, nsegs); if (error) break; } return (error); } /* * Load an unmapped mbuf */ static int _bus_dmamap_load_mbuf_epg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m, bus_dma_segment_t *segs, int *nsegs, int flags) { int error, i, off, len, pglen, pgoff, seglen, segoff; M_ASSERTEXTPG(m); len = m->m_len; error = 0; /* Skip over any data removed from the front. */ off = mtod(m, vm_offset_t); if (m->m_epg_hdrlen != 0) { if (off >= m->m_epg_hdrlen) { off -= m->m_epg_hdrlen; } else { seglen = m->m_epg_hdrlen - off; segoff = off; seglen = min(seglen, len); off = 0; len -= seglen; error = _bus_dmamap_load_buffer(dmat, map, &m->m_epg_hdr[segoff], seglen, kernel_pmap, flags, segs, nsegs); } } pgoff = m->m_epg_1st_off; for (i = 0; i < m->m_epg_npgs && error == 0 && len > 0; i++) { pglen = m_epg_pagelen(m, i, pgoff); if (off >= pglen) { off -= pglen; pgoff = 0; continue; } seglen = pglen - off; segoff = pgoff + off; off = 0; seglen = min(seglen, len); len -= seglen; error = _bus_dmamap_load_phys(dmat, map, m->m_epg_pa[i] + segoff, seglen, flags, segs, nsegs); pgoff = 0; }; if (len != 0 && error == 0) { KASSERT((off + len) <= m->m_epg_trllen, ("off + len > trail (%d + %d > %d)", off, len, m->m_epg_trllen)); error = _bus_dmamap_load_buffer(dmat, map, &m->m_epg_trail[off], len, kernel_pmap, flags, segs, nsegs); } return (error); } /* * Load a single mbuf. */ static int _bus_dmamap_load_single_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m, bus_dma_segment_t *segs, int *nsegs, int flags) { int error; error = 0; if ((m->m_flags & M_EXTPG) != 0) error = _bus_dmamap_load_mbuf_epg(dmat, map, m, segs, nsegs, flags); else error = _bus_dmamap_load_buffer(dmat, map, m->m_data, m->m_len, kernel_pmap, flags | BUS_DMA_LOAD_MBUF, segs, nsegs); CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", __func__, dmat, flags, error, *nsegs); return (error); } /* * Load an mbuf chain. */ static int _bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, int flags) { struct mbuf *m; int error; error = 0; for (m = m0; m != NULL && error == 0; m = m->m_next) { if (m->m_len > 0) { if ((m->m_flags & M_EXTPG) != 0) error = _bus_dmamap_load_mbuf_epg(dmat, map, m, segs, nsegs, flags); else error = _bus_dmamap_load_buffer(dmat, map, m->m_data, m->m_len, kernel_pmap, flags | BUS_DMA_LOAD_MBUF, segs, nsegs); } } CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", __func__, dmat, flags, error, *nsegs); return (error); } int bus_dmamap_load_ma_triv(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs, int *segp) { vm_paddr_t paddr; bus_size_t len; int error, i; error = 0; for (i = 0; tlen > 0; i++, tlen -= len) { len = min(PAGE_SIZE - ma_offs, tlen); paddr = VM_PAGE_TO_PHYS(ma[i]) + ma_offs; error = _bus_dmamap_load_phys(dmat, map, paddr, len, flags, segs, segp); if (error != 0) break; ma_offs = 0; } return (error); } /* * Load a uio. */ static int _bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, int *nsegs, int flags) { bus_size_t resid; bus_size_t minlen; struct iovec *iov; pmap_t pmap; caddr_t addr; int error, i; if (uio->uio_segflg == UIO_USERSPACE) { KASSERT(uio->uio_td != NULL, ("bus_dmamap_load_uio: USERSPACE but no proc")); pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); } else pmap = kernel_pmap; resid = uio->uio_resid; iov = uio->uio_iov; error = 0; for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { /* * Now at the first iovec to load. Load each iovec * until we have exhausted the residual count. */ addr = (caddr_t) iov[i].iov_base; minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len; if (minlen > 0) { error = _bus_dmamap_load_buffer(dmat, map, addr, minlen, pmap, flags, NULL, nsegs); resid -= minlen; } } return (error); } /* * Map the buffer buf into bus space using the dmamap map. */ int bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, bus_size_t buflen, bus_dmamap_callback_t *callback, void *callback_arg, int flags) { bus_dma_segment_t *segs; struct memdesc mem; int error; int nsegs; #ifdef KMSAN mem = memdesc_vaddr(buf, buflen); _bus_dmamap_load_kmsan(dmat, map, &mem); #endif if ((flags & BUS_DMA_NOWAIT) == 0) { mem = memdesc_vaddr(buf, buflen); _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); } nsegs = -1; error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, kernel_pmap, flags, NULL, &nsegs); nsegs++; CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", __func__, dmat, flags, error, nsegs); if (error == EINPROGRESS) return (error); segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); if (error) (*callback)(callback_arg, segs, 0, error); else (*callback)(callback_arg, segs, nsegs, 0); /* * Return ENOMEM to the caller so that it can pass it up the stack. * This error only happens when NOWAIT is set, so deferral is disabled. */ if (error == ENOMEM) return (error); return (0); } int bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, bus_dmamap_callback2_t *callback, void *callback_arg, int flags) { bus_dma_segment_t *segs; int nsegs, error; M_ASSERTPKTHDR(m0); #ifdef KMSAN struct memdesc mem = memdesc_mbuf(m0); _bus_dmamap_load_kmsan(dmat, map, &mem); #endif flags |= BUS_DMA_NOWAIT; nsegs = -1; error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, NULL, &nsegs, flags); ++nsegs; segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); if (error) (*callback)(callback_arg, segs, 0, 0, error); else (*callback)(callback_arg, segs, nsegs, m0->m_pkthdr.len, error); CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", __func__, dmat, flags, error, nsegs); return (error); } int bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, int flags) { int error; #ifdef KMSAN struct memdesc mem = memdesc_mbuf(m0); _bus_dmamap_load_kmsan(dmat, map, &mem); #endif flags |= BUS_DMA_NOWAIT; *nsegs = -1; error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags); ++*nsegs; _bus_dmamap_complete(dmat, map, segs, *nsegs, error); return (error); } int bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, bus_dmamap_callback2_t *callback, void *callback_arg, int flags) { bus_dma_segment_t *segs; int nsegs, error; #ifdef KMSAN struct memdesc mem = memdesc_uio(uio); _bus_dmamap_load_kmsan(dmat, map, &mem); #endif flags |= BUS_DMA_NOWAIT; nsegs = -1; error = _bus_dmamap_load_uio(dmat, map, uio, &nsegs, flags); nsegs++; segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); if (error) (*callback)(callback_arg, segs, 0, 0, error); else (*callback)(callback_arg, segs, nsegs, uio->uio_resid, error); CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", __func__, dmat, flags, error, nsegs); return (error); } -int -bus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb, - bus_dmamap_callback_t *callback, void *callback_arg, - int flags) -{ - struct ccb_hdr *ccb_h; - struct memdesc mem; - - ccb_h = &ccb->ccb_h; - if ((ccb_h->flags & CAM_DIR_MASK) == CAM_DIR_NONE) { - callback(callback_arg, NULL, 0, 0); - return (0); - } - - mem = memdesc_ccb(ccb); - return (bus_dmamap_load_mem(dmat, map, &mem, callback, callback_arg, - flags)); -} - int bus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bio, bus_dmamap_callback_t *callback, void *callback_arg, int flags) { struct memdesc mem; mem = memdesc_bio(bio); return (bus_dmamap_load_mem(dmat, map, &mem, callback, callback_arg, flags)); } int bus_dmamap_load_mem(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg, int flags) { bus_dma_segment_t *segs; int error; int nsegs; #ifdef KMSAN _bus_dmamap_load_kmsan(dmat, map, mem); #endif if ((flags & BUS_DMA_NOWAIT) == 0) _bus_dmamap_waitok(dmat, map, mem, callback, callback_arg); nsegs = -1; error = 0; switch (mem->md_type) { case MEMDESC_VADDR: error = _bus_dmamap_load_buffer(dmat, map, mem->u.md_vaddr, mem->md_len, kernel_pmap, flags, NULL, &nsegs); break; case MEMDESC_PADDR: error = _bus_dmamap_load_phys(dmat, map, mem->u.md_paddr, mem->md_len, flags, NULL, &nsegs); break; case MEMDESC_VLIST: error = _bus_dmamap_load_vlist(dmat, map, mem->u.md_list, mem->md_nseg, kernel_pmap, &nsegs, flags, 0, SIZE_T_MAX); break; case MEMDESC_PLIST: error = _bus_dmamap_load_plist(dmat, map, mem->u.md_list, mem->md_nseg, &nsegs, flags); break; case MEMDESC_UIO: error = _bus_dmamap_load_uio(dmat, map, mem->u.md_uio, &nsegs, flags); break; case MEMDESC_MBUF: error = _bus_dmamap_load_mbuf_sg(dmat, map, mem->u.md_mbuf, NULL, &nsegs, flags); break; case MEMDESC_VMPAGES: error = _bus_dmamap_load_ma(dmat, map, mem->u.md_ma, mem->md_len, mem->md_offset, flags, NULL, &nsegs); break; } nsegs++; CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", __func__, dmat, flags, error, nsegs); if (error == EINPROGRESS) return (error); segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); if (error) (*callback)(callback_arg, segs, 0, error); else (*callback)(callback_arg, segs, nsegs, 0); /* * Return ENOMEM to the caller so that it can pass it up the stack. * This error only happens when NOWAIT is set, so deferral is disabled. */ if (error == ENOMEM) return (error); return (0); } int bus_dmamap_load_crp_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, struct crypto_buffer *cb, bus_dmamap_callback_t *callback, void *callback_arg, int flags) { bus_dma_segment_t *segs; int error; int nsegs; flags |= BUS_DMA_NOWAIT; nsegs = -1; error = 0; switch (cb->cb_type) { case CRYPTO_BUF_CONTIG: error = _bus_dmamap_load_buffer(dmat, map, cb->cb_buf, cb->cb_buf_len, kernel_pmap, flags, NULL, &nsegs); break; case CRYPTO_BUF_MBUF: error = _bus_dmamap_load_mbuf_sg(dmat, map, cb->cb_mbuf, NULL, &nsegs, flags); break; case CRYPTO_BUF_SINGLE_MBUF: error = _bus_dmamap_load_single_mbuf(dmat, map, cb->cb_mbuf, NULL, &nsegs, flags); break; case CRYPTO_BUF_UIO: error = _bus_dmamap_load_uio(dmat, map, cb->cb_uio, &nsegs, flags); break; case CRYPTO_BUF_VMPAGE: error = _bus_dmamap_load_ma(dmat, map, cb->cb_vm_page, cb->cb_vm_page_len, cb->cb_vm_page_offset, flags, NULL, &nsegs); break; default: error = EINVAL; } nsegs++; CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", __func__, dmat, flags, error, nsegs); if (error == EINPROGRESS) return (error); segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); if (error) (*callback)(callback_arg, segs, 0, error); else (*callback)(callback_arg, segs, nsegs, 0); /* * Return ENOMEM to the caller so that it can pass it up the stack. * This error only happens when NOWAIT is set, so deferral is disabled. */ if (error == ENOMEM) return (error); return (0); } int bus_dmamap_load_crp(bus_dma_tag_t dmat, bus_dmamap_t map, struct cryptop *crp, bus_dmamap_callback_t *callback, void *callback_arg, int flags) { return (bus_dmamap_load_crp_buffer(dmat, map, &crp->crp_buf, callback, callback_arg, flags)); } void bus_dma_template_init(bus_dma_template_t *t, bus_dma_tag_t parent) { if (t == NULL) return; t->parent = parent; t->alignment = 1; t->boundary = 0; t->lowaddr = t->highaddr = BUS_SPACE_MAXADDR; t->maxsize = t->maxsegsize = BUS_SPACE_MAXSIZE; t->nsegments = BUS_SPACE_UNRESTRICTED; t->lockfunc = NULL; t->lockfuncarg = NULL; t->flags = 0; } int bus_dma_template_tag(bus_dma_template_t *t, bus_dma_tag_t *dmat) { if (t == NULL || dmat == NULL) return (EINVAL); return (bus_dma_tag_create(t->parent, t->alignment, t->boundary, t->lowaddr, t->highaddr, NULL, NULL, t->maxsize, t->nsegments, t->maxsegsize, t->flags, t->lockfunc, t->lockfuncarg, dmat)); } void bus_dma_template_fill(bus_dma_template_t *t, bus_dma_param_t *kv, u_int count) { bus_dma_param_t *pkv; while (count) { pkv = &kv[--count]; switch (pkv->key) { case BD_PARAM_PARENT: t->parent = pkv->ptr; break; case BD_PARAM_ALIGNMENT: t->alignment = pkv->num; break; case BD_PARAM_BOUNDARY: t->boundary = pkv->num; break; case BD_PARAM_LOWADDR: t->lowaddr = pkv->pa; break; case BD_PARAM_HIGHADDR: t->highaddr = pkv->pa; break; case BD_PARAM_MAXSIZE: t->maxsize = pkv->num; break; case BD_PARAM_NSEGMENTS: t->nsegments = pkv->num; break; case BD_PARAM_MAXSEGSIZE: t->maxsegsize = pkv->num; break; case BD_PARAM_FLAGS: t->flags = pkv->num; break; case BD_PARAM_LOCKFUNC: t->lockfunc = pkv->ptr; break; case BD_PARAM_LOCKFUNCARG: t->lockfuncarg = pkv->ptr; break; case BD_PARAM_NAME: t->name = pkv->ptr; break; case BD_PARAM_INVALID: default: KASSERT(0, ("Invalid key %d\n", pkv->key)); break; } } return; } #ifndef IOMMU bool bus_dma_iommu_set_buswide(device_t dev); int bus_dma_iommu_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t start, vm_size_t length, int flags); bool bus_dma_iommu_set_buswide(device_t dev) { return (false); } int bus_dma_iommu_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t start, vm_size_t length, int flags) { return (0); } #endif