diff --git a/share/man/man9/bus_dma.9 b/share/man/man9/bus_dma.9 index 832ddb4daa22..b644eeb2a476 100644 --- a/share/man/man9/bus_dma.9 +++ b/share/man/man9/bus_dma.9 @@ -1,1341 +1,1333 @@ .\" Copyright (c) 2002, 2003 Hiten M. Pandya. .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions, and the following disclaimer, .\" without modification, immediately at the beginning of the file. .\" 2. The name of the author may not be used to endorse or promote products .\" derived from this software without specific prior written permission. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR, CONTRIBUTORS OR THE .\" VOICES IN HITEN PANDYA'S HEAD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, .\" SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED .\" TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR .\" PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF .\" LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING .\" NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS .\" SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .\" .\" Copyright (c) 1996, 1997, 1998, 2001 The NetBSD Foundation, Inc. .\" All rights reserved. .\" .\" This code is derived from software contributed to The NetBSD Foundation .\" by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, .\" NASA Ames Research Center. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS .\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED .\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR .\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS .\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR .\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF .\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS .\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN .\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) .\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE .\" POSSIBILITY OF SUCH DAMAGE. .\" $NetBSD: bus_dma.9,v 1.25 2002/10/14 13:43:16 wiz Exp $ .\" .Dd May 25, 2020 .Dt BUS_DMA 9 .Os .Sh NAME .Nm bus_dma , .Nm bus_dma_tag_create , .Nm bus_dma_tag_destroy , .Nm bus_dma_template_init , .Nm bus_dma_template_tag , .Nm bus_dma_template_clone , .Nm bus_dma_template_fill , .Nm BUS_DMA_TEMPLATE_FILL , .Nm bus_dmamap_create , .Nm bus_dmamap_destroy , .Nm bus_dmamap_load , .Nm bus_dmamap_load_bio , .Nm bus_dmamap_load_ccb , .Nm bus_dmamap_load_crp , .Nm bus_dmamap_load_crp_buffer , .Nm bus_dmamap_load_mbuf , .Nm bus_dmamap_load_mbuf_sg , .Nm bus_dmamap_load_uio , .Nm bus_dmamap_unload , .Nm bus_dmamap_sync , .Nm bus_dmamem_alloc , .Nm bus_dmamem_free .Nd Bus and Machine Independent DMA Mapping Interface .Sh SYNOPSIS .In machine/bus.h .Ft int .Fn bus_dma_tag_create "bus_dma_tag_t parent" "bus_size_t alignment" \ "bus_addr_t boundary" "bus_addr_t lowaddr" "bus_addr_t highaddr" \ "bus_dma_filter_t *filtfunc" "void *filtfuncarg" "bus_size_t maxsize" \ "int nsegments" "bus_size_t maxsegsz" "int flags" "bus_dma_lock_t *lockfunc" \ "void *lockfuncarg" "bus_dma_tag_t *dmat" .Ft int .Fn bus_dma_tag_destroy "bus_dma_tag_t dmat" .Ft void .Fo bus_dma_template_init .Fa "bus_dma_template_t *template" .Fa "bus_dma_tag_t parent" .Fc .Ft int .Fo bus_dma_template_tag .Fa "bus_dma_template_t *template" .Fa "bus_dma_tag_t *dmat" .Fc .Ft void .Fo bus_dma_template_clone .Fa "bus_dma_template_t *template" .Fa "bus_dma_tag_t dmat" .Fc .Ft void .Fo bus_dma_template_fill .Fa "bus_dma_template_t *template" .Fa "bus_dma_param_t params[]" .Fa "u_int count" .Fc .Fo BUS_DMA_TEMPLATE_FILL .Fa "bus_dma_template_t *template" .Fa "bus_dma_param_t param ..." .Fc .Ft int .Fn bus_dmamap_create "bus_dma_tag_t dmat" "int flags" "bus_dmamap_t *mapp" .Ft int .Fn bus_dmamap_destroy "bus_dma_tag_t dmat" "bus_dmamap_t map" .Ft int .Fn bus_dmamap_load "bus_dma_tag_t dmat" "bus_dmamap_t map" "void *buf" \ "bus_size_t buflen" "bus_dmamap_callback_t *callback" "void *callback_arg" \ "int flags" .Ft int .Fn bus_dmamap_load_bio "bus_dma_tag_t dmat" "bus_dmamap_t map" \ "struct bio *bio" "bus_dmamap_callback_t *callback" "void *callback_arg" \ "int flags" .Ft int .Fn bus_dmamap_load_ccb "bus_dma_tag_t dmat" "bus_dmamap_t map" \ "union ccb *ccb" "bus_dmamap_callback_t *callback" "void *callback_arg" \ "int flags" .Ft int .Fn bus_dmamap_load_crp "bus_dma_tag_t dmat" "bus_dmamap_t map" \ "struct crypto *crp" "bus_dmamap_callback_t *callback" "void *callback_arg" \ "int flags" .Ft int .Fn bus_dmamap_load_crp_buffer "bus_dma_tag_t dmat" "bus_dmamap_t map" \ "struct crypto_buffer *cb" "bus_dmamap_callback_t *callback" \ "void *callback_arg" "int flags" .Ft int .Fn bus_dmamap_load_mbuf "bus_dma_tag_t dmat" "bus_dmamap_t map" \ "struct mbuf *mbuf" "bus_dmamap_callback2_t *callback" "void *callback_arg" \ "int flags" .Ft int .Fn bus_dmamap_load_mbuf_sg "bus_dma_tag_t dmat" "bus_dmamap_t map" \ "struct mbuf *mbuf" "bus_dma_segment_t *segs" "int *nsegs" "int flags" .Ft int .Fn bus_dmamap_load_uio "bus_dma_tag_t dmat" "bus_dmamap_t map" \ "struct uio *uio" "bus_dmamap_callback2_t *callback" "void *callback_arg" \ "int flags" .Ft void .Fn bus_dmamap_unload "bus_dma_tag_t dmat" "bus_dmamap_t map" .Ft void .Fn bus_dmamap_sync "bus_dma_tag_t dmat" "bus_dmamap_t map" \ "op" .Ft int .Fn bus_dmamem_alloc "bus_dma_tag_t dmat" "void **vaddr" \ "int flags" "bus_dmamap_t *mapp" .Ft void .Fn bus_dmamem_free "bus_dma_tag_t dmat" "void *vaddr" \ "bus_dmamap_t map" .Sh DESCRIPTION Direct Memory Access (DMA) is a method of transferring data without involving the CPU, thus providing higher performance. A DMA transaction can be achieved between device to memory, device to device, or memory to memory. .Pp The .Nm API is a bus, device, and machine-independent (MI) interface to DMA mechanisms. It provides the client with flexibility and simplicity by abstracting machine dependent issues like setting up DMA mappings, handling cache issues, bus specific features and limitations. .Sh OVERVIEW A tag structure .Vt ( bus_dma_tag_t ) is used to describe the properties of a group of related DMA transactions. One way to view this is that a tag describes the limitations of a DMA engine. For example, if a DMA engine in a device is limited to 32-bit addresses, that limitation is specified by a parameter when creating the tag for that device. Similarly, a tag can be marked as requiring buffers whose addresses are aligned to a specific boundary. .Pp Some devices may require multiple tags to describe DMA transactions with differing properties. For example, a device might require 16-byte alignment of its descriptor ring while permitting arbitrary alignment of I/O buffers. In this case, the driver must create one tag for the descriptor ring and a separate tag for I/O buffers. If a device has restrictions that are common to all DMA transactions in addition to restrictions that differ between unrelated groups of transactions, the driver can first create a .Dq parent tag that decribes the common restrictions. The per-group tags can then inherit these restrictions from this .Dq parent tag rather than having to list them explicitly when creating the per-group tags. .Pp A mapping structure .Vt ( bus_dmamap_t ) represents a mapping of a memory region for DMA. On systems with I/O MMUs, the mapping structure tracks any I/O MMU entries used by a request. For DMA requests that require bounce pages, the mapping tracks the bounce pages used. .Pp To prepare for one or more DMA transactions, a mapping must be bound to a memory region by calling one of the .Fn bus_dmamap_load functions. These functions configure the mapping which can include programming entries in an I/O MMU and/or allocating bounce pages. An output of these functions (either directly or indirectly by invoking a callback routine) is the list of scatter/gather address ranges a consumer can pass to a DMA engine to access the memory region. When a mapping is no longer needed, the mapping must be unloaded via .Fn bus_dmamap_unload . .Pp Before and after each DMA transaction, .Fn bus_dmamap_sync must be used to ensure that the correct data is used by the DMA engine and the CPU. If a mapping uses bounce pages, the sync operations copy data between the bounce pages and the memory region bound to the mapping. Sync operations also handle architecture-specific details such as CPU cache flushing and CPU memory operation ordering. .Sh STATIC VS DYNAMIC .Nm handles two types of DMA transactions: static and dynamic. Static transactions are used with a long-lived memory region that is reused for many transactions such as a descriptor ring. Dynamic transactions are used for transfers to or from transient buffers such as I/O buffers holding a network packet or disk block. Each transaction type uses a different subset of the .Nm API. .Ss Static Transactions Static transactions use memory regions allocated by .Nm . Each static memory region is allocated by calling .Fn bus_dmamem_alloc . This function requires a valid tag describing the properties of the DMA transactions to this region such as alignment or address restrictions. Multiple regions can share a single tag if they share the same restrictions. .Pp .Fn bus_dmamem_alloc allocates a memory region along with a mapping object. The associated tag, memory region, and mapping object must then be passed to .Fn bus_dmamap_load to bind the mapping to the allocated region and obtain the scatter/gather list. .Pp It is expected that .Fn bus_dmamem_alloc will attempt to allocate memory requiring less expensive sync operations (for example, implementations should not allocate regions requiring bounce pages), but sync operations should still be used. For example, a driver should use .Fn bus_dmamap_sync in an interrupt handler before reading descriptor ring entries written by the device prior to the interrupt. .Pp When a consumer is finished with a memory region, it should unload the mapping via .Fn bus_dmamap_unload and then release the memory region and mapping object via .Fn bus_dmamem_free . .Ss Dynamic Transactions Dynamic transactions map memory regions provided by other parts of the system. A tag must be created via .Fn bus_dma_tag_create to describe the DMA transactions to and from these memory regions, and a pool of mapping objects must be allocated via .Fn bus_dmamap_create to track the mappings of any in-flight transactions. .Pp When a consumer wishes to schedule a transaction for a memory region, the consumer must first obtain an unused mapping object from its pool of mapping objects. The memory region must be bound to the mapping object via one of the .Fn bus_dmamap_load functions. Before scheduling the transaction, the consumer should sync the memory region via .Fn bus_dmamap_sync with one or more of the .Dq PRE flags. After the transaction has completed, the consumer should sync the memory region via .Fn bus_dmamap_sync with one or more of the .Dq POST flags. The mapping can then be unloaded via .Fn bus_dmamap_unload , and the mapping object can be returned to the pool of unused mapping objects. .Pp When a consumer is no longer scheduling DMA transactions, the mapping objects should be freed via .Fn bus_dmamap_destroy , and the tag should be freed via .Fn bus_dma_tag_destroy . .Sh STRUCTURES AND TYPES .Bl -tag -width indent .It Vt bus_dma_tag_t A machine-dependent (MD) opaque type that describes the characteristics of a group of DMA transactions. DMA tags are organized into a hierarchy, with each child tag inheriting the restrictions of its parent. This allows all devices along the path of DMA transactions to contribute to the constraints of those transactions. .It Vt bus_dma_template_t A template is a structure for creating a .Fa bus_dma_tag_t from a set of defaults. Once initialized with .Fn bus_dma_template_init , a driver can over-ride individual fields to suit its needs. The following fields start with the indicated default values: .Bd -literal alignment 1 boundary 0 lowaddr BUS_SPACE_MAXADDR highaddr BUS_SPACE_MAXADDR maxsize BUS_SPACE_MAXSIZE nsegments BUS_SPACE_UNRESTRICTED maxsegsize BUS_SPACE_MAXSIZE flags 0 lockfunc NULL lockfuncarg NULL .Ed .Pp Descriptions of each field are documented with .Fn bus_dma_tag_create . Note that the .Fa filtfunc and .Fa filtfuncarg attributes of the DMA tag are not supported with templates. .It Vt bus_dma_filter_t Client specified address filter having the format: .Bl -tag -width indent .It Ft int .Fn "client_filter" "void *filtarg" "bus_addr_t testaddr" .El .Pp Address filters can be specified during tag creation to allow for devices whose DMA address restrictions cannot be specified by a single window. The .Fa filtarg argument is specified by the client during tag creation to be passed to all invocations of the callback. The .Fa testaddr argument contains a potential starting address of a DMA mapping. The filter function operates on the set of addresses from .Fa testaddr to .Ql trunc_page(testaddr) + PAGE_SIZE - 1 , inclusive. The filter function should return zero if any mapping in this range can be accommodated by the device and non-zero otherwise. .Pp -.Em Note: The use of filters is deprecated. Proper operation is not guaranteed. +.Em Note: The use of filters is no longer supported and will result in an error. .It Vt bus_dma_segment_t A machine-dependent type that describes individual DMA segments. It contains the following fields: .Bd -literal bus_addr_t ds_addr; bus_size_t ds_len; .Ed .Pp The .Fa ds_addr field contains the device visible address of the DMA segment, and .Fa ds_len contains the length of the DMA segment. Although the DMA segments returned by a mapping call will adhere to all restrictions necessary for a successful DMA operation, some conversion (e.g.\& a conversion from host byte order to the device's byte order) is almost always required when presenting segment information to the device. .It Vt bus_dmamap_t A machine-dependent opaque type describing an individual mapping. One map is used for each memory allocation that will be loaded. Maps can be reused once they have been unloaded. Multiple maps can be associated with one DMA tag. While the value of the map may evaluate to .Dv NULL on some platforms under certain conditions, it should never be assumed that it will be .Dv NULL in all cases. .It Vt bus_dmamap_callback_t Client specified callback for receiving mapping information resulting from the load of a .Vt bus_dmamap_t via .Fn bus_dmamap_load , .Fn bus_dmamap_load_bio , .Fn bus_dmamap_load_ccb , .Fn bus_dmamap_load_crp , or .Fn bus_dmamap_load_crp_buffer . Callbacks are of the format: .Bl -tag -width indent .It Ft void .Fn "client_callback" "void *callback_arg" "bus_dma_segment_t *segs" \ "int nseg" "int error" .El .Pp The .Fa callback_arg is the callback argument passed to dmamap load functions. The .Fa segs and .Fa nseg arguments describe an array of .Vt bus_dma_segment_t structures that represent the mapping. This array is only valid within the scope of the callback function. The success or failure of the mapping is indicated by the .Fa error argument. More information on the use of callbacks can be found in the description of the individual dmamap load functions. .It Vt bus_dmamap_callback2_t Client specified callback for receiving mapping information resulting from the load of a .Vt bus_dmamap_t via .Fn bus_dmamap_load_uio or .Fn bus_dmamap_load_mbuf . .Pp Callback2s are of the format: .Bl -tag -width indent .It Ft void .Fn "client_callback2" "void *callback_arg" "bus_dma_segment_t *segs" \ "int nseg" "bus_size_t mapsize" "int error" .El .Pp Callback2's behavior is the same as .Vt bus_dmamap_callback_t with the addition that the length of the data mapped is provided via .Fa mapsize . .It Vt bus_dmasync_op_t Memory synchronization operation specifier. Bus DMA requires explicit synchronization of memory with its device visible mapping in order to guarantee memory coherency. The .Vt bus_dmasync_op_t allows the type of DMA operation that will be or has been performed to be communicated to the system so that the correct coherency measures are taken. The operations are represented as bitfield flags that can be combined together, though it only makes sense to combine PRE flags or POST flags, not both. See the .Fn bus_dmamap_sync description below for more details on how to use these operations. .Pp All operations specified below are performed from the host memory point of view, where a read implies data coming from the device to the host memory, and a write implies data going from the host memory to the device. Alternatively, the operations can be thought of in terms of driver operations, where reading a network packet or storage sector corresponds to a read operation in .Nm . .Bl -tag -width ".Dv BUS_DMASYNC_POSTWRITE" .It Dv BUS_DMASYNC_PREREAD Perform any synchronization required prior to an update of host memory by the device. .It Dv BUS_DMASYNC_PREWRITE Perform any synchronization required after an update of host memory by the CPU and prior to device access to host memory. .It Dv BUS_DMASYNC_POSTREAD Perform any synchronization required after an update of host memory by the device and prior to CPU access to host memory. .It Dv BUS_DMASYNC_POSTWRITE Perform any synchronization required after device access to host memory. .El .It Vt bus_dma_lock_t Client specified lock/mutex manipulation method. This will be called from within busdma whenever a client lock needs to be manipulated. In its current form, the function will be called immediately before the callback for a DMA load operation that has been deferred with .Dv BUS_DMA_LOCK and immediately after with .Dv BUS_DMA_UNLOCK . If the load operation does not need to be deferred, then it will not be called since the function loading the map should be holding the appropriate locks. This method is of the format: .Bl -tag -width indent .It Ft void .Fn "lockfunc" "void *lockfunc_arg" "bus_dma_lock_op_t op" .El .Pp The .Fa lockfuncarg argument is specified by the client during tag creation to be passed to all invocations of the callback. The .Fa op argument specifies the lock operation to perform. .Pp Two .Vt lockfunc implementations are provided for convenience. .Fn busdma_lock_mutex performs standard mutex operations on the sleep mutex provided via .Fa lockfuncarg . .Fn dflt_lock will generate a system panic if it is called. It is substituted into the tag when .Fa lockfunc is passed as .Dv NULL to .Fn bus_dma_tag_create and is useful for tags that should not be used with deferred load operations. .It Vt bus_dma_lock_op_t Operations to be performed by the client-specified .Fn lockfunc . .Bl -tag -width ".Dv BUS_DMA_UNLOCK" .It Dv BUS_DMA_LOCK Acquires and/or locks the client locking primitive. .It Dv BUS_DMA_UNLOCK Releases and/or unlocks the client locking primitive. .El .El .Sh FUNCTIONS .Bl -tag -width indent .It Fn bus_dma_tag_create "parent" "alignment" "boundary" "lowaddr" \ "highaddr" "*filtfunc" "*filtfuncarg" "maxsize" "nsegments" "maxsegsz" \ "flags" "lockfunc" "lockfuncarg" "*dmat" Allocates a DMA tag, and initializes it according to the arguments provided: .Bl -tag -width ".Fa filtfuncarg" .It Fa parent A parent tag from which to inherit restrictions. The restrictions passed in other arguments can only further tighten the restrictions inherited from the parent tag. .Pp All tags created by a device driver must inherit from the tag returned by .Fn bus_get_dma_tag to honor restrictions between the parent bridge, CPU memory, and the device. .It Fa alignment Alignment constraint, in bytes, of any mappings created using this tag. The alignment must be a power of 2. Hardware that can DMA starting at any address would specify .Em 1 for byte alignment. Hardware requiring DMA transfers to start on a multiple of 4K would specify .Em 4096 . .It Fa boundary Boundary constraint, in bytes, of the target DMA memory region. The boundary indicates the set of addresses, all multiples of the boundary argument, that cannot be crossed by a single .Vt bus_dma_segment_t . The boundary must be a power of 2 and must be no smaller than the maximum segment size. .Ql 0 indicates that there are no boundary restrictions. .It Fa lowaddr , highaddr Bounds of the window of bus address space that .Em cannot be directly accessed by the device. The window contains all addresses greater than .Fa lowaddr and less than or equal to .Fa highaddr . For example, a device incapable of DMA above 4GB, would specify a .Fa highaddr of .Dv BUS_SPACE_MAXADDR and a .Fa lowaddr of .Dv BUS_SPACE_MAXADDR_32BIT . Similarly a device that can only perform DMA to addresses below 16MB would specify a .Fa highaddr of .Dv BUS_SPACE_MAXADDR and a .Fa lowaddr of .Dv BUS_SPACE_MAXADDR_24BIT . Some implementations require that some region of device visible address space, overlapping available host memory, be outside the window. This area of .Ql safe memory is used to bounce requests that would otherwise conflict with the exclusion window. .It Fa filtfunc -Optional filter function (may be -.Dv NULL ) -to be called for any attempt to -map memory into the window described by -.Fa lowaddr -and -.Fa highaddr . -A filter function is only required when the single window described -by -.Fa lowaddr -and -.Fa highaddr -cannot adequately describe the constraints of the device. -The filter function will be called for every machine page -that overlaps the exclusion window. -.Pp -.Em Note: The use of filters is deprecated. Proper operation is not guaranteed. +Formerly the optional filter function; must be +.Dv NULL . .It Fa filtfuncarg -Argument passed to all calls to the filter function for this tag. -May be +Must be .Dv NULL . .It Fa maxsize Maximum size, in bytes, of the sum of all segment lengths in a given DMA mapping associated with this tag. .It Fa nsegments Number of discontinuities (scatter/gather segments) allowed in a DMA mapped region. .It Fa maxsegsz Maximum size, in bytes, of a segment in any DMA mapped region associated with .Fa dmat . .It Fa flags Are as follows: .Bl -tag -width ".Dv BUS_DMA_ALLOCNOW" .It Dv BUS_DMA_ALLOCNOW Pre-allocate enough resources to handle at least one map load operation on this tag. If sufficient resources are not available, .Er ENOMEM is returned. This should not be used for tags that only describe buffers that will be allocated with .Fn bus_dmamem_alloc . Also, due to resource sharing with other tags, this flag does not guarantee that resources will be allocated or reserved exclusively for this tag. It should be treated only as a minor optimization. .It Dv BUS_DMA_COHERENT Indicate that the DMA engine and CPU are cache-coherent. Cached memory may be used to back allocations created by .Fn bus_dmamem_alloc . For .Fn bus_dma_tag_create , the .Dv BUS_DMA_COHERENT flag is currently implemented on arm64. .El .It Fa lockfunc Optional lock manipulation function (may be .Dv NULL ) to be called when busdma needs to manipulate a lock on behalf of the client. If .Dv NULL is specified, .Fn dflt_lock is used. .It Fa lockfuncarg Optional argument to be passed to the function specified by .Fa lockfunc . .It Fa dmat Pointer to a bus_dma_tag_t where the resulting DMA tag will be stored. .El .Pp Returns .Er ENOMEM if sufficient memory is not available for tag creation or allocating mapping resources. +Returns +.Er EINVAL +if either +.Fa filtfunc +or +.Fa filtarg +arguments are not +.Dv NULL . .It Fn bus_dma_tag_destroy "dmat" Deallocate the DMA tag .Fa dmat that was created by .Fn bus_dma_tag_create . .Pp Returns .Er EBUSY if any DMA maps remain associated with .Fa dmat or .Ql 0 on success. .It Fn bus_dma_template_init "*template" "parent" Initializes a .Fa bus_dma_template_t structure. If the .Fa parent argument is non-NULL, this parent tag is associated with the template and will be compiled into the dma tag that is later created. The values of the parent are not copied into the template. During tag creation in .Fn bus_dma_tag_template , any parameters from the parent tag that are more restrictive than what is in the provided template will overwrite what goes into the new tag. .It Fn bus_dma_template_tag "*template" "*dmat" Unpacks a template into a tag, and returns the tag via the .Fa dmat . All return values are identical to .Fn bus_dma_tag_create . The template is not modified by this function, and can be reused and/or freed upon return. .It Fn bus_dma_template_clone "*template" "dmat" Copies the fields from an existing tag to a template. The template does not need to be initialized first. All of its fields will be overwritten by the values contained in the tag. When paired with .Fn bus_dma_template_tag , this function is useful for creating copies of tags. .It Fn bus_dma_template_fill "*template" "params[]" "count" Fills in the selected fields of the template with the keyed values from the .Fa params array. This is not meant to be called directly, use .Fn BUS_DMA_TEMPLATE_FILL instead. .It Fn BUS_DMA_TEMPLATE_FILL "*template" "param ..." Fills in the selected fields of the template with a variable number of key-value parameters. The macros listed below take an argument of the specified type and encapsulate it into a key-value structure that is directly usable as a parameter argument. Muliple parameters may be provided at once. .Bd -literal BD_PARENT() void * BD_ALIGNMENT() uintmax_t BD_BOUNDARY() uintmax_t BD_LOWADDR() vm_paddr_t BD_HIGHADDR() vm_paddr_t BD_MAXSIZE() uintmax_t BD_NSEGMENTS() uintmax_t BD_MAXSEGSIZE() uintmax_t BD_FLAGS() uintmax_t BD_LOCKFUNC() void * BD_LOCKFUNCARG() void * .Ed .It Fn bus_dmamap_create "dmat" "flags" "*mapp" Allocates and initializes a DMA map. Arguments are as follows: .Bl -tag -width ".Fa nsegments" .It Fa dmat DMA tag. .It Fa flags Are as follows: .Bl -tag -width ".Dv BUS_DMA_COHERENT" .It Dv BUS_DMA_COHERENT Attempt to map the memory loaded with this map such that cache sync operations are as cheap as possible. This flag is typically set on maps when the memory loaded with these will be accessed by both a CPU and a DMA engine, frequently such as control data and as opposed to streamable data such as receive and transmit buffers. Use of this flag does not remove the requirement of using .Fn bus_dmamap_sync , but it may reduce the cost of performing these operations. .El .It Fa mapp Pointer to a .Vt bus_dmamap_t where the resulting DMA map will be stored. .El .Pp Returns .Er ENOMEM if sufficient memory is not available for creating the map or allocating mapping resources. .It Fn bus_dmamap_destroy "dmat" "map" Frees all resources associated with a given DMA map. Arguments are as follows: .Bl -tag -width ".Fa dmat" .It Fa dmat DMA tag used to allocate .Fa map . .It Fa map The DMA map to destroy. .El .Pp Returns .Er EBUSY if a mapping is still active for .Fa map . .It Fn bus_dmamap_load "dmat" "map" "buf" "buflen" "*callback" \ "callback_arg" "flags" Creates a mapping in device visible address space of .Fa buflen bytes of .Fa buf , associated with the DMA map .Fa map . This call will always return immediately and will not block for any reason. Arguments are as follows: .Bl -tag -width ".Fa buflen" .It Fa dmat DMA tag used to allocate .Fa map . .It Fa map A DMA map without a currently active mapping. .It Fa buf A kernel virtual address pointer to a contiguous (in KVA) buffer, to be mapped into device visible address space. .It Fa buflen The size of the buffer. .It Fa callback Fa callback_arg The callback function, and its argument. This function is called once sufficient mapping resources are available for the DMA operation. If resources are temporarily unavailable, this function will be deferred until later, but the load operation will still return immediately to the caller. Thus, callers should not assume that the callback will be called before the load returns, and code should be structured appropriately to handle this. See below for specific flags and error codes that control this behavior. .It Fa flags Are as follows: .Bl -tag -width ".Dv BUS_DMA_NOWAIT" .It Dv BUS_DMA_NOWAIT The load should not be deferred in case of insufficient mapping resources, and instead should return immediately with an appropriate error. .It Dv BUS_DMA_NOCACHE The generated transactions to and from the virtual page are non-cacheable. .El .El .Pp Return values to the caller are as follows: .Bl -tag -width ".Er EINPROGRESS" .It 0 The callback has been called and completed. The status of the mapping has been delivered to the callback. .It Er EINPROGRESS The mapping has been deferred for lack of resources. The callback will be called as soon as resources are available. Callbacks are serviced in FIFO order. .Pp Note that subsequent load operations for the same tag that do not require extra resources will still succeed. This may result in out-of-order processing of requests. If the caller requires the order of requests to be preserved, then the caller is required to stall subsequent requests until a pending request's callback is invoked. .It Er ENOMEM The load request has failed due to insufficient resources, and the caller specifically used the .Dv BUS_DMA_NOWAIT flag. .It Er EINVAL The load request was invalid. The callback has been called and has been provided the same error. This error value may indicate that .Fa dmat , .Fa map , .Fa buf , or .Fa callback were invalid, or .Fa buflen was larger than the .Fa maxsize argument used to create the dma tag .Fa dmat . .El .Pp When the callback is called, it is presented with an error value indicating the disposition of the mapping. Error may be one of the following: .Bl -tag -width ".Er EINPROGRESS" .It 0 The mapping was successful and the .Fa dm_segs callback argument contains an array of .Vt bus_dma_segment_t elements describing the mapping. This array is only valid during the scope of the callback function. .It Er EFBIG A mapping could not be achieved within the segment constraints provided in the tag even though the requested allocation size was less than maxsize. .El .It Fn bus_dmamap_load_bio "dmat" "map" "bio" "callback" "callback_arg" "flags" This is a variation of .Fn bus_dmamap_load which maps buffers pointed to by .Fa bio for DMA transfers. .Fa bio may point to either a mapped or unmapped buffer. .It Fn bus_dmamap_load_ccb "dmat" "map" "ccb" "callback" "callback_arg" "flags" This is a variation of .Fn bus_dmamap_load which maps data pointed to by .Fa ccb for DMA transfers. The data for .Fa ccb may be any of the following types: .Bl -tag -width ".Er CAM_DATA_SG_PADDR" .It CAM_DATA_VADDR The data is a single KVA buffer. .It CAM_DATA_PADDR The data is a single bus address range. .It CAM_DATA_SG The data is a scatter/gather list of KVA buffers. .It CAM_DATA_SG_PADDR The data is a scatter/gather list of bus address ranges. .It CAM_DATA_BIO The data is contained in a .Vt struct bio attached to the CCB. .El .Pp .Fn bus_dmamap_load_ccb supports the following CCB XPT function codes: .Pp .Bl -item -offset indent -compact .It XPT_ATA_IO .It XPT_CONT_TARGET_IO .It XPT_SCSI_IO .El .It Fn bus_dmamap_load_crp "dmat" "map" "crp" "callback" "callback_arg" "flags" This is a variation of .Fn bus_dmamap_load which maps the input buffer pointed to by .Fa crp for DMA transfers. The .Dv BUS_DMA_NOWAIT flag is implied, thus no callback deferral will happen. .It Fn bus_dmamap_load_crp_buffer "dmat" "map" "cb" "callback" "callback_arg" \ "flags" This is a variation of .Fn bus_dmamap_load which maps the crypto data buffer pointed to by .Fa cb for DMA transfers. The .Dv BUS_DMA_NOWAIT flag is implied, thus no callback deferral will happen. .It Fn bus_dmamap_load_mbuf "dmat" "map" "mbuf" "callback2" "callback_arg" \ "flags" This is a variation of .Fn bus_dmamap_load which maps mbuf chains for DMA transfers. A .Vt bus_size_t argument is also passed to the callback routine, which contains the mbuf chain's packet header length. The .Dv BUS_DMA_NOWAIT flag is implied, thus no callback deferral will happen. .Pp Mbuf chains are assumed to be in kernel virtual address space. .Pp Beside the error values listed for .Fn bus_dmamap_load , .Er EINVAL will be returned if the size of the mbuf chain exceeds the maximum limit of the DMA tag. .It Fn bus_dmamap_load_mbuf_sg "dmat" "map" "mbuf" "segs" "nsegs" "flags" This is just like .Fn bus_dmamap_load_mbuf except that it returns immediately without calling a callback function. It is provided for efficiency. The scatter/gather segment array .Va segs is provided by the caller and filled in directly by the function. The .Va nsegs argument is returned with the number of segments filled in. Returns the same errors as .Fn bus_dmamap_load_mbuf . .It Fn bus_dmamap_load_uio "dmat" "map" "uio" "callback2" "callback_arg" "flags" This is a variation of .Fn bus_dmamap_load which maps buffers pointed to by .Fa uio for DMA transfers. A .Vt bus_size_t argument is also passed to the callback routine, which contains the size of .Fa uio , i.e. .Fa uio->uio_resid . The .Dv BUS_DMA_NOWAIT flag is implied, thus no callback deferral will happen. Returns the same errors as .Fn bus_dmamap_load . .Pp If .Fa uio->uio_segflg is .Dv UIO_USERSPACE , then it is assumed that the buffer, .Fa uio is in .Fa "uio->uio_td->td_proc" Ns 's address space. User space memory must be in-core and wired prior to attempting a map load operation. Pages may be locked using .Xr vslock 9 . .It Fn bus_dmamap_unload "dmat" "map" Unloads a DMA map. Arguments are as follows: .Bl -tag -width ".Fa dmam" .It Fa dmat DMA tag used to allocate .Fa map . .It Fa map The DMA map that is to be unloaded. .El .Pp .Fn bus_dmamap_unload will not perform any implicit synchronization of DMA buffers. This must be done explicitly by a call to .Fn bus_dmamap_sync prior to unloading the map. .It Fn bus_dmamap_sync "dmat" "map" "op" Performs synchronization of a device visible mapping with the CPU visible memory referenced by that mapping. Arguments are as follows: .Bl -tag -width ".Fa dmat" .It Fa dmat DMA tag used to allocate .Fa map . .It Fa map The DMA mapping to be synchronized. .It Fa op Type of synchronization operation to perform. See the definition of .Vt bus_dmasync_op_t for a description of the acceptable values for .Fa op . .El .Pp The .Fn bus_dmamap_sync function is the method used to ensure that CPU's and device's direct memory access (DMA) to shared memory is coherent. For example, the CPU might be used to set up the contents of a buffer that is to be made available to a device. To ensure that the data are visible via the device's mapping of that memory, the buffer must be loaded and a DMA sync operation of .Dv BUS_DMASYNC_PREWRITE must be performed after the CPU has updated the buffer and before the device access is initiated. If the CPU modifies this buffer again later, another .Dv BUS_DMASYNC_PREWRITE sync operation must be performed before an additional device access. Conversely, suppose a device updates memory that is to be read by a CPU. In this case, the buffer must be loaded, and a DMA sync operation of .Dv BUS_DMASYNC_PREREAD must be performed before the device access is initiated. The CPU will only be able to see the results of this memory update once the DMA operation has completed and a .Dv BUS_DMASYNC_POSTREAD sync operation has been performed. .Pp If read and write operations are not preceded and followed by the appropriate synchronization operations, behavior is undefined. .It Fn bus_dmamem_alloc "dmat" "**vaddr" "flags" "*mapp" Allocates memory that is mapped into KVA at the address returned in .Fa vaddr and that is permanently loaded into the newly created .Vt bus_dmamap_t returned via .Fa mapp . Arguments are as follows: .Bl -tag -width ".Fa alignment" .It Fa dmat DMA tag describing the constraints of the DMA mapping. .It Fa vaddr Pointer to a pointer that will hold the returned KVA mapping of the allocated region. .It Fa flags Flags are defined as follows: .Bl -tag -width ".Dv BUS_DMA_NOWAIT" .It Dv BUS_DMA_WAITOK The routine can safely wait (sleep) for resources. .It Dv BUS_DMA_NOWAIT The routine is not allowed to wait for resources. If resources are not available, .Dv ENOMEM is returned. .It Dv BUS_DMA_COHERENT Attempt to map this memory in a coherent fashion. See .Fn bus_dmamap_create above for a description of this flag. For .Fn bus_dmamem_alloc , the .Dv BUS_DMA_COHERENT flag is currently implemented on arm and arm64. .It Dv BUS_DMA_ZERO Causes the allocated memory to be set to all zeros. .It Dv BUS_DMA_NOCACHE The allocated memory will not be cached in the processor caches. All memory accesses appear on the bus and are executed without reordering. For .Fn bus_dmamem_alloc , the .Dv BUS_DMA_NOCACHE flag is currently implemented on amd64 and i386 where it results in the Strong Uncacheable PAT to be set for the allocated virtual address range. .El .It Fa mapp Pointer to a .Vt bus_dmamap_t where the resulting DMA map will be stored. .El .Pp The size of memory to be allocated is .Fa maxsize as specified in the call to .Fn bus_dma_tag_create for .Fa dmat . .Pp The current implementation of .Fn bus_dmamem_alloc will allocate all requests as a single segment. .Pp An initial load operation is required to obtain the bus address of the allocated memory, and an unload operation is required before freeing the memory, as described below in .Fn bus_dmamem_free . Maps are automatically handled by this function and should not be explicitly allocated or destroyed. .Pp Although an explicit load is not required for each access to the memory referenced by the returned map, the synchronization requirements as described in the .Fn bus_dmamap_sync section still apply and should be used to achieve portability on architectures without coherent buses. .Pp Returns .Er ENOMEM if sufficient memory is not available for completing the operation. .It Fn bus_dmamem_free "dmat" "*vaddr" "map" Frees memory previously allocated by .Fn bus_dmamem_alloc . Any mappings will be invalidated. Arguments are as follows: .Bl -tag -width ".Fa vaddr" .It Fa dmat DMA tag. .It Fa vaddr Kernel virtual address of the memory. .It Fa map DMA map to be invalidated. .El .El .Sh RETURN VALUES Behavior is undefined if invalid arguments are passed to any of the above functions. If sufficient resources cannot be allocated for a given transaction, .Er ENOMEM is returned. All routines that are not of type .Vt void will return 0 on success or an error code on failure as discussed above. .Pp All .Vt void routines will succeed if provided with valid arguments. .Sh LOCKING Two locking protocols are used by .Nm . The first is a private global lock that is used to synchronize access to the bounce buffer pool on the architectures that make use of them. This lock is strictly a leaf lock that is only used internally to .Nm and is not exposed to clients of the API. .Pp The second protocol involves protecting various resources stored in the tag. Since almost all .Nm operations are done through requests from the driver that created the tag, the most efficient way to protect the tag resources is through the lock that the driver uses. In cases where .Nm acts on its own without being called by the driver, the lock primitive specified in the tag is acquired and released automatically. An example of this is when the .Fn bus_dmamap_load callback function is called from a deferred context instead of the driver context. This means that certain .Nm functions must always be called with the same lock held that is specified in the tag. These functions include: .Pp .Bl -item -offset indent -compact .It .Fn bus_dmamap_load .It .Fn bus_dmamap_load_bio .It .Fn bus_dmamap_load_ccb .It .Fn bus_dmamap_load_mbuf .It .Fn bus_dmamap_load_mbuf_sg .It .Fn bus_dmamap_load_uio .It .Fn bus_dmamap_unload .It .Fn bus_dmamap_sync .El .Pp There is one exception to this rule. It is common practice to call some of these functions during driver start-up without any locks held. So long as there is a guarantee of no possible concurrent use of the tag by different threads during this operation, it is safe to not hold a lock for these functions. .Pp Certain .Nm operations should not be called with the driver lock held, either because they are already protected by an internal lock, or because they might sleep due to memory or resource allocation. The following functions must not be called with any non-sleepable locks held: .Pp .Bl -item -offset indent -compact .It .Fn bus_dma_tag_create .It .Fn bus_dmamap_create .It .Fn bus_dmamem_alloc .El .Pp All other functions do not have a locking protocol and can thus be called with or without any system or driver locks held. .Sh SEE ALSO .Xr devclass 9 , .Xr device 9 , .Xr driver 9 , .Xr rman 9 , .Xr vslock 9 .Pp .Rs .%A "Jason R. Thorpe" .%T "A Machine-Independent DMA Framework for NetBSD" .%J "Proceedings of the Summer 1998 USENIX Technical Conference" .%Q "USENIX Association" .%D "June 1998" .Re .Sh HISTORY The .Nm interface first appeared in .Nx 1.3 . .Pp The .Nm API was adopted from .Nx for use in the CAM SCSI subsystem. The alterations to the original API were aimed to remove the need for a .Vt bus_dma_segment_t array stored in each .Vt bus_dmamap_t while allowing callers to queue up on scarce resources. .Sh AUTHORS The .Nm interface was designed and implemented by .An Jason R. Thorpe of the Numerical Aerospace Simulation Facility, NASA Ames Research Center. Additional input on the .Nm design was provided by .An -nosplit .An Chris Demetriou , .An Charles Hannum , .An Ross Harvey , .An Matthew Jacob , .An Jonathan Stone , and .An Matt Thomas . .Pp The .Nm interface in .Fx benefits from the contributions of .An Justin T. Gibbs , .An Peter Wemm , .An Doug Rabson , .An Matthew N. Dodd , .An Sam Leffler , .An Maxime Henrion , .An Jake Burkholder , .An Takahashi Yoshihiro , .An Scott Long and many others. .Pp This manual page was written by .An Hiten M. Pandya and .An Justin T. Gibbs . diff --git a/sys/arm/arm/busdma_machdep.c b/sys/arm/arm/busdma_machdep.c index aa6aa76234cb..282a8ccea690 100644 --- a/sys/arm/arm/busdma_machdep.c +++ b/sys/arm/arm/busdma_machdep.c @@ -1,1405 +1,1409 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012-2015 Ian Lepore * Copyright (c) 2010 Mark Tinguely * Copyright (c) 2004 Olivier Houchard * Copyright (c) 2002 Peter Grehan * Copyright (c) 1997, 1998 Justin T. Gibbs. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * From i386/busdma_machdep.c 191438 2009-04-23 20:24:19Z jhb */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include //#define ARM_BUSDMA_MAPLOAD_STATS #define BUSDMA_DCACHE_ALIGN cpuinfo.dcache_line_size #define BUSDMA_DCACHE_MASK cpuinfo.dcache_line_mask #define MAX_BPAGES 64 #define MAX_DMA_SEGMENTS 4096 #define BUS_DMA_EXCL_BOUNCE BUS_DMA_BUS2 #define BUS_DMA_ALIGN_BOUNCE BUS_DMA_BUS3 #define BUS_DMA_COULD_BOUNCE (BUS_DMA_EXCL_BOUNCE | BUS_DMA_ALIGN_BOUNCE) #define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 struct bounce_page; struct bounce_zone; struct bus_dma_tag { bus_dma_tag_t parent; bus_size_t alignment; bus_addr_t boundary; bus_addr_t lowaddr; bus_addr_t highaddr; bus_dma_filter_t *filter; void *filterarg; bus_size_t maxsize; u_int nsegments; bus_size_t maxsegsz; int flags; int ref_count; int map_count; bus_dma_lock_t *lockfunc; void *lockfuncarg; struct bounce_zone *bounce_zone; }; struct sync_list { vm_offset_t vaddr; /* kva of client data */ bus_addr_t paddr; /* physical address */ vm_page_t pages; /* starting page of client data */ bus_size_t datacount; /* client data count */ }; static uint32_t tags_total; static uint32_t maps_total; static uint32_t maps_dmamem; static uint32_t maps_coherent; #ifdef ARM_BUSDMA_MAPLOAD_STATS static counter_u64_t maploads_total; static counter_u64_t maploads_bounced; static counter_u64_t maploads_coherent; static counter_u64_t maploads_dmamem; static counter_u64_t maploads_mbuf; static counter_u64_t maploads_physmem; #endif SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "Busdma parameters"); SYSCTL_UINT(_hw_busdma, OID_AUTO, tags_total, CTLFLAG_RD, &tags_total, 0, "Number of active tags"); SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_total, CTLFLAG_RD, &maps_total, 0, "Number of active maps"); SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_dmamem, CTLFLAG_RD, &maps_dmamem, 0, "Number of active maps for bus_dmamem_alloc buffers"); SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_coherent, CTLFLAG_RD, &maps_coherent, 0, "Number of active maps with BUS_DMA_COHERENT flag set"); #ifdef ARM_BUSDMA_MAPLOAD_STATS SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_total, CTLFLAG_RD, &maploads_total, "Number of load operations performed"); SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_bounced, CTLFLAG_RD, &maploads_bounced, "Number of load operations that used bounce buffers"); SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_coherent, CTLFLAG_RD, &maploads_dmamem, "Number of load operations on BUS_DMA_COHERENT memory"); SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_dmamem, CTLFLAG_RD, &maploads_dmamem, "Number of load operations on bus_dmamem_alloc buffers"); SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_mbuf, CTLFLAG_RD, &maploads_mbuf, "Number of load operations for mbufs"); SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_physmem, CTLFLAG_RD, &maploads_physmem, "Number of load operations on physical buffers"); #endif struct bus_dmamap { STAILQ_HEAD(, bounce_page) bpages; int pagesneeded; int pagesreserved; bus_dma_tag_t dmat; struct memdesc mem; bus_dmamap_callback_t *callback; void *callback_arg; __sbintime_t queued_time; int flags; #define DMAMAP_COHERENT (1 << 0) #define DMAMAP_DMAMEM_ALLOC (1 << 1) #define DMAMAP_MBUF (1 << 2) STAILQ_ENTRY(bus_dmamap) links; bus_dma_segment_t *segments; int sync_count; struct sync_list slist[]; }; static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, pmap_t pmap, bus_dmamap_t map, void *buf, bus_size_t buflen, int flags); static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, bus_size_t buflen, int flags); static void dma_preread_safe(vm_offset_t va, vm_paddr_t pa, vm_size_t size); static void dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op); static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata"); #define dmat_alignment(dmat) ((dmat)->alignment) #define dmat_flags(dmat) ((dmat)->flags) #define dmat_lowaddr(dmat) ((dmat)->lowaddr) #define dmat_lockfunc(dmat) ((dmat)->lockfunc) #define dmat_lockfuncarg(dmat) ((dmat)->lockfuncarg) #include "../../kern/subr_busdma_bounce.c" static void busdma_init(void *dummy) { int uma_flags; #ifdef ARM_BUSDMA_MAPLOAD_STATS maploads_total = counter_u64_alloc(M_WAITOK); maploads_bounced = counter_u64_alloc(M_WAITOK); maploads_coherent = counter_u64_alloc(M_WAITOK); maploads_dmamem = counter_u64_alloc(M_WAITOK); maploads_mbuf = counter_u64_alloc(M_WAITOK); maploads_physmem = counter_u64_alloc(M_WAITOK); #endif uma_flags = 0; /* Create a cache of buffers in standard (cacheable) memory. */ standard_allocator = busdma_bufalloc_create("buffer", BUSDMA_DCACHE_ALIGN,/* minimum_alignment */ NULL, /* uma_alloc func */ NULL, /* uma_free func */ uma_flags); /* uma_zcreate_flags */ #ifdef INVARIANTS /* * Force UMA zone to allocate service structures like * slabs using own allocator. uma_debug code performs * atomic ops on uma_slab_t fields and safety of this * operation is not guaranteed for write-back caches */ uma_flags = UMA_ZONE_NOTOUCH; #endif /* * Create a cache of buffers in uncacheable memory, to implement the * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. */ coherent_allocator = busdma_bufalloc_create("coherent", BUSDMA_DCACHE_ALIGN,/* minimum_alignment */ busdma_bufalloc_alloc_uncacheable, busdma_bufalloc_free_uncacheable, uma_flags); /* uma_zcreate_flags */ } /* * This init historically used SI_SUB_VM, but now the init code requires * malloc(9) using M_BUSDMA memory and the pcpu zones for counter(9), which get * set up by SI_SUB_KMEM and SI_ORDER_LAST, so we'll go right after that by * using SI_SUB_KMEM+1. */ SYSINIT(busdma, SI_SUB_KMEM+1, SI_ORDER_FIRST, busdma_init, NULL); /* * This routine checks the exclusion zone constraints from a tag against the * physical RAM available on the machine. If a tag specifies an exclusion zone * but there's no RAM in that zone, then we avoid allocating resources to bounce * a request, and we can use any memory allocator (as opposed to needing * kmem_alloc_contig() just because it can allocate pages in an address range). * * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the * same value on 32-bit architectures) as their lowaddr constraint, and we can't * possibly have RAM at an address higher than the highest address we can * express, so we take a fast out. */ static int exclusion_bounce_check(vm_offset_t lowaddr, vm_offset_t highaddr) { int i; if (lowaddr >= BUS_SPACE_MAXADDR) return (0); for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { if ((lowaddr >= phys_avail[i] && lowaddr < phys_avail[i + 1]) || (lowaddr < phys_avail[i] && highaddr >= phys_avail[i])) return (1); } return (0); } /* * Return true if the tag has an exclusion zone that could lead to bouncing. */ static __inline int exclusion_bounce(bus_dma_tag_t dmat) { return (dmat->flags & BUS_DMA_EXCL_BOUNCE); } /* * Return true if the given address does not fall on the alignment boundary. */ static __inline int alignment_bounce(bus_dma_tag_t dmat, bus_addr_t addr) { return (!vm_addr_align_ok(addr, dmat->alignment)); } /* * Return true if the DMA should bounce because the start or end does not fall * on a cacheline boundary (which would require a partial cacheline flush). * COHERENT memory doesn't trigger cacheline flushes. Memory allocated by * bus_dmamem_alloc() is always aligned to cacheline boundaries, and there's a * strict rule that such memory cannot be accessed by the CPU while DMA is in * progress (or by multiple DMA engines at once), so that it's always safe to do * full cacheline flushes even if that affects memory outside the range of a * given DMA operation that doesn't involve the full allocated buffer. If we're * mapping an mbuf, that follows the same rules as a buffer we allocated. */ static __inline int cacheline_bounce(bus_dmamap_t map, bus_addr_t addr, bus_size_t size) { if (map->flags & (DMAMAP_DMAMEM_ALLOC | DMAMAP_COHERENT | DMAMAP_MBUF)) return (0); return ((addr | size) & BUSDMA_DCACHE_MASK); } /* * Return true if we might need to bounce the DMA described by addr and size. * * This is used to quick-check whether we need to do the more expensive work of * checking the DMA page-by-page looking for alignment and exclusion bounces. * * Note that the addr argument might be either virtual or physical. It doesn't * matter because we only look at the low-order bits, which are the same in both * address spaces and maximum alignment of generic buffer is limited up to page * size. * Bouncing of buffers allocated by bus_dmamem_alloc()is not necessary, these * always comply with the required rules (alignment, boundary, and address * range). */ static __inline int might_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t addr, bus_size_t size) { KASSERT(map->flags & DMAMAP_DMAMEM_ALLOC || dmat->alignment <= PAGE_SIZE, ("%s: unsupported alignment (0x%08lx) for buffer not " "allocated by bus_dmamem_alloc()", __func__, dmat->alignment)); return (!(map->flags & DMAMAP_DMAMEM_ALLOC) && ((dmat->flags & BUS_DMA_EXCL_BOUNCE) || alignment_bounce(dmat, addr) || cacheline_bounce(map, addr, size))); } /* * Return true if we must bounce the DMA described by paddr and size. * * Bouncing can be triggered by DMA that doesn't begin and end on cacheline * boundaries, or doesn't begin on an alignment boundary, or falls within the * exclusion zone of any tag in the ancestry chain. * * For exclusions, walk the chain of tags comparing paddr to the exclusion zone * within each tag. If the tag has a filter function, use it to decide whether * the DMA needs to bounce, otherwise any DMA within the zone bounces. */ static int must_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, bus_size_t size) { if (cacheline_bounce(map, paddr, size)) return (1); /* * The tag already contains ancestors' alignment restrictions so this * check doesn't need to be inside the loop. */ if (alignment_bounce(dmat, paddr)) return (1); /* * Even though each tag has an exclusion zone that is a superset of its * own and all its ancestors' exclusions, the exclusion zone of each tag * up the chain must be checked within the loop, because the busdma * rules say the filter function is called only when the address lies * within the low-highaddr range of the tag that filterfunc belongs to. */ while (dmat != NULL && exclusion_bounce(dmat)) { if ((paddr >= dmat->lowaddr && paddr <= dmat->highaddr) && (dmat->filter == NULL || dmat->filter(dmat->filterarg, paddr) != 0)) return (1); dmat = dmat->parent; } return (0); } /* * Allocate a device specific dma_tag. */ int bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat) { bus_dma_tag_t newtag; int error = 0; /* Basic sanity checking. */ KASSERT(boundary == 0 || powerof2(boundary), ("dma tag boundary %lu, must be a power of 2", boundary)); KASSERT(boundary == 0 || boundary >= maxsegsz, ("dma tag boundary %lu is < maxsegsz %lu\n", boundary, maxsegsz)); KASSERT(alignment != 0 && powerof2(alignment), ("dma tag alignment %lu, must be non-zero power of 2", alignment)); KASSERT(maxsegsz != 0, ("dma tag maxsegsz must not be zero")); /* Return a NULL tag on failure */ *dmat = NULL; + /* Filters are no longer supported. */ + if (filter != NULL || filterarg != NULL) + return (EINVAL); + newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_BUSDMA, M_ZERO | M_NOWAIT); if (newtag == NULL) { CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", __func__, newtag, 0, error); return (ENOMEM); } newtag->parent = parent; newtag->alignment = alignment; newtag->boundary = boundary; newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); newtag->filter = filter; newtag->filterarg = filterarg; newtag->maxsize = maxsize; newtag->nsegments = nsegments; newtag->maxsegsz = maxsegsz; newtag->flags = flags; newtag->ref_count = 1; /* Count ourself */ newtag->map_count = 0; if (lockfunc != NULL) { newtag->lockfunc = lockfunc; newtag->lockfuncarg = lockfuncarg; } else { newtag->lockfunc = _busdma_dflt_lock; newtag->lockfuncarg = NULL; } /* Take into account any restrictions imposed by our parent tag */ if (parent != NULL) { newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); newtag->alignment = MAX(parent->alignment, newtag->alignment); newtag->flags |= parent->flags & BUS_DMA_COULD_BOUNCE; newtag->flags |= parent->flags & BUS_DMA_COHERENT; if (newtag->boundary == 0) newtag->boundary = parent->boundary; else if (parent->boundary != 0) newtag->boundary = MIN(parent->boundary, newtag->boundary); if (newtag->filter == NULL) { /* * Short circuit to looking at our parent directly * since we have encapsulated all of its information */ newtag->filter = parent->filter; newtag->filterarg = parent->filterarg; newtag->parent = parent->parent; } if (newtag->parent != NULL) atomic_add_int(&parent->ref_count, 1); } if (exclusion_bounce_check(newtag->lowaddr, newtag->highaddr)) newtag->flags |= BUS_DMA_EXCL_BOUNCE; if (alignment_bounce(newtag, 1)) newtag->flags |= BUS_DMA_ALIGN_BOUNCE; /* * Any request can auto-bounce due to cacheline alignment, in addition * to any alignment or boundary specifications in the tag, so if the * ALLOCNOW flag is set, there's always work to do. */ if ((flags & BUS_DMA_ALLOCNOW) != 0) { struct bounce_zone *bz; /* * Round size up to a full page, and add one more page because * there can always be one more boundary crossing than the * number of pages in a transfer. */ maxsize = roundup2(maxsize, PAGE_SIZE) + PAGE_SIZE; if ((error = alloc_bounce_zone(newtag)) != 0) { free(newtag, M_BUSDMA); return (error); } bz = newtag->bounce_zone; if (ptoa(bz->total_bpages) < maxsize) { int pages; pages = atop(maxsize) - bz->total_bpages; /* Add pages to our bounce pool */ if (alloc_bounce_pages(newtag, pages) < pages) error = ENOMEM; } /* Performed initial allocation */ newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; } else newtag->bounce_zone = NULL; if (error != 0) { free(newtag, M_BUSDMA); } else { atomic_add_32(&tags_total, 1); *dmat = newtag; } CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); return (error); } void bus_dma_template_clone(bus_dma_template_t *t, bus_dma_tag_t dmat) { if (t == NULL || dmat == NULL) return; t->parent = dmat->parent; t->alignment = dmat->alignment; t->boundary = dmat->boundary; t->lowaddr = dmat->lowaddr; t->highaddr = dmat->highaddr; t->maxsize = dmat->maxsize; t->nsegments = dmat->nsegments; t->maxsegsize = dmat->maxsegsz; t->flags = dmat->flags; t->lockfunc = dmat->lockfunc; t->lockfuncarg = dmat->lockfuncarg; } int bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) { return (0); } int bus_dma_tag_destroy(bus_dma_tag_t dmat) { #ifdef KTR bus_dma_tag_t dmat_copy = dmat; #endif int error; error = 0; if (dmat != NULL) { if (dmat->map_count != 0) { error = EBUSY; goto out; } while (dmat != NULL) { bus_dma_tag_t parent; parent = dmat->parent; atomic_subtract_int(&dmat->ref_count, 1); if (dmat->ref_count == 0) { atomic_subtract_32(&tags_total, 1); free(dmat, M_BUSDMA); /* * Last reference count, so * release our reference * count on our parent. */ dmat = parent; } else dmat = NULL; } } out: CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); return (error); } static int allocate_bz_and_pages(bus_dma_tag_t dmat, bus_dmamap_t mapp) { struct bounce_zone *bz; int maxpages; int error; if (dmat->bounce_zone == NULL) if ((error = alloc_bounce_zone(dmat)) != 0) return (error); bz = dmat->bounce_zone; /* Initialize the new map */ STAILQ_INIT(&(mapp->bpages)); /* * Attempt to add pages to our pool on a per-instance basis up to a sane * limit. Even if the tag isn't flagged as COULD_BOUNCE due to * alignment and boundary constraints, it could still auto-bounce due to * cacheline alignment, which requires at most two bounce pages. */ if (dmat->flags & BUS_DMA_COULD_BOUNCE) maxpages = MAX_BPAGES; else maxpages = 2 * bz->map_count; if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { int pages; pages = atop(roundup2(dmat->maxsize, PAGE_SIZE)) + 1; pages = MIN(maxpages - bz->total_bpages, pages); pages = MAX(pages, 2); if (alloc_bounce_pages(dmat, pages) < pages) return (ENOMEM); if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; } bz->map_count++; return (0); } static bus_dmamap_t allocate_map(bus_dma_tag_t dmat, int mflags) { int mapsize, segsize; bus_dmamap_t map; /* * Allocate the map. The map structure ends with an embedded * variable-sized array of sync_list structures. Following that * we allocate enough extra space to hold the array of bus_dma_segments. */ KASSERT(dmat->nsegments <= MAX_DMA_SEGMENTS, ("cannot allocate %u dma segments (max is %u)", dmat->nsegments, MAX_DMA_SEGMENTS)); segsize = sizeof(struct bus_dma_segment) * dmat->nsegments; mapsize = sizeof(*map) + sizeof(struct sync_list) * dmat->nsegments; map = malloc(mapsize + segsize, M_BUSDMA, mflags | M_ZERO); if (map == NULL) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); return (NULL); } map->segments = (bus_dma_segment_t *)((uintptr_t)map + mapsize); STAILQ_INIT(&map->bpages); return (map); } /* * Allocate a handle for mapping from kva/uva/physical * address space into bus device space. */ int bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) { bus_dmamap_t map; int error = 0; *mapp = map = allocate_map(dmat, M_NOWAIT); if (map == NULL) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); return (ENOMEM); } /* * Bouncing might be required if the driver asks for an exclusion * region, a data alignment that is stricter than 1, or DMA that begins * or ends with a partial cacheline. Whether bouncing will actually * happen can't be known until mapping time, but we need to pre-allocate * resources now because we might not be allowed to at mapping time. */ error = allocate_bz_and_pages(dmat, map); if (error != 0) { free(map, M_BUSDMA); *mapp = NULL; return (error); } if (map->flags & DMAMAP_COHERENT) atomic_add_32(&maps_coherent, 1); atomic_add_32(&maps_total, 1); dmat->map_count++; return (0); } /* * Destroy a handle for mapping from kva/uva/physical * address space into bus device space. */ int bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) { if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, EBUSY); return (EBUSY); } if (dmat->bounce_zone) dmat->bounce_zone->map_count--; if (map->flags & DMAMAP_COHERENT) atomic_subtract_32(&maps_coherent, 1); atomic_subtract_32(&maps_total, 1); free(map, M_BUSDMA); dmat->map_count--; CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); return (0); } /* * Allocate a piece of memory that can be efficiently mapped into bus device * space based on the constraints listed in the dma tag. Returns a pointer to * the allocated memory, and a pointer to an associated bus_dmamap. */ int bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags, bus_dmamap_t *mapp) { busdma_bufalloc_t ba; struct busdma_bufzone *bufzone; bus_dmamap_t map; vm_memattr_t memattr; int mflags; if (flags & BUS_DMA_NOWAIT) mflags = M_NOWAIT; else mflags = M_WAITOK; if (flags & BUS_DMA_ZERO) mflags |= M_ZERO; *mapp = map = allocate_map(dmat, mflags); if (map == NULL) { CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->flags, ENOMEM); return (ENOMEM); } map->flags = DMAMAP_DMAMEM_ALLOC; /* For coherent memory, set the map flag that disables sync ops. */ if (flags & BUS_DMA_COHERENT) map->flags |= DMAMAP_COHERENT; /* * Choose a busdma buffer allocator based on memory type flags. * If the tag's COHERENT flag is set, that means normal memory * is already coherent, use the normal allocator. */ if ((flags & BUS_DMA_COHERENT) && ((dmat->flags & BUS_DMA_COHERENT) == 0)) { memattr = VM_MEMATTR_UNCACHEABLE; ba = coherent_allocator; } else { memattr = VM_MEMATTR_DEFAULT; ba = standard_allocator; } /* * Try to find a bufzone in the allocator that holds a cache of buffers * of the right size for this request. If the buffer is too big to be * held in the allocator cache, this returns NULL. */ bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); /* * Allocate the buffer from the uma(9) allocator if... * - It's small enough to be in the allocator (bufzone not NULL). * - The alignment constraint isn't larger than the allocation size * (the allocator aligns buffers to their size boundaries). * - There's no need to handle lowaddr/highaddr exclusion zones. * else allocate non-contiguous pages if... * - The page count that could get allocated doesn't exceed * nsegments also when the maximum segment size is less * than PAGE_SIZE. * - The alignment constraint isn't larger than a page boundary. * - There are no boundary-crossing constraints. * else allocate a block of contiguous pages because one or more of the * constraints is something that only the contig allocator can fulfill. */ if (bufzone != NULL && dmat->alignment <= bufzone->size && !exclusion_bounce(dmat)) { *vaddr = uma_zalloc(bufzone->umazone, mflags); } else if (dmat->nsegments >= howmany(dmat->maxsize, MIN(dmat->maxsegsz, PAGE_SIZE)) && dmat->alignment <= PAGE_SIZE && (dmat->boundary % PAGE_SIZE) == 0) { *vaddr = kmem_alloc_attr(dmat->maxsize, mflags, 0, dmat->lowaddr, memattr); } else { *vaddr = kmem_alloc_contig(dmat->maxsize, mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, memattr); } if (*vaddr == NULL) { CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->flags, ENOMEM); free(map, M_BUSDMA); *mapp = NULL; return (ENOMEM); } if (map->flags & DMAMAP_COHERENT) atomic_add_32(&maps_coherent, 1); atomic_add_32(&maps_dmamem, 1); atomic_add_32(&maps_total, 1); dmat->map_count++; CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->flags, 0); return (0); } /* * Free a piece of memory that was allocated via bus_dmamem_alloc, along with * its associated map. */ void bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) { struct busdma_bufzone *bufzone; busdma_bufalloc_t ba; if ((map->flags & DMAMAP_COHERENT) && ((dmat->flags & BUS_DMA_COHERENT) == 0)) ba = coherent_allocator; else ba = standard_allocator; bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); if (bufzone != NULL && dmat->alignment <= bufzone->size && !exclusion_bounce(dmat)) uma_zfree(bufzone->umazone, vaddr); else kmem_free(vaddr, dmat->maxsize); dmat->map_count--; if (map->flags & DMAMAP_COHERENT) atomic_subtract_32(&maps_coherent, 1); atomic_subtract_32(&maps_total, 1); atomic_subtract_32(&maps_dmamem, 1); free(map, M_BUSDMA); CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); } static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, bus_size_t buflen, int flags) { bus_addr_t curaddr; bus_size_t sgsize; if (map->pagesneeded == 0) { CTR5(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d" " map= %p, pagesneeded= %d", dmat->lowaddr, dmat->boundary, dmat->alignment, map, map->pagesneeded); /* * Count the number of bounce pages * needed in order to complete this transfer */ curaddr = buf; while (buflen != 0) { sgsize = MIN(buflen, dmat->maxsegsz); if (must_bounce(dmat, map, curaddr, sgsize) != 0) { sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); map->pagesneeded++; } curaddr += sgsize; buflen -= sgsize; } CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded); } } static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, pmap_t pmap, bus_dmamap_t map, void *buf, bus_size_t buflen, int flags) { vm_offset_t vaddr; vm_offset_t vendaddr; bus_addr_t paddr; if (map->pagesneeded == 0) { CTR5(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d" " map= %p, pagesneeded= %d", dmat->lowaddr, dmat->boundary, dmat->alignment, map, map->pagesneeded); /* * Count the number of bounce pages * needed in order to complete this transfer */ vaddr = (vm_offset_t)buf; vendaddr = (vm_offset_t)buf + buflen; while (vaddr < vendaddr) { if (__predict_true(pmap == kernel_pmap)) paddr = pmap_kextract(vaddr); else paddr = pmap_extract(pmap, vaddr); if (must_bounce(dmat, map, paddr, min(vendaddr - vaddr, (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK)))) != 0) { map->pagesneeded++; } vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK)); } CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded); } } /* * Add a single contiguous physical range to the segment list. */ static int _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) { int seg; /* * Make sure we don't cross any boundaries. */ if (!vm_addr_bound_ok(curaddr, sgsize, dmat->boundary)) sgsize = roundup2(curaddr, dmat->boundary) - curaddr; /* * Insert chunk into a segment, coalescing with * previous segment if possible. */ seg = *segp; if (seg == -1) { seg = 0; segs[seg].ds_addr = curaddr; segs[seg].ds_len = sgsize; } else { if (curaddr == segs[seg].ds_addr + segs[seg].ds_len && (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && vm_addr_bound_ok(segs[seg].ds_addr, segs[seg].ds_len + sgsize, dmat->boundary)) segs[seg].ds_len += sgsize; else { if (++seg >= dmat->nsegments) return (0); segs[seg].ds_addr = curaddr; segs[seg].ds_len = sgsize; } } *segp = seg; return (sgsize); } /* * Utility function to load a physical buffer. segp contains * the starting segment on entrace, and the ending segment on exit. */ int _bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) { bus_addr_t curaddr; bus_addr_t sl_end = 0; bus_size_t sgsize; struct sync_list *sl; int error; if (segs == NULL) segs = map->segments; #ifdef ARM_BUSDMA_MAPLOAD_STATS counter_u64_add(maploads_total, 1); counter_u64_add(maploads_physmem, 1); #endif if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); if (map->pagesneeded != 0) { #ifdef ARM_BUSDMA_MAPLOAD_STATS counter_u64_add(maploads_bounced, 1); #endif error = _bus_dmamap_reserve_pages(dmat, map, flags); if (error) return (error); } } sl = map->slist + map->sync_count - 1; while (buflen > 0) { curaddr = buf; sgsize = MIN(buflen, dmat->maxsegsz); if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr, sgsize)) { sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); curaddr = add_bounce_page(dmat, map, 0, curaddr, sgsize); } else if ((dmat->flags & BUS_DMA_COHERENT) == 0) { if (map->sync_count > 0) sl_end = sl->paddr + sl->datacount; if (map->sync_count == 0 || curaddr != sl_end) { if (++map->sync_count > dmat->nsegments) break; sl++; sl->vaddr = 0; sl->paddr = curaddr; sl->datacount = sgsize; sl->pages = PHYS_TO_VM_PAGE(curaddr); KASSERT(sl->pages != NULL, ("%s: page at PA:0x%08lx is not in " "vm_page_array", __func__, curaddr)); } else sl->datacount += sgsize; } sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, segp); if (sgsize == 0) break; buf += sgsize; buflen -= sgsize; } /* * Did we fit? */ if (buflen != 0) { bus_dmamap_unload(dmat, map); return (EFBIG); /* XXX better return value here? */ } return (0); } int _bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs, int *segp) { return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, segs, segp)); } /* * Utility function to load a linear buffer. segp contains * the starting segment on entrance, and the ending segment on exit. */ int _bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, int *segp) { bus_size_t sgsize; bus_addr_t curaddr; bus_addr_t sl_pend = 0; vm_offset_t kvaddr, vaddr, sl_vend = 0; struct sync_list *sl; int error; #ifdef ARM_BUSDMA_MAPLOAD_STATS counter_u64_add(maploads_total, 1); if (map->flags & DMAMAP_COHERENT) counter_u64_add(maploads_coherent, 1); if (map->flags & DMAMAP_DMAMEM_ALLOC) counter_u64_add(maploads_dmamem, 1); #endif if (segs == NULL) segs = map->segments; if (flags & BUS_DMA_LOAD_MBUF) { #ifdef ARM_BUSDMA_MAPLOAD_STATS counter_u64_add(maploads_mbuf, 1); #endif map->flags |= DMAMAP_MBUF; } if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { _bus_dmamap_count_pages(dmat, pmap, map, buf, buflen, flags); if (map->pagesneeded != 0) { #ifdef ARM_BUSDMA_MAPLOAD_STATS counter_u64_add(maploads_bounced, 1); #endif error = _bus_dmamap_reserve_pages(dmat, map, flags); if (error) return (error); } } sl = map->slist + map->sync_count - 1; vaddr = (vm_offset_t)buf; while (buflen > 0) { /* * Get the physical address for this segment. */ if (__predict_true(pmap == kernel_pmap)) { curaddr = pmap_kextract(vaddr); kvaddr = vaddr; } else { curaddr = pmap_extract(pmap, vaddr); kvaddr = 0; } /* * Compute the segment size, and adjust counts. */ sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); if (sgsize > dmat->maxsegsz) sgsize = dmat->maxsegsz; if (buflen < sgsize) sgsize = buflen; if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr, sgsize)) { curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, sgsize); } else if ((dmat->flags & BUS_DMA_COHERENT) == 0) { if (map->sync_count > 0) { sl_pend = sl->paddr + sl->datacount; sl_vend = sl->vaddr + sl->datacount; } if (map->sync_count == 0 || (kvaddr != 0 && kvaddr != sl_vend) || (curaddr != sl_pend)) { if (++map->sync_count > dmat->nsegments) goto cleanup; sl++; sl->vaddr = kvaddr; sl->paddr = curaddr; if (kvaddr != 0) { sl->pages = NULL; } else { sl->pages = PHYS_TO_VM_PAGE(curaddr); KASSERT(sl->pages != NULL, ("%s: page at PA:0x%08lx is not " "in vm_page_array", __func__, curaddr)); } sl->datacount = sgsize; } else sl->datacount += sgsize; } sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, segp); if (sgsize == 0) break; vaddr += sgsize; buflen -= sgsize; } cleanup: /* * Did we fit? */ if (buflen != 0) { bus_dmamap_unload(dmat, map); return (EFBIG); /* XXX better return value here? */ } return (0); } void _bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) { map->mem = *mem; map->dmat = dmat; map->callback = callback; map->callback_arg = callback_arg; } bus_dma_segment_t * _bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int error) { if (segs == NULL) segs = map->segments; return (segs); } /* * Release the mapping held by map. */ void bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) { struct bounce_zone *bz; if ((bz = dmat->bounce_zone) != NULL) { free_bounce_pages(dmat, map); if (map->pagesreserved != 0) { mtx_lock(&bounce_lock); bz->free_bpages += map->pagesreserved; bz->reserved_bpages -= map->pagesreserved; mtx_unlock(&bounce_lock); map->pagesreserved = 0; } map->pagesneeded = 0; } map->sync_count = 0; map->flags &= ~DMAMAP_MBUF; } static void dma_preread_safe(vm_offset_t va, vm_paddr_t pa, vm_size_t size) { /* * Write back any partial cachelines immediately before and * after the DMA region. We don't need to round the address * down to the nearest cacheline or specify the exact size, * as dcache_wb_poc() will do the rounding for us and works * at cacheline granularity. */ if (va & BUSDMA_DCACHE_MASK) dcache_wb_poc(va, pa, 1); if ((va + size) & BUSDMA_DCACHE_MASK) dcache_wb_poc(va + size, pa + size, 1); dcache_inv_poc_dma(va, pa, size); } static void dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op) { uint32_t len, offset; vm_page_t m; vm_paddr_t pa; vm_offset_t va, tempva; bus_size_t size; offset = sl->paddr & PAGE_MASK; m = sl->pages; size = sl->datacount; pa = sl->paddr; for ( ; size != 0; size -= len, pa += len, offset = 0, ++m) { tempva = 0; if (sl->vaddr == 0) { len = min(PAGE_SIZE - offset, size); tempva = pmap_quick_enter_page(m); va = tempva | offset; KASSERT(pa == (VM_PAGE_TO_PHYS(m) | offset), ("unexpected vm_page_t phys: 0x%08x != 0x%08x", VM_PAGE_TO_PHYS(m) | offset, pa)); } else { len = sl->datacount; va = sl->vaddr; } switch (op) { case BUS_DMASYNC_PREWRITE: case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD: dcache_wb_poc(va, pa, len); break; case BUS_DMASYNC_PREREAD: /* * An mbuf may start in the middle of a cacheline. There * will be no cpu writes to the beginning of that line * (which contains the mbuf header) while dma is in * progress. Handle that case by doing a writeback of * just the first cacheline before invalidating the * overall buffer. Any mbuf in a chain may have this * misalignment. Buffers which are not mbufs bounce if * they are not aligned to a cacheline. */ dma_preread_safe(va, pa, len); break; case BUS_DMASYNC_POSTREAD: case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE: dcache_inv_poc(va, pa, len); break; default: panic("unsupported combination of sync operations: " "0x%08x\n", op); } if (tempva != 0) pmap_quick_remove_page(tempva); } } void bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { struct bounce_page *bpage; struct sync_list *sl, *end; vm_offset_t datavaddr, tempvaddr; if (op == BUS_DMASYNC_POSTWRITE) return; /* * If the buffer was from user space, it is possible that this is not * the same vm map, especially on a POST operation. It's not clear that * dma on userland buffers can work at all right now. To be safe, until * we're able to test direct userland dma, panic on a map mismatch. */ if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " "performing bounce", __func__, dmat, dmat->flags, op); /* * For PREWRITE do a writeback. Clean the caches from the * innermost to the outermost levels. */ if (op & BUS_DMASYNC_PREWRITE) { while (bpage != NULL) { tempvaddr = 0; datavaddr = bpage->datavaddr; if (datavaddr == 0) { tempvaddr = pmap_quick_enter_page( bpage->datapage); datavaddr = tempvaddr | bpage->dataoffs; } bcopy((void *)datavaddr, (void *)bpage->vaddr, bpage->datacount); if (tempvaddr != 0) pmap_quick_remove_page(tempvaddr); if ((dmat->flags & BUS_DMA_COHERENT) == 0) dcache_wb_poc(bpage->vaddr, bpage->busaddr, bpage->datacount); bpage = STAILQ_NEXT(bpage, links); } dmat->bounce_zone->total_bounced++; } /* * Do an invalidate for PREREAD unless a writeback was already * done above due to PREWRITE also being set. The reason for a * PREREAD invalidate is to prevent dirty lines currently in the * cache from being evicted during the DMA. If a writeback was * done due to PREWRITE also being set there will be no dirty * lines and the POSTREAD invalidate handles the rest. The * invalidate is done from the innermost to outermost level. If * L2 were done first, a dirty cacheline could be automatically * evicted from L1 before we invalidated it, re-dirtying the L2. */ if ((op & BUS_DMASYNC_PREREAD) && !(op & BUS_DMASYNC_PREWRITE)) { bpage = STAILQ_FIRST(&map->bpages); while (bpage != NULL) { if ((dmat->flags & BUS_DMA_COHERENT) == 0) dcache_inv_poc_dma(bpage->vaddr, bpage->busaddr, bpage->datacount); bpage = STAILQ_NEXT(bpage, links); } } /* * Re-invalidate the caches on a POSTREAD, even though they were * already invalidated at PREREAD time. Aggressive prefetching * due to accesses to other data near the dma buffer could have * brought buffer data into the caches which is now stale. The * caches are invalidated from the outermost to innermost; the * prefetches could be happening right now, and if L1 were * invalidated first, stale L2 data could be prefetched into L1. */ if (op & BUS_DMASYNC_POSTREAD) { while (bpage != NULL) { if ((dmat->flags & BUS_DMA_COHERENT) == 0) dcache_inv_poc(bpage->vaddr, bpage->busaddr, bpage->datacount); tempvaddr = 0; datavaddr = bpage->datavaddr; if (datavaddr == 0) { tempvaddr = pmap_quick_enter_page( bpage->datapage); datavaddr = tempvaddr | bpage->dataoffs; } bcopy((void *)bpage->vaddr, (void *)datavaddr, bpage->datacount); if (tempvaddr != 0) pmap_quick_remove_page(tempvaddr); bpage = STAILQ_NEXT(bpage, links); } dmat->bounce_zone->total_bounced++; } } /* * For COHERENT memory no cache maintenance is necessary, but ensure all * writes have reached memory for the PREWRITE case. No action is * needed for a PREREAD without PREWRITE also set, because that would * imply that the cpu had written to the COHERENT buffer and expected * the dma device to see that change, and by definition a PREWRITE sync * is required to make that happen. */ if (map->flags & DMAMAP_COHERENT) { if (op & BUS_DMASYNC_PREWRITE) { dsb(); if ((dmat->flags & BUS_DMA_COHERENT) == 0) cpu_l2cache_drain_writebuf(); } return; } /* * Cache maintenance for normal (non-COHERENT non-bounce) buffers. All * the comments about the sequences for flushing cache levels in the * bounce buffer code above apply here as well. In particular, the fact * that the sequence is inner-to-outer for PREREAD invalidation and * outer-to-inner for POSTREAD invalidation is not a mistake. */ if (map->sync_count != 0) { sl = &map->slist[0]; end = &map->slist[map->sync_count]; CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " "performing sync", __func__, dmat, dmat->flags, op); for ( ; sl != end; ++sl) dma_dcache_sync(sl, op); } } diff --git a/sys/arm64/arm64/busdma_machdep.c b/sys/arm64/arm64/busdma_machdep.c index 9f1137636ca7..0c1550267ae6 100644 --- a/sys/arm64/arm64/busdma_machdep.c +++ b/sys/arm64/arm64/busdma_machdep.c @@ -1,224 +1,228 @@ /*- * Copyright (c) 1997, 1998 Justin T. Gibbs. * Copyright (c) 2013, 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Portions of this software were developed by Semihalf * under sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Return true if a match is made. * * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. * * If paddr is within the bounds of the dma tag then call the filter callback * to check for a match, if there is no filter callback then assume a match. */ int bus_dma_run_filter(struct bus_dma_tag_common *tc, bus_addr_t paddr) { while (tc != NULL) { if ((paddr > tc->lowaddr && paddr <= tc->highaddr) && (tc->filter == NULL || (*tc->filter)(tc->filterarg, paddr) != 0)) return (1); tc = tc->parent; } return (0); } int common_bus_dma_tag_create(struct bus_dma_tag_common *parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, size_t sz, void **dmat) { void *newtag; struct bus_dma_tag_common *common; KASSERT(sz >= sizeof(struct bus_dma_tag_common), ("sz")); /* Return a NULL tag on failure */ *dmat = NULL; /* Basic sanity checking */ if (boundary != 0 && boundary < maxsegsz) maxsegsz = boundary; if (maxsegsz == 0) return (EINVAL); newtag = malloc(sz, M_DEVBUF, M_ZERO | M_NOWAIT); if (newtag == NULL) { CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", __func__, newtag, 0, ENOMEM); return (ENOMEM); } common = newtag; common->impl = &bus_dma_bounce_impl; common->parent = parent; common->alignment = alignment; common->boundary = boundary; common->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); common->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); common->filter = filter; common->filterarg = filterarg; common->maxsize = maxsize; common->nsegments = nsegments; common->maxsegsz = maxsegsz; common->flags = flags; common->ref_count = 1; /* Count ourself */ if (lockfunc != NULL) { common->lockfunc = lockfunc; common->lockfuncarg = lockfuncarg; } else { common->lockfunc = _busdma_dflt_lock; common->lockfuncarg = NULL; } /* Take into account any restrictions imposed by our parent tag */ if (parent != NULL) { common->impl = parent->impl; common->lowaddr = MIN(parent->lowaddr, common->lowaddr); common->highaddr = MAX(parent->highaddr, common->highaddr); common->alignment = MAX(parent->alignment, common->alignment); if (common->boundary == 0) common->boundary = parent->boundary; else if (parent->boundary != 0) { common->boundary = MIN(parent->boundary, common->boundary); } if (common->filter == NULL) { /* * Short circuit looking at our parent directly * since we have encapsulated all of its information */ common->filter = parent->filter; common->filterarg = parent->filterarg; common->parent = parent->parent; } common->domain = parent->domain; atomic_add_int(&parent->ref_count, 1); } common->domain = vm_phys_domain_match(common->domain, 0ul, common->lowaddr); *dmat = common; return (0); } /* * Allocate a device specific dma_tag. */ int bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat) { struct bus_dma_tag_common *tc; int error; + /* Filters are no longer supported. */ + if (filter != NULL || filterarg != NULL) + return (EINVAL); + if (parent == NULL) { error = bus_dma_bounce_impl.tag_create(parent, alignment, boundary, lowaddr, highaddr, filter, filterarg, maxsize, nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat); } else { tc = (struct bus_dma_tag_common *)parent; error = tc->impl->tag_create(parent, alignment, boundary, lowaddr, highaddr, filter, filterarg, maxsize, nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat); } return (error); } void bus_dma_template_clone(bus_dma_template_t *t, bus_dma_tag_t dmat) { struct bus_dma_tag_common *common; if (t == NULL || dmat == NULL) return; common = (struct bus_dma_tag_common *)dmat; t->parent = (bus_dma_tag_t)common->parent; t->alignment = common->alignment; t->boundary = common->boundary; t->lowaddr = common->lowaddr; t->highaddr = common->highaddr; t->maxsize = common->maxsize; t->nsegments = common->nsegments; t->maxsegsize = common->maxsegsz; t->flags = common->flags; t->lockfunc = common->lockfunc; t->lockfuncarg = common->lockfuncarg; } int bus_dma_tag_destroy(bus_dma_tag_t dmat) { struct bus_dma_tag_common *tc; tc = (struct bus_dma_tag_common *)dmat; return (tc->impl->tag_destroy(dmat)); } int bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) { struct bus_dma_tag_common *tc; tc = (struct bus_dma_tag_common *)dmat; domain = vm_phys_domain_match(domain, 0ul, tc->lowaddr); /* Only call the callback if it changes. */ if (domain == tc->domain) return (0); tc->domain = domain; return (tc->impl->tag_set_domain(dmat)); } diff --git a/sys/powerpc/powerpc/busdma_machdep.c b/sys/powerpc/powerpc/busdma_machdep.c index 21828e422d50..c1717140181e 100644 --- a/sys/powerpc/powerpc/busdma_machdep.c +++ b/sys/powerpc/powerpc/busdma_machdep.c @@ -1,878 +1,882 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1997, 1998 Justin T. Gibbs. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * From amd64/busdma_machdep.c, r204214 */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "iommu_if.h" #define MAX_BPAGES MIN(8192, physmem/40) struct bounce_page; struct bounce_zone; struct bus_dma_tag { bus_dma_tag_t parent; bus_size_t alignment; bus_addr_t boundary; bus_addr_t lowaddr; bus_addr_t highaddr; bus_dma_filter_t *filter; void *filterarg; bus_size_t maxsize; bus_size_t maxsegsz; u_int nsegments; int flags; int ref_count; int map_count; bus_dma_lock_t *lockfunc; void *lockfuncarg; struct bounce_zone *bounce_zone; device_t iommu; void *iommu_cookie; }; static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "Busdma parameters"); struct bus_dmamap { STAILQ_HEAD(, bounce_page) bpages; int pagesneeded; int pagesreserved; bus_dma_tag_t dmat; struct memdesc mem; bus_dma_segment_t *segments; int nsegs; bus_dmamap_callback_t *callback; void *callback_arg; __sbintime_t queued_time; STAILQ_ENTRY(bus_dmamap) links; int contigalloc; }; static MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata"); static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); #define dmat_alignment(dmat) ((dmat)->alignment) #define dmat_flags(dmat) ((dmat)->flags) #define dmat_lowaddr(dmat) ((dmat)->lowaddr) #define dmat_lockfunc(dmat) ((dmat)->lockfunc) #define dmat_lockfuncarg(dmat) ((dmat)->lockfuncarg) #include "../../kern/subr_busdma_bounce.c" /* * Return true if a match is made. * * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. * * If paddr is within the bounds of the dma tag then call the filter callback * to check for a match, if there is no filter callback then assume a match. */ static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) { int retval; retval = 0; do { if (dmat->filter == NULL && dmat->iommu == NULL && paddr > dmat->lowaddr && paddr <= dmat->highaddr) retval = 1; if (dmat->filter == NULL && !vm_addr_align_ok(paddr, dmat->alignment)) retval = 1; if (dmat->filter != NULL && (*dmat->filter)(dmat->filterarg, paddr) != 0) retval = 1; dmat = dmat->parent; } while (retval == 0 && dmat != NULL); return (retval); } #define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 #define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 /* * Allocate a device specific dma_tag. */ int bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat) { bus_dma_tag_t newtag; int error = 0; /* Basic sanity checking */ if (boundary != 0 && boundary < maxsegsz) maxsegsz = boundary; if (maxsegsz == 0) { return (EINVAL); } + /* Filters are no longer supported. */ + if (filter != NULL || filterarg != NULL) + return (EINVAL); + /* Return a NULL tag on failure */ *dmat = NULL; newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_ZERO | M_NOWAIT); if (newtag == NULL) { CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", __func__, newtag, 0, error); return (ENOMEM); } newtag->parent = parent; newtag->alignment = alignment; newtag->boundary = boundary; newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); newtag->filter = filter; newtag->filterarg = filterarg; newtag->maxsize = maxsize; newtag->nsegments = nsegments; newtag->maxsegsz = maxsegsz; newtag->flags = flags; newtag->ref_count = 1; /* Count ourself */ newtag->map_count = 0; if (lockfunc != NULL) { newtag->lockfunc = lockfunc; newtag->lockfuncarg = lockfuncarg; } else { newtag->lockfunc = _busdma_dflt_lock; newtag->lockfuncarg = NULL; } /* Take into account any restrictions imposed by our parent tag */ if (parent != NULL) { newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); if (newtag->boundary == 0) newtag->boundary = parent->boundary; else if (parent->boundary != 0) newtag->boundary = MIN(parent->boundary, newtag->boundary); if (newtag->filter == NULL) { /* * Short circuit looking at our parent directly * since we have encapsulated all of its information */ newtag->filter = parent->filter; newtag->filterarg = parent->filterarg; newtag->parent = parent->parent; } if (newtag->parent != NULL) atomic_add_int(&parent->ref_count, 1); newtag->iommu = parent->iommu; newtag->iommu_cookie = parent->iommu_cookie; } if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) && newtag->iommu == NULL) newtag->flags |= BUS_DMA_COULD_BOUNCE; if (newtag->alignment > 1) newtag->flags |= BUS_DMA_COULD_BOUNCE; if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && (flags & BUS_DMA_ALLOCNOW) != 0) { struct bounce_zone *bz; /* Must bounce */ if ((error = alloc_bounce_zone(newtag)) != 0) { free(newtag, M_DEVBUF); return (error); } bz = newtag->bounce_zone; if (ptoa(bz->total_bpages) < maxsize) { int pages; pages = atop(maxsize) - bz->total_bpages; /* Add pages to our bounce pool */ if (alloc_bounce_pages(newtag, pages) < pages) error = ENOMEM; } /* Performed initial allocation */ newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; } if (error != 0) { free(newtag, M_DEVBUF); } else { *dmat = newtag; } CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); return (error); } void bus_dma_template_clone(bus_dma_template_t *t, bus_dma_tag_t dmat) { if (t == NULL || dmat == NULL) return; t->parent = dmat->parent; t->alignment = dmat->alignment; t->boundary = dmat->boundary; t->lowaddr = dmat->lowaddr; t->highaddr = dmat->highaddr; t->maxsize = dmat->maxsize; t->nsegments = dmat->nsegments; t->maxsegsize = dmat->maxsegsz; t->flags = dmat->flags; t->lockfunc = dmat->lockfunc; t->lockfuncarg = dmat->lockfuncarg; } int bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) { return (0); } int bus_dma_tag_destroy(bus_dma_tag_t dmat) { bus_dma_tag_t dmat_copy __unused; int error; error = 0; dmat_copy = dmat; if (dmat != NULL) { if (dmat->map_count != 0) { error = EBUSY; goto out; } while (dmat != NULL) { bus_dma_tag_t parent; parent = dmat->parent; atomic_subtract_int(&dmat->ref_count, 1); if (dmat->ref_count == 0) { free(dmat, M_DEVBUF); /* * Last reference count, so * release our reference * count on our parent. */ dmat = parent; } else dmat = NULL; } } out: CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); return (error); } /* * Allocate a handle for mapping from kva/uva/physical * address space into bus device space. */ int bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) { int error; error = 0; *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, M_NOWAIT | M_ZERO); if (*mapp == NULL) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); return (ENOMEM); } /* * Bouncing might be required if the driver asks for an active * exclusion region, a data alignment that is stricter than 1, and/or * an active address boundary. */ if (dmat->flags & BUS_DMA_COULD_BOUNCE) { /* Must bounce */ struct bounce_zone *bz; int maxpages; if (dmat->bounce_zone == NULL) { if ((error = alloc_bounce_zone(dmat)) != 0) return (error); } bz = dmat->bounce_zone; /* Initialize the new map */ STAILQ_INIT(&((*mapp)->bpages)); /* * Attempt to add pages to our pool on a per-instance * basis up to a sane limit. */ if (dmat->alignment > 1) maxpages = MAX_BPAGES; else maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr)); if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { int pages; pages = MAX(atop(dmat->maxsize), 1); pages = MIN(maxpages - bz->total_bpages, pages); pages = MAX(pages, 1); if (alloc_bounce_pages(dmat, pages) < pages) error = ENOMEM; if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { if (error == 0) dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; } else { error = 0; } } bz->map_count++; } (*mapp)->nsegs = 0; (*mapp)->segments = (bus_dma_segment_t *)malloc( sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, M_NOWAIT); if ((*mapp)->segments == NULL) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); return (ENOMEM); } if (error == 0) dmat->map_count++; CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->flags, error); return (error); } /* * Destroy a handle for mapping from kva/uva/physical * address space into bus device space. */ int bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) { if (dmat->flags & BUS_DMA_COULD_BOUNCE) { if (STAILQ_FIRST(&map->bpages) != NULL) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, EBUSY); return (EBUSY); } if (dmat->bounce_zone) dmat->bounce_zone->map_count--; } free(map->segments, M_DEVBUF); free(map, M_DEVBUF); dmat->map_count--; CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); return (0); } /* * Allocate a piece of memory that can be efficiently mapped into * bus device space based on the constraints lited in the dma tag. * A dmamap to for use with dmamap_load is also allocated. */ int bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, bus_dmamap_t *mapp) { vm_memattr_t attr; int mflags; if (flags & BUS_DMA_NOWAIT) mflags = M_NOWAIT; else mflags = M_WAITOK; bus_dmamap_create(dmat, flags, mapp); if (flags & BUS_DMA_ZERO) mflags |= M_ZERO; if (flags & BUS_DMA_NOCACHE) attr = VM_MEMATTR_UNCACHEABLE; else attr = VM_MEMATTR_DEFAULT; /* * XXX: * (dmat->alignment <= dmat->maxsize) is just a quick hack; the exact * alignment guarantees of malloc need to be nailed down, and the * code below should be rewritten to take that into account. * * In the meantime, we'll warn the user if malloc gets it wrong. */ if ((dmat->maxsize <= PAGE_SIZE) && (dmat->alignment <= dmat->maxsize) && dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem) && attr == VM_MEMATTR_DEFAULT) { *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags); } else { /* * XXX Use Contigmalloc until it is merged into this facility * and handles multi-seg allocations. Nobody is doing * multi-seg allocations yet though. * XXX Certain AGP hardware does. */ *vaddr = kmem_alloc_contig(dmat->maxsize, mflags, 0ul, dmat->lowaddr, dmat->alignment ? dmat->alignment : 1ul, dmat->boundary, attr); (*mapp)->contigalloc = 1; } if (*vaddr == NULL) { CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->flags, ENOMEM); return (ENOMEM); } else if (!vm_addr_align_ok(vtophys(*vaddr), dmat->alignment)) { printf("bus_dmamem_alloc failed to align memory properly.\n"); } CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->flags, 0); return (0); } /* * Free a piece of memory and it's allociated dmamap, that was allocated * via bus_dmamem_alloc. Make the same choice for free/contigfree. */ void bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) { if (!map->contigalloc) free(vaddr, M_DEVBUF); else kmem_free(vaddr, dmat->maxsize); bus_dmamap_destroy(dmat, map); CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); } static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, bus_size_t buflen, int flags) { bus_addr_t curaddr; bus_size_t sgsize; if (map->pagesneeded == 0) { CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem), dmat->boundary, dmat->alignment); CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded); /* * Count the number of bounce pages * needed in order to complete this transfer */ curaddr = buf; while (buflen != 0) { sgsize = MIN(buflen, dmat->maxsegsz); if (run_filter(dmat, curaddr) != 0) { sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); map->pagesneeded++; } curaddr += sgsize; buflen -= sgsize; } CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); } } static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, void *buf, bus_size_t buflen, int flags) { vm_offset_t vaddr; vm_offset_t vendaddr; bus_addr_t paddr; if (map->pagesneeded == 0) { CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem), dmat->boundary, dmat->alignment); CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded); /* * Count the number of bounce pages * needed in order to complete this transfer */ vaddr = (vm_offset_t)buf; vendaddr = (vm_offset_t)buf + buflen; while (vaddr < vendaddr) { bus_size_t sg_len; sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK); if (pmap == kernel_pmap) paddr = pmap_kextract(vaddr); else paddr = pmap_extract(pmap, vaddr); if (run_filter(dmat, paddr) != 0) { sg_len = roundup2(sg_len, dmat->alignment); map->pagesneeded++; } vaddr += sg_len; } CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); } } /* * Add a single contiguous physical range to the segment list. */ static int _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) { int seg; /* * Make sure we don't cross any boundaries. */ if (!vm_addr_bound_ok(curaddr, sgsize, dmat->boundary)) sgsize = roundup2(curaddr, dmat->boundary) - curaddr; /* * Insert chunk into a segment, coalescing with * previous segment if possible. */ seg = *segp; if (seg == -1) { seg = 0; segs[seg].ds_addr = curaddr; segs[seg].ds_len = sgsize; } else { if (curaddr == segs[seg].ds_addr + segs[seg].ds_len && (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && vm_addr_bound_ok(segs[seg].ds_addr, segs[seg].ds_len + sgsize, dmat->boundary)) segs[seg].ds_len += sgsize; else { if (++seg >= dmat->nsegments) return (0); segs[seg].ds_addr = curaddr; segs[seg].ds_len = sgsize; } } *segp = seg; return (sgsize); } /* * Utility function to load a physical buffer. segp contains * the starting segment on entrace, and the ending segment on exit. */ int _bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) { bus_addr_t curaddr; bus_size_t sgsize; int error; if (segs == NULL) segs = map->segments; if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); if (map->pagesneeded != 0) { error = _bus_dmamap_reserve_pages(dmat, map, flags); if (error) return (error); } } while (buflen > 0) { curaddr = buf; sgsize = MIN(buflen, dmat->maxsegsz); if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) { sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); curaddr = add_bounce_page(dmat, map, 0, curaddr, sgsize); } sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, segp); if (sgsize == 0) break; buf += sgsize; buflen -= sgsize; } /* * Did we fit? */ return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ } int _bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs, int *segp) { return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, segs, segp)); } /* * Utility function to load a linear buffer. segp contains * the starting segment on entrance, and the ending segment on exit. */ int _bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, int *segp) { bus_size_t sgsize; bus_addr_t curaddr; vm_offset_t kvaddr, vaddr; int error; if (segs == NULL) segs = map->segments; if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); if (map->pagesneeded != 0) { error = _bus_dmamap_reserve_pages(dmat, map, flags); if (error) return (error); } } vaddr = (vm_offset_t)buf; while (buflen > 0) { bus_size_t max_sgsize; /* * Get the physical address for this segment. */ if (pmap == kernel_pmap) { curaddr = pmap_kextract(vaddr); kvaddr = vaddr; } else { curaddr = pmap_extract(pmap, vaddr); kvaddr = 0; } /* * Compute the segment size, and adjust counts. */ max_sgsize = MIN(buflen, dmat->maxsegsz); sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) { sgsize = roundup2(sgsize, dmat->alignment); sgsize = MIN(sgsize, max_sgsize); curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, sgsize); } else { sgsize = MIN(sgsize, max_sgsize); } sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, segp); if (sgsize == 0) break; vaddr += sgsize; buflen -= sgsize; } /* * Did we fit? */ return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ } void _bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) { if (dmat->flags & BUS_DMA_COULD_BOUNCE) { map->dmat = dmat; map->mem = *mem; map->callback = callback; map->callback_arg = callback_arg; } } bus_dma_segment_t * _bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int error) { map->nsegs = nsegs; if (segs != NULL) memcpy(map->segments, segs, map->nsegs*sizeof(segs[0])); if (dmat->iommu != NULL) IOMMU_MAP(dmat->iommu, map->segments, &map->nsegs, dmat->lowaddr, dmat->highaddr, dmat->alignment, dmat->boundary, dmat->iommu_cookie); if (segs != NULL) memcpy(segs, map->segments, map->nsegs*sizeof(segs[0])); else segs = map->segments; return (segs); } /* * Release the mapping held by map. */ void bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) { if (dmat->iommu) { IOMMU_UNMAP(dmat->iommu, map->segments, map->nsegs, dmat->iommu_cookie); map->nsegs = 0; } free_bounce_pages(dmat, map); } void bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { struct bounce_page *bpage; vm_offset_t datavaddr, tempvaddr; if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { /* * Handle data bouncing. We might also * want to add support for invalidating * the caches on broken hardware */ CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " "performing bounce", __func__, dmat, dmat->flags, op); if (op & BUS_DMASYNC_PREWRITE) { while (bpage != NULL) { tempvaddr = 0; datavaddr = bpage->datavaddr; if (datavaddr == 0) { tempvaddr = pmap_quick_enter_page( bpage->datapage); datavaddr = tempvaddr | bpage->dataoffs; } bcopy((void *)datavaddr, (void *)bpage->vaddr, bpage->datacount); if (tempvaddr != 0) pmap_quick_remove_page(tempvaddr); bpage = STAILQ_NEXT(bpage, links); } dmat->bounce_zone->total_bounced++; } if (op & BUS_DMASYNC_POSTREAD) { while (bpage != NULL) { tempvaddr = 0; datavaddr = bpage->datavaddr; if (datavaddr == 0) { tempvaddr = pmap_quick_enter_page( bpage->datapage); datavaddr = tempvaddr | bpage->dataoffs; } bcopy((void *)bpage->vaddr, (void *)datavaddr, bpage->datacount); if (tempvaddr != 0) pmap_quick_remove_page(tempvaddr); bpage = STAILQ_NEXT(bpage, links); } dmat->bounce_zone->total_bounced++; } } powerpc_sync(); } int bus_dma_tag_set_iommu(bus_dma_tag_t tag, device_t iommu, void *cookie) { tag->iommu = iommu; tag->iommu_cookie = cookie; return (0); } diff --git a/sys/riscv/riscv/busdma_machdep.c b/sys/riscv/riscv/busdma_machdep.c index 711717661908..712aad4cb5c4 100644 --- a/sys/riscv/riscv/busdma_machdep.c +++ b/sys/riscv/riscv/busdma_machdep.c @@ -1,214 +1,218 @@ /*- * Copyright (c) 1997, 1998 Justin T. Gibbs. * Copyright (c) 2013, 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Portions of this software were developed by Semihalf * under sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Return true if a match is made. * * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. * * If paddr is within the bounds of the dma tag then call the filter callback * to check for a match, if there is no filter callback then assume a match. */ int bus_dma_run_filter(struct bus_dma_tag_common *tc, bus_addr_t paddr) { int retval; retval = 0; do { if (((paddr > tc->lowaddr && paddr <= tc->highaddr) || !vm_addr_align_ok(paddr, tc->alignment)) && (tc->filter == NULL || (*tc->filter)(tc->filterarg, paddr) != 0)) retval = 1; tc = tc->parent; } while (retval == 0 && tc != NULL); return (retval); } int common_bus_dma_tag_create(struct bus_dma_tag_common *parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, size_t sz, void **dmat) { void *newtag; struct bus_dma_tag_common *common; KASSERT(sz >= sizeof(struct bus_dma_tag_common), ("sz")); /* Return a NULL tag on failure */ *dmat = NULL; /* Basic sanity checking */ if (boundary != 0 && boundary < maxsegsz) maxsegsz = boundary; if (maxsegsz == 0) return (EINVAL); newtag = malloc(sz, M_DEVBUF, M_ZERO | M_NOWAIT); if (newtag == NULL) { CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", __func__, newtag, 0, ENOMEM); return (ENOMEM); } common = newtag; common->impl = &bus_dma_bounce_impl; common->parent = parent; common->alignment = alignment; common->boundary = boundary; common->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); common->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); common->filter = filter; common->filterarg = filterarg; common->maxsize = maxsize; common->nsegments = nsegments; common->maxsegsz = maxsegsz; common->flags = flags; common->ref_count = 1; /* Count ourself */ if (lockfunc != NULL) { common->lockfunc = lockfunc; common->lockfuncarg = lockfuncarg; } else { common->lockfunc = _busdma_dflt_lock; common->lockfuncarg = NULL; } /* Take into account any restrictions imposed by our parent tag */ if (parent != NULL) { common->impl = parent->impl; common->lowaddr = MIN(parent->lowaddr, common->lowaddr); common->highaddr = MAX(parent->highaddr, common->highaddr); if (common->boundary == 0) common->boundary = parent->boundary; else if (parent->boundary != 0) { common->boundary = MIN(parent->boundary, common->boundary); } if (common->filter == NULL) { /* * Short circuit looking at our parent directly * since we have encapsulated all of its information */ common->filter = parent->filter; common->filterarg = parent->filterarg; common->parent = parent->parent; } atomic_add_int(&parent->ref_count, 1); } *dmat = common; return (0); } /* * Allocate a device specific dma_tag. */ int bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat) { struct bus_dma_tag_common *tc; int error; + /* Filters are no longer supported. */ + if (filter != NULL || filterarg != NULL) + return (EINVAL); + if (parent == NULL) { error = bus_dma_bounce_impl.tag_create(parent, alignment, boundary, lowaddr, highaddr, filter, filterarg, maxsize, nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat); } else { tc = (struct bus_dma_tag_common *)parent; error = tc->impl->tag_create(parent, alignment, boundary, lowaddr, highaddr, filter, filterarg, maxsize, nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat); } return (error); } void bus_dma_template_clone(bus_dma_template_t *t, bus_dma_tag_t dmat) { struct bus_dma_tag_common *common; if (t == NULL || dmat == NULL) return; common = (struct bus_dma_tag_common *)dmat; t->parent = (bus_dma_tag_t)common->parent; t->alignment = common->alignment; t->boundary = common->boundary; t->lowaddr = common->lowaddr; t->highaddr = common->highaddr; t->maxsize = common->maxsize; t->nsegments = common->nsegments; t->maxsegsize = common->maxsegsz; t->flags = common->flags; t->lockfunc = common->lockfunc; t->lockfuncarg = common->lockfuncarg; } int bus_dma_tag_destroy(bus_dma_tag_t dmat) { struct bus_dma_tag_common *tc; tc = (struct bus_dma_tag_common *)dmat; return (tc->impl->tag_destroy(dmat)); } int bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) { return (0); } diff --git a/sys/x86/x86/busdma_machdep.c b/sys/x86/x86/busdma_machdep.c index 0e72b09684cc..40afcee0651e 100644 --- a/sys/x86/x86/busdma_machdep.c +++ b/sys/x86/x86/busdma_machdep.c @@ -1,230 +1,234 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1997, 1998 Justin T. Gibbs. * Copyright (c) 2013 The FreeBSD Foundation * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "opt_acpi.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Return true if a match is made. * * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. * * If paddr is within the bounds of the dma tag then call the filter callback * to check for a match, if there is no filter callback then assume a match. */ int bus_dma_run_filter(struct bus_dma_tag_common *tc, vm_paddr_t paddr) { int retval; retval = 0; do { if ((paddr >= BUS_SPACE_MAXADDR || (paddr > tc->lowaddr && paddr <= tc->highaddr) || !vm_addr_align_ok(paddr, tc->alignment)) && (tc->filter == NULL || (*tc->filter)(tc->filterarg, paddr) != 0)) retval = 1; tc = tc->parent; } while (retval == 0 && tc != NULL); return (retval); } int common_bus_dma_tag_create(struct bus_dma_tag_common *parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, size_t sz, void **dmat) { void *newtag; struct bus_dma_tag_common *common; KASSERT(sz >= sizeof(struct bus_dma_tag_common), ("sz")); /* Basic sanity checking */ if (boundary != 0 && boundary < maxsegsz) maxsegsz = boundary; if (maxsegsz == 0) return (EINVAL); /* Return a NULL tag on failure */ *dmat = NULL; newtag = malloc(sz, M_DEVBUF, M_ZERO | M_NOWAIT); if (newtag == NULL) { CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", __func__, newtag, 0, ENOMEM); return (ENOMEM); } common = newtag; common->impl = &bus_dma_bounce_impl; common->parent = parent; common->alignment = alignment; common->boundary = boundary; common->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); common->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); common->filter = filter; common->filterarg = filterarg; common->maxsize = maxsize; common->nsegments = nsegments; common->maxsegsz = maxsegsz; common->flags = flags; common->ref_count = 1; /* Count ourself */ if (lockfunc != NULL) { common->lockfunc = lockfunc; common->lockfuncarg = lockfuncarg; } else { common->lockfunc = _busdma_dflt_lock; common->lockfuncarg = NULL; } /* Take into account any restrictions imposed by our parent tag */ if (parent != NULL) { common->impl = parent->impl; common->lowaddr = MIN(parent->lowaddr, common->lowaddr); common->highaddr = MAX(parent->highaddr, common->highaddr); if (common->boundary == 0) common->boundary = parent->boundary; else if (parent->boundary != 0) { common->boundary = MIN(parent->boundary, common->boundary); } if (common->filter == NULL) { /* * Short circuit looking at our parent directly * since we have encapsulated all of its information */ common->filter = parent->filter; common->filterarg = parent->filterarg; common->parent = parent->parent; } common->domain = parent->domain; atomic_add_int(&parent->ref_count, 1); } common->domain = vm_phys_domain_match(common->domain, 0ul, common->lowaddr); *dmat = common; return (0); } int bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) { struct bus_dma_tag_common *tc; tc = (struct bus_dma_tag_common *)dmat; domain = vm_phys_domain_match(domain, 0ul, tc->lowaddr); /* Only call the callback if it changes. */ if (domain == tc->domain) return (0); tc->domain = domain; return (tc->impl->tag_set_domain(dmat)); } /* * Allocate a device specific dma_tag. */ int bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat) { struct bus_dma_tag_common *tc; int error; + /* Filters are no longer supported. */ + if (filter != NULL || filterarg != NULL) + return (EINVAL); + if (parent == NULL) { error = bus_dma_bounce_impl.tag_create(parent, alignment, boundary, lowaddr, highaddr, filter, filterarg, maxsize, nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat); } else { tc = (struct bus_dma_tag_common *)parent; error = tc->impl->tag_create(parent, alignment, boundary, lowaddr, highaddr, filter, filterarg, maxsize, nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat); } return (error); } void bus_dma_template_clone(bus_dma_template_t *t, bus_dma_tag_t dmat) { struct bus_dma_tag_common *common; if (t == NULL || dmat == NULL) return; common = (struct bus_dma_tag_common *)dmat; t->parent = (bus_dma_tag_t)common->parent; t->alignment = common->alignment; t->boundary = common->boundary; t->lowaddr = common->lowaddr; t->highaddr = common->highaddr; t->maxsize = common->maxsize; t->nsegments = common->nsegments; t->maxsegsize = common->maxsegsz; t->flags = common->flags; t->lockfunc = common->lockfunc; t->lockfuncarg = common->lockfuncarg; } int bus_dma_tag_destroy(bus_dma_tag_t dmat) { struct bus_dma_tag_common *tc; tc = (struct bus_dma_tag_common *)dmat; return (tc->impl->tag_destroy(dmat)); }