Index: head/sys/contrib/ncsw/inc/xx_ext.h =================================================================== --- head/sys/contrib/ncsw/inc/xx_ext.h (revision 308717) +++ head/sys/contrib/ncsw/inc/xx_ext.h (revision 308718) @@ -1,938 +1,933 @@ /* Copyright (c) 2008-2011 Freescale Semiconductor, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of Freescale Semiconductor nor the * names of its contributors may be used to endorse or promote products * derived from this software without specific prior written permission. * * * ALTERNATIVELY, this software may be distributed under the terms of the * GNU General Public License ("GPL") as published by the Free Software * Foundation, either version 2 of that License or (at your option) any * later version. * * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /**************************************************************************//** @File xx_ext.h @Description Prototypes, externals and typedefs for system-supplied (external) routines *//***************************************************************************/ #ifndef __XX_EXT_H #define __XX_EXT_H #include "std_ext.h" #include "part_ext.h" #if defined(__MWERKS__) && defined(OPTIMIZED_FOR_SPEED) #include "xx_integration_ext.h" #endif /* defined(__MWERKS__) && defined(OPTIMIZED_FOR_SPEED) */ /**************************************************************************//** @Group xx_id XX Interface (System call hooks) @Description Prototypes, externals and typedefs for system-supplied (external) routines @{ *//***************************************************************************/ #if (defined(REPORT_EVENTS) && (REPORT_EVENTS > 0)) /**************************************************************************//** @Function XX_EventById @Description Event reporting routine - executed only when REPORT_EVENTS=1. @Param[in] event - Event code (e_Event). @Param[in] appId - Application identifier. @Param[in] flags - Event flags. @Param[in] msg - Event message. @Return None *//***************************************************************************/ void XX_EventById(uint32_t event, t_Handle appId, uint16_t flags, char *msg); #else /* not REPORT_EVENTS */ #define XX_EventById(event, appId, flags, msg) #endif /* REPORT_EVENTS */ #ifdef DEBUG_XX_MALLOC void * XX_MallocDebug(uint32_t size, char *fname, int line); void * XX_MallocSmartDebug(uint32_t size, int memPartitionId, uint32_t alignment, char *fname, int line); #define XX_Malloc(sz) \ XX_MallocDebug((sz), __FILE__, __LINE__) #define XX_MallocSmart(sz, memt, al) \ XX_MallocSmartDebug((sz), (memt), (al), __FILE__, __LINE__) #else /* not DEBUG_XX_MALLOC */ /**************************************************************************//** @Function XX_Malloc @Description allocates contiguous block of memory. @Param[in] size - Number of bytes to allocate. @Return The address of the newly allocated block on success, NULL on failure. *//***************************************************************************/ void * XX_Malloc(uint32_t size); /**************************************************************************//** @Function XX_MallocSmartInit @Description Initializes SmartMalloc allocator. @Return E_OK on success, error code otherwise. *//***************************************************************************/ int XX_MallocSmartInit(void); /**************************************************************************//** @Function XX_MallocSmart @Description Allocates contiguous block of memory in a specified alignment and from the specified segment. @Param[in] size - Number of bytes to allocate. @Param[in] memPartitionId - Memory partition ID; The value zero must be mapped to the default heap partition. @Param[in] alignment - Required memory alignment (in bytes). @Return The address of the newly allocated block on success, NULL on failure. *//***************************************************************************/ void * XX_MallocSmart(uint32_t size, int memPartitionId, uint32_t alignment); #endif /* not DEBUG_XX_MALLOC */ /**************************************************************************//** @Function XX_FreeSmart @Description Frees the memory block pointed to by "p". Only for memory allocated by XX_MallocSmart @Param[in] p_Memory - pointer to the memory block. @Return None. *//***************************************************************************/ void XX_FreeSmart(void *p_Memory); /**************************************************************************//** @Function XX_Free @Description frees the memory block pointed to by "p". @Param[in] p_Memory - pointer to the memory block. @Return None. *//***************************************************************************/ void XX_Free(void *p_Memory); #ifndef NCSW_LINUX /**************************************************************************//** @Function XX_GetMemPartitionBase @Description This routine gets the address of a memory segment according to the memory type. @Param[in] memPartitionId - Memory partition ID; The value zero must be mapped to the default heap partition. @Return The address of the required memory type. *//***************************************************************************/ void * XX_GetMemPartitionBase(int memPartitionId); #endif /**************************************************************************//** @Function XX_Print @Description print a string. @Param[in] str - string to print. @Return None. *//***************************************************************************/ void XX_Print(char *str, ...); /**************************************************************************//** @Function XX_GetChar @Description Get character from console. @Return Character is returned on success. Zero is returned otherwise. *//***************************************************************************/ char XX_GetChar(void); /**************************************************************************//** @Function XX_PreallocAndBindIntr @Description Preallocate and optionally bind it to given CPU. @Param[in] irq - Interrupt ID (system-specific number). @Param[in] cpu - CPU to bind to or -1 if iRQ should be unbound. @Return E_OK on success; error code otherwise.. *//***************************************************************************/ t_Error XX_PreallocAndBindIntr(uintptr_t irq, unsigned int cpu); /**************************************************************************//** @Function XX_DeallocIntr @Description Deallocate preallocated interupt. @Param[in] irq - Interrupt ID (system-specific number). @Return E_OK on success; error code otherwise.. *//***************************************************************************/ t_Error XX_DeallocIntr(uintptr_t irq); /**************************************************************************//** @Function XX_SetIntr @Description Set an interrupt service routine for a specific interrupt source. @Param[in] irq - Interrupt ID (system-specific number). @Param[in] f_Isr - Callback routine that will be called when the interrupt occurs. @Param[in] handle - The argument for the user callback routine. @Return E_OK on success; error code otherwise.. *//***************************************************************************/ t_Error XX_SetIntr(uintptr_t irq, t_Isr *f_Isr, t_Handle handle); /**************************************************************************//** @Function XX_FreeIntr @Description Free a specific interrupt and a specific callback routine. @Param[in] irq - Interrupt ID (system-specific number). @Return E_OK on success; error code otherwise.. *//***************************************************************************/ t_Error XX_FreeIntr(uintptr_t irq); /**************************************************************************//** @Function XX_EnableIntr @Description Enable a specific interrupt. @Param[in] irq - Interrupt ID (system-specific number). @Return E_OK on success; error code otherwise.. *//***************************************************************************/ t_Error XX_EnableIntr(uintptr_t irq); /**************************************************************************//** @Function XX_DisableIntr @Description Disable a specific interrupt. @Param[in] irq - Interrupt ID (system-specific number). @Return E_OK on success; error code otherwise.. *//***************************************************************************/ t_Error XX_DisableIntr(uintptr_t irq); #if !(defined(__MWERKS__) && defined(OPTIMIZED_FOR_SPEED)) /**************************************************************************//** @Function XX_DisableAllIntr @Description Disable all interrupts by masking them at the CPU. @Return A value that represents the interrupts state before the operation, and should be passed to the matching XX_RestoreAllIntr() call. *//***************************************************************************/ uint32_t XX_DisableAllIntr(void); /**************************************************************************//** @Function XX_RestoreAllIntr @Description Restore previous state of interrupts level at the CPU. @Param[in] flags - A value that represents the interrupts state to restore, as returned by the matching call for XX_DisableAllIntr(). @Return None. *//***************************************************************************/ void XX_RestoreAllIntr(uint32_t flags); #endif /* !(defined(__MWERKS__) && defined(OPTIMIZED_FOR_SPEED)) */ /**************************************************************************//** @Function XX_Call @Description Call a service in another task. Activate the routine "f" via the queue identified by "IntrManagerId". The parameter to "f" is Id - the handle of the destination object @Param[in] intrManagerId - Queue ID. @Param[in] f - routine pointer. @Param[in] Id - the parameter to be passed to f(). @Param[in] h_App - Application handle. @Param[in] flags - Unused, @Return E_OK is returned on success. E_FAIL is returned otherwise (usually an operating system level failure). *//***************************************************************************/ t_Error XX_Call( uint32_t intrManagerId, t_Error (* f)(t_Handle), t_Handle Id, t_Handle h_App, uint16_t flags ); /**************************************************************************//** @Function XX_Exit @Description Stop execution and report status (where it is applicable) @Param[in] status - exit status *//***************************************************************************/ void XX_Exit(int status); /*****************************************************************************/ /* Tasklet Service Routines */ /*****************************************************************************/ typedef t_Handle t_TaskletHandle; /**************************************************************************//** @Function XX_InitTasklet @Description Create and initialize a tasklet object. @Param[in] routine - A routine to be ran as a tasklet. @Param[in] data - An argument to pass to the tasklet. @Return Tasklet handle is returned on success. NULL is returned otherwise. *//***************************************************************************/ t_TaskletHandle XX_InitTasklet (void (*routine)(void *), void *data); /**************************************************************************//** @Function XX_FreeTasklet @Description Free a tasklet object. @Param[in] h_Tasklet - A handle to a tasklet to be free. @Return None. *//***************************************************************************/ void XX_FreeTasklet (t_TaskletHandle h_Tasklet); /**************************************************************************//** @Function XX_ScheduleTask @Description Schedule a tasklet object. @Param[in] h_Tasklet - A handle to a tasklet to be scheduled. @Param[in] immediate - Indicate whether to schedule this tasklet on the immediate queue or on the delayed one. @Return 0 - on success. Error code - otherwise. *//***************************************************************************/ int XX_ScheduleTask(t_TaskletHandle h_Tasklet, int immediate); /**************************************************************************//** @Function XX_FlushScheduledTasks @Description Flush all tasks there are in the scheduled tasks queue. @Return None. *//***************************************************************************/ void XX_FlushScheduledTasks(void); /**************************************************************************//** @Function XX_TaskletIsQueued @Description Check if task is queued. @Param[in] h_Tasklet - A handle to a tasklet to be scheduled. @Return 1 - task is queued. 0 - otherwise. *//***************************************************************************/ int XX_TaskletIsQueued(t_TaskletHandle h_Tasklet); /**************************************************************************//** @Function XX_SetTaskletData @Description Set data to a scheduled task. Used to change data of already scheduled task. @Param[in] h_Tasklet - A handle to a tasklet to be scheduled. @Param[in] data - Data to be set. *//***************************************************************************/ void XX_SetTaskletData(t_TaskletHandle h_Tasklet, t_Handle data); /**************************************************************************//** @Function XX_GetTaskletData @Description Get the data of scheduled task. @Param[in] h_Tasklet - A handle to a tasklet to be scheduled. @Return handle to the data of the task. *//***************************************************************************/ t_Handle XX_GetTaskletData(t_TaskletHandle h_Tasklet); /**************************************************************************//** @Function XX_BottomHalf @Description Bottom half implementation, invoked by the interrupt handler. This routine handles all bottom-half tasklets with interrupts enabled. @Return None. *//***************************************************************************/ void XX_BottomHalf(void); /*****************************************************************************/ /* Spinlock Service Routines */ /*****************************************************************************/ /**************************************************************************//** @Function XX_InitSpinlock @Description Creates a spinlock. @Return Spinlock handle is returned on success; NULL otherwise. *//***************************************************************************/ t_Handle XX_InitSpinlock(void); /**************************************************************************//** @Function XX_FreeSpinlock @Description Frees the memory allocated for the spinlock creation. @Param[in] h_Spinlock - A handle to a spinlock. @Return None. *//***************************************************************************/ void XX_FreeSpinlock(t_Handle h_Spinlock); /**************************************************************************//** @Function XX_LockSpinlock @Description Locks a spinlock. @Param[in] h_Spinlock - A handle to a spinlock. @Return None. *//***************************************************************************/ void XX_LockSpinlock(t_Handle h_Spinlock); /**************************************************************************//** @Function XX_UnlockSpinlock @Description Unlocks a spinlock. @Param[in] h_Spinlock - A handle to a spinlock. @Return None. *//***************************************************************************/ void XX_UnlockSpinlock(t_Handle h_Spinlock); /**************************************************************************//** @Function XX_LockIntrSpinlock @Description Locks a spinlock (interrupt safe). @Param[in] h_Spinlock - A handle to a spinlock. @Return A value that represents the interrupts state before the operation, and should be passed to the matching XX_UnlockIntrSpinlock() call. *//***************************************************************************/ uint32_t XX_LockIntrSpinlock(t_Handle h_Spinlock); /**************************************************************************//** @Function XX_UnlockIntrSpinlock @Description Unlocks a spinlock (interrupt safe). @Param[in] h_Spinlock - A handle to a spinlock. @Param[in] intrFlags - A value that represents the interrupts state to restore, as returned by the matching call for XX_LockIntrSpinlock(). @Return None. *//***************************************************************************/ void XX_UnlockIntrSpinlock(t_Handle h_Spinlock, uint32_t intrFlags); /*****************************************************************************/ /* Timers Service Routines */ /*****************************************************************************/ /**************************************************************************//** @Function XX_CurrentTime @Description Returns current system time. @Return Current system time (in milliseconds). *//***************************************************************************/ uint32_t XX_CurrentTime(void); /**************************************************************************//** @Function XX_CreateTimer @Description Creates a timer. @Return Timer handle is returned on success; NULL otherwise. *//***************************************************************************/ t_Handle XX_CreateTimer(void); /**************************************************************************//** @Function XX_FreeTimer @Description Frees the memory allocated for the timer creation. @Param[in] h_Timer - A handle to a timer. @Return None. *//***************************************************************************/ void XX_FreeTimer(t_Handle h_Timer); /**************************************************************************//** @Function XX_StartTimer @Description Starts a timer. The user can select to start the timer as periodic timer or as one-shot timer. The user should provide a callback routine that will be called when the timer expires. @Param[in] h_Timer - A handle to a timer. @Param[in] msecs - Timer expiration period (in milliseconds). @Param[in] periodic - TRUE for a periodic timer; FALSE for a one-shot timer.. @Param[in] f_TimerExpired - A callback routine to be called when the timer expires. @Param[in] h_Arg - The argument to pass in the timer-expired callback routine. @Return None. *//***************************************************************************/ void XX_StartTimer(t_Handle h_Timer, uint32_t msecs, bool periodic, void (*f_TimerExpired)(t_Handle h_Arg), t_Handle h_Arg); /**************************************************************************//** @Function XX_StopTimer @Description Frees the memory allocated for the timer creation. @Param[in] h_Timer - A handle to a timer. @Return None. *//***************************************************************************/ void XX_StopTimer(t_Handle h_Timer); /**************************************************************************//** @Function XX_GetExpirationTime @Description Returns the time (in milliseconds) remaining until the expiration of a timer. @Param[in] h_Timer - A handle to a timer. @Return The time left until the timer expires. *//***************************************************************************/ uint32_t XX_GetExpirationTime(t_Handle h_Timer); /**************************************************************************//** @Function XX_ModTimer @Description Updates the expiration time of a timer. This routine adds the given time to the current system time, and sets this value as the new expiration time of the timer. @Param[in] h_Timer - A handle to a timer. @Param[in] msecs - The new interval until timer expiration (in milliseconds). @Return None. *//***************************************************************************/ void XX_ModTimer(t_Handle h_Timer, uint32_t msecs); /**************************************************************************//** @Function XX_TimerIsActive @Description Checks whether a timer is active (pending) or not. @Param[in] h_Timer - A handle to a timer. @Return 0 - the timer is inactive; Non-zero value - the timer is active; *//***************************************************************************/ int XX_TimerIsActive(t_Handle h_Timer); /**************************************************************************//** @Function XX_Sleep @Description Non-busy wait until the desired time (in milliseconds) has passed. @Param[in] msecs - The requested sleep time (in milliseconds). @Return None. @Cautions This routine enables interrupts during its wait time. *//***************************************************************************/ uint32_t XX_Sleep(uint32_t msecs); /**************************************************************************//** @Function XX_UDelay @Description Busy-wait until the desired time (in microseconds) has passed. @Param[in] usecs - The requested delay time (in microseconds). @Return None. @Cautions It is highly unrecommended to call this routine during interrupt time, because the system time may not be updated properly during the delay loop. The behavior of this routine during interrupt time is unexpected. *//***************************************************************************/ void XX_UDelay(uint32_t usecs); /*****************************************************************************/ /* Other Service Routines */ /*****************************************************************************/ /**************************************************************************//** @Function XX_PhysToVirt @Description Translates a physical address to the matching virtual address. @Param[in] addr - The physical address to translate. @Return Virtual address. *//***************************************************************************/ void * XX_PhysToVirt(physAddress_t addr); /**************************************************************************//** @Function XX_VirtToPhys @Description Translates a virtual address to the matching physical address. @Param[in] addr - The virtual address to translate. @Return Physical address. *//***************************************************************************/ physAddress_t XX_VirtToPhys(void *addr); /**************************************************************************//** @Function XX_PortalSetInfo @Description Save physical and virtual adresses of the portals. @Param[in] dev - Portals device - either bman or qman. @Return Physical, virtual addresses and size. *//***************************************************************************/ void XX_PortalSetInfo(device_t dev); /**************************************************************************//** @Function XX_FmanSetIntrInfo @Description Workaround for FMan interrupt, which must be binded to one CPU only. @Param[in] irq - Interrupt number. @Return None. *//***************************************************************************/ void XX_FmanFixIntr(int irq); /**************************************************************************//** @Group xx_ipc XX Inter-Partition-Communication API @Description The following API is to be used when working with multiple partitions configuration. @{ *//***************************************************************************/ #define XX_IPC_MAX_ADDR_NAME_LENGTH 16 /**< Maximum length of an endpoint name string; The IPC service can use this constant to limit the storage space for IPC endpoint names. */ /**************************************************************************//** @Function t_IpcMsgCompletion @Description Callback function used upon IPC non-blocking transaction completion to return message buffer to the caller and to forward reply if available. This callback function may be attached by the source endpoint to any outgoing IPC message to indicate a non-blocking send (see also XX_IpcSendMessage() routine). Upon completion of an IPC transaction (consisting of a message and an optional reply), the IPC service invokes this callback routine to return the message buffer to the sender and to provide the received reply, if requested. User provides this function. Driver invokes it. @Param[in] h_Module - Abstract handle to the sending module - the same handle as was passed in the XX_IpcSendMessage() function; This handle is typically used to point to the internal data structure of the source endpoint. @Param[in] p_Msg - Pointer to original (sent) message buffer; The source endpoint can free (or reuse) this buffer when message completion callback is called. @Param[in] p_Reply - Pointer to (received) reply buffer; This pointer is the same as was provided by the source endpoint in XX_IpcSendMessage(). @Param[in] replyLength - Length (in bytes) of actual data in the reply buffer. @Param[in] status - Completion status - E_OK or failure indication, e.g. IPC transaction completion timeout. @Return None *//***************************************************************************/ typedef void (t_IpcMsgCompletion)(t_Handle h_Module, uint8_t *p_Msg, uint8_t *p_Reply, uint32_t replyLength, t_Error status); /**************************************************************************//** @Function t_IpcMsgHandler @Description Callback function used as IPC message handler. The IPC service invokes message handlers for each IPC message received. The actual function pointer should be registered by each destination endpoint via the XX_IpcRegisterMsgHandler() routine. User provides this function. Driver invokes it. @Param[in] h_Module - Abstract handle to the message handling module - the same handle as was passed in the XX_IpcRegisterMsgHandler() function; this handle is typically used to point to the internal data structure of the destination endpoint. @Param[in] p_Msg - Pointer to message buffer with data received from peer. @Param[in] msgLength - Length (in bytes) of message data. @Param[in] p_Reply - Pointer to reply buffer, to be filled by the message handler and then sent by the IPC service; The reply buffer is allocated by the IPC service with size equals to the replyLength parameter provided in message handler registration (see XX_IpcRegisterMsgHandler() function); If replyLength was initially specified as zero during message handler registration, the IPC service may set this pointer to NULL and assume that a reply is not needed; The IPC service is also responsible for freeing the reply buffer after the reply has been sent or dismissed. @Param[in,out] p_ReplyLength - Pointer to reply length, which has a dual role in this function: [In] equals the replyLength parameter provided in message handler registration (see XX_IpcRegisterMsgHandler() function), and [Out] should be updated by message handler to the actual reply length; if this value is set to zero, the IPC service must assume that a reply should not be sent; Note: If p_Reply is not NULL, p_ReplyLength must not be NULL as well. @Return E_OK on success; Error code otherwise. *//***************************************************************************/ typedef t_Error (t_IpcMsgHandler)(t_Handle h_Module, uint8_t *p_Msg, uint32_t msgLength, uint8_t *p_Reply, uint32_t *p_ReplyLength); /**************************************************************************//** @Function XX_IpcRegisterMsgHandler @Description IPC mailbox registration. This function is used for registering an IPC message handler in the IPC service. This function is called by each destination endpoint to indicate that it is ready to handle incoming messages. The IPC service invokes the message handler upon receiving a message addressed to the specified destination endpoint. @Param[in] addr - The address name string associated with the destination endpoint; This address must be unique across the IPC service domain to ensure correct message routing. @Param[in] f_MsgHandler - Pointer to the message handler callback for processing incoming message; invoked by the IPC service upon receiving a message addressed to the destination endpoint specified by the addr parameter. @Param[in] h_Module - Abstract handle to the message handling module, passed unchanged to f_MsgHandler callback function. @Param[in] replyLength - The maximal data length (in bytes) of any reply that the specified message handler may generate; the IPC service provides the message handler with buffer for reply according to the length specified here (refer also to the description of #t_IpcMsgHandler callback function type); This size shall be zero if the message handler never generates replies. @Return E_OK on success; Error code otherwise. *//***************************************************************************/ t_Error XX_IpcRegisterMsgHandler(char addr[XX_IPC_MAX_ADDR_NAME_LENGTH], t_IpcMsgHandler *f_MsgHandler, t_Handle h_Module, uint32_t replyLength); /**************************************************************************//** @Function XX_IpcUnregisterMsgHandler @Description Release IPC mailbox routine. This function is used for unregistering an IPC message handler from the IPC service. This function is called by each destination endpoint to indicate that it is no longer capable of handling incoming messages. @Param[in] addr - The address name string associated with the destination endpoint; This address is the same as was used when the message handler was registered via XX_IpcRegisterMsgHandler(). @Return E_OK on success; Error code otherwise. *//***************************************************************************/ t_Error XX_IpcUnregisterMsgHandler(char addr[XX_IPC_MAX_ADDR_NAME_LENGTH]); /**************************************************************************//** @Function XX_IpcInitSession @Description This function is used for creating an IPC session between the source endpoint and the destination endpoint. The actual implementation and representation of a session is left for the IPC service. The function returns an abstract handle to the created session. This handle shall be used by the source endpoint in subsequent calls to XX_IpcSendMessage(). The IPC service assumes that before this function is called, no messages are sent from the specified source endpoint to the specified destination endpoint. The IPC service may use a connection-oriented approach or a connectionless approach (or both) as described below. @par Connection-Oriented Approach The IPC service may implement a session in a connection-oriented approach - when this function is called, the IPC service should take the necessary steps to bring up a source-to-destination channel for messages and a destination-to-source channel for replies. The returned handle should represent the internal representation of these channels. @par Connectionless Approach The IPC service may implement a session in a connectionless approach - when this function is called, the IPC service should not perform any particular steps, but it must store the pair of source and destination addresses in some session representation and return it as a handle. When XX_IpcSendMessage() shall be called, the IPC service may use this handle to provide the necessary identifiers for routing the messages through the connectionless medium. @Param[in] destAddr - The address name string associated with the destination endpoint. @Param[in] srcAddr - The address name string associated with the source endpoint. @Return Abstract handle to the initialized session, or NULL on error. *//***************************************************************************/ t_Handle XX_IpcInitSession(char destAddr[XX_IPC_MAX_ADDR_NAME_LENGTH], char srcAddr[XX_IPC_MAX_ADDR_NAME_LENGTH]); /**************************************************************************//** @Function XX_IpcFreeSession @Description This function is used for terminating an existing IPC session between a source endpoint and a destination endpoint. The IPC service assumes that after this function is called, no messages shall be sent from the associated source endpoint to the associated destination endpoint. @Param[in] h_Session - Abstract handle to the IPC session - the same handle as was originally returned by the XX_IpcInitSession() function. @Return E_OK on success; Error code otherwise. *//***************************************************************************/ t_Error XX_IpcFreeSession(t_Handle h_Session); /**************************************************************************//** @Function XX_IpcSendMessage @Description IPC message send routine. This function may be used by a source endpoint to send an IPC message to a destination endpoint. The source endpoint cannot send a message to the destination endpoint without first initiating a session with that destination endpoint via XX_IpcInitSession() routine. The source endpoint must provide the buffer pointer and length of the outgoing message. Optionally, it may also provide a buffer for an expected reply. In the latter case, the transaction is not considered complete by the IPC service until the reply has been received. If the source endpoint does not provide a reply buffer, the transaction is considered complete after the message has been sent. The source endpoint must keep the message (and optional reply) buffers valid until the transaction is complete. @par Non-blocking mode The source endpoint may request a non-blocking send by providing a non-NULL pointer to a message completion callback function (f_Completion). Upon completion of the IPC transaction (consisting of a message and an optional reply), the IPC service invokes this callback routine to return the message buffer to the sender and to provide the received reply, if requested. @par Blocking mode The source endpoint may request a blocking send by setting f_Completion to NULL. The function is expected to block until the IPC transaction is complete - either the reply has been received or (if no reply was requested) the message has been sent. @Param[in] h_Session - Abstract handle to the IPC session - the same handle as was originally returned by the XX_IpcInitSession() function. @Param[in] p_Msg - Pointer to message buffer to send. @Param[in] msgLength - Length (in bytes) of actual data in the message buffer. @Param[in] p_Reply - Pointer to reply buffer - if this buffer is not NULL, the IPC service fills this buffer with the received reply data; In blocking mode, the reply data must be valid when the function returns; In non-blocking mode, the reply data is valid when f_Completion is called; If this pointer is NULL, no reply is expected. @Param[in,out] p_ReplyLength - Pointer to reply length, which has a dual role in this function: [In] specifies the maximal length (in bytes) of the reply buffer pointed by p_Reply, and [Out] in non-blocking mode this value is updated by the IPC service to the actual reply length (in bytes). @Param[in] f_Completion - Pointer to a completion callback to be used in non-blocking send mode; The completion callback is invoked by the IPC service upon completion of the IPC transaction (consisting of a message and an optional reply); If this pointer is NULL, the function is expected to block until the IPC transaction is complete. @Param[in] h_Arg - Abstract handle to the sending module; passed unchanged to the f_Completion callback function as the first argument. @Return E_OK on success; Error code otherwise. *//***************************************************************************/ t_Error XX_IpcSendMessage(t_Handle h_Session, uint8_t *p_Msg, uint32_t msgLength, uint8_t *p_Reply, uint32_t *p_ReplyLength, t_IpcMsgCompletion *f_Completion, t_Handle h_Arg); /** @} */ /* end of xx_ipc group */ /** @} */ /* end of xx_id group */ -/** FreeBSD Specific additions. */ -void XX_TrackInit(void); -physAddress_t XX_TrackAddress(void *addr); -void XX_UntrackAddress(void *addr); - #endif /* __XX_EXT_H */ Index: head/sys/contrib/ncsw/user/env/xx.c =================================================================== --- head/sys/contrib/ncsw/user/env/xx.c (revision 308717) +++ head/sys/contrib/ncsw/user/env/xx.c (revision 308718) @@ -1,948 +1,876 @@ /*- * Copyright (c) 2011 Semihalf. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include "error_ext.h" #include "std_ext.h" #include "list_ext.h" #include "mm_ext.h" /* Configuration */ /* Define the number of dTSEC ports active in system */ #define MALLOCSMART_DTSEC_IN_USE 4 /* * Calculate malloc's pool size for dTSEC's buffers. * We reserve 1MB pool for each dTSEC port. */ #define MALLOCSMART_POOL_SIZE \ (MALLOCSMART_DTSEC_IN_USE * 1024 * 1024) #define MALLOCSMART_SLICE_SIZE (PAGE_SIZE / 2) /* 2kB */ /* Defines */ #define MALLOCSMART_SIZE_TO_SLICE(x) \ (((x) + MALLOCSMART_SLICE_SIZE - 1) / MALLOCSMART_SLICE_SIZE) #define MALLOCSMART_SLICES \ MALLOCSMART_SIZE_TO_SLICE(MALLOCSMART_POOL_SIZE) /* Malloc Pool for NetCommSW */ MALLOC_DEFINE(M_NETCOMMSW, "NetCommSW", "NetCommSW software stack"); MALLOC_DEFINE(M_NETCOMMSW_MT, "NetCommSWTrack", "NetCommSW software allocation tracker"); /* MallocSmart data structures */ static void *XX_MallocSmartPool; static int XX_MallocSmartMap[MALLOCSMART_SLICES]; static struct mtx XX_MallocSmartLock; static struct mtx XX_MallocTrackLock; MTX_SYSINIT(XX_MallocSmartLockInit, &XX_MallocSmartLock, "NetCommSW MallocSmart Lock", MTX_DEF); MTX_SYSINIT(XX_MallocTrackLockInit, &XX_MallocTrackLock, "NetCommSW MallocTrack Lock", MTX_DEF); /* Interrupt info */ #define XX_INTR_FLAG_PREALLOCATED (1 << 0) #define XX_INTR_FLAG_BOUND (1 << 1) #define XX_INTR_FLAG_FMAN_FIX (1 << 2) struct XX_IntrInfo { driver_intr_t *handler; void *arg; int cpu; int flags; void *cookie; }; static struct XX_IntrInfo XX_IntrInfo[INTR_VECTORS]; /* Portal type identifiers */ enum XX_PortalIdent{ BM_PORTAL = 0, QM_PORTAL, }; /* Structure to store portals' properties */ struct XX_PortalInfo { vm_paddr_t portal_ce_pa[2][MAXCPU]; vm_paddr_t portal_ci_pa[2][MAXCPU]; uint32_t portal_ce_size[2][MAXCPU]; uint32_t portal_ci_size[2][MAXCPU]; vm_offset_t portal_ce_va[2]; vm_offset_t portal_ci_va[2]; uint32_t portal_intr[2][MAXCPU]; }; static struct XX_PortalInfo XX_PInfo; -/* The lower 9 bits, through emprical testing, tend to be 0. */ -#define XX_MALLOC_TRACK_SHIFT 9 - -typedef struct XX_MallocTrackStruct { - LIST_ENTRY(XX_MallocTrackStruct) entries; - physAddress_t pa; - void *va; -} XX_MallocTrackStruct; - -LIST_HEAD(XX_MallocTrackerList, XX_MallocTrackStruct) *XX_MallocTracker; -u_long XX_MallocHashMask; -static XX_MallocTrackStruct * XX_FindTracker(physAddress_t pa); - void XX_Exit(int status) { panic("NetCommSW: Exit called with status %i", status); } void XX_Print(char *str, ...) { va_list ap; va_start(ap, str); vprintf(str, ap); va_end(ap); } void * XX_Malloc(uint32_t size) { void *p = (malloc(size, M_NETCOMMSW, M_NOWAIT)); return (p); } static int XX_MallocSmartMapCheck(unsigned int start, unsigned int slices) { unsigned int i; mtx_assert(&XX_MallocSmartLock, MA_OWNED); for (i = start; i < start + slices; i++) if (XX_MallocSmartMap[i]) return (FALSE); return (TRUE); } static void XX_MallocSmartMapSet(unsigned int start, unsigned int slices) { unsigned int i; mtx_assert(&XX_MallocSmartLock, MA_OWNED); for (i = start; i < start + slices; i++) XX_MallocSmartMap[i] = ((i == start) ? slices : -1); } static void XX_MallocSmartMapClear(unsigned int start, unsigned int slices) { unsigned int i; mtx_assert(&XX_MallocSmartLock, MA_OWNED); for (i = start; i < start + slices; i++) XX_MallocSmartMap[i] = 0; } int XX_MallocSmartInit(void) { int error; error = E_OK; mtx_lock(&XX_MallocSmartLock); if (XX_MallocSmartPool) goto out; /* Allocate MallocSmart pool */ XX_MallocSmartPool = contigmalloc(MALLOCSMART_POOL_SIZE, M_NETCOMMSW, M_NOWAIT, 0, 0xFFFFFFFFFull, MALLOCSMART_POOL_SIZE, 0); if (!XX_MallocSmartPool) { error = E_NO_MEMORY; goto out; } out: mtx_unlock(&XX_MallocSmartLock); return (error); } void * XX_MallocSmart(uint32_t size, int memPartitionId, uint32_t alignment) { unsigned int i; vm_offset_t addr; addr = 0; /* Convert alignment and size to number of slices */ alignment = MALLOCSMART_SIZE_TO_SLICE(alignment); size = MALLOCSMART_SIZE_TO_SLICE(size); /* Lock resources */ mtx_lock(&XX_MallocSmartLock); /* Allocate region */ for (i = 0; i + size <= MALLOCSMART_SLICES; i += alignment) { if (XX_MallocSmartMapCheck(i, size)) { XX_MallocSmartMapSet(i, size); addr = (vm_offset_t)XX_MallocSmartPool + (i * MALLOCSMART_SLICE_SIZE); break; } } /* Unlock resources */ mtx_unlock(&XX_MallocSmartLock); return ((void *)addr); } void XX_FreeSmart(void *p) { unsigned int start, slices; /* Calculate first slice of region */ start = MALLOCSMART_SIZE_TO_SLICE((vm_offset_t)(p) - (vm_offset_t)XX_MallocSmartPool); /* Lock resources */ mtx_lock(&XX_MallocSmartLock); KASSERT(XX_MallocSmartMap[start] > 0, ("XX_FreeSmart: Double or mid-block free!\n")); - XX_UntrackAddress(p); /* Free region */ slices = XX_MallocSmartMap[start]; XX_MallocSmartMapClear(start, slices); /* Unlock resources */ mtx_unlock(&XX_MallocSmartLock); } void XX_Free(void *p) { - if (p != NULL) - XX_UntrackAddress(p); free(p, M_NETCOMMSW); } uint32_t XX_DisableAllIntr(void) { return (intr_disable()); } void XX_RestoreAllIntr(uint32_t flags) { intr_restore(flags); } t_Error XX_Call(uint32_t qid, t_Error (* f)(t_Handle), t_Handle id, t_Handle appId, uint16_t flags ) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); return (E_OK); } static bool XX_IsPortalIntr(int irq) { int cpu, type; /* Check interrupt numbers of all available portals */ for (cpu = 0, type = 0; XX_PInfo.portal_intr[type][cpu] != 0; cpu++) { if (irq == XX_PInfo.portal_intr[type][cpu]) { /* Found it! */ return (1); } if (XX_PInfo.portal_intr[type][cpu + 1] == 0) { type++; cpu = 0; } } return (0); } void XX_FmanFixIntr(int irq) { XX_IntrInfo[irq].flags |= XX_INTR_FLAG_FMAN_FIX; } static bool XX_FmanNeedsIntrFix(int irq) { if (XX_IntrInfo[irq].flags & XX_INTR_FLAG_FMAN_FIX) return (1); return (0); } static void XX_Dispatch(void *arg) { struct XX_IntrInfo *info; info = arg; /* Bind this thread to proper CPU when SMP has been already started. */ if ((info->flags & XX_INTR_FLAG_BOUND) == 0 && smp_started && info->cpu >= 0) { thread_lock(curthread); sched_bind(curthread, info->cpu); thread_unlock(curthread); info->flags |= XX_INTR_FLAG_BOUND; } if (info->handler == NULL) { printf("%s(): IRQ handler is NULL!\n", __func__); return; } info->handler(info->arg); } t_Error XX_PreallocAndBindIntr(uintptr_t irq, unsigned int cpu) { struct resource *r; unsigned int inum; t_Error error; r = (struct resource *)irq; inum = rman_get_start(r); error = XX_SetIntr(irq, XX_Dispatch, &XX_IntrInfo[inum]); if (error != 0) return (error); XX_IntrInfo[inum].flags = XX_INTR_FLAG_PREALLOCATED; XX_IntrInfo[inum].cpu = cpu; return (E_OK); } t_Error XX_DeallocIntr(uintptr_t irq) { struct resource *r; unsigned int inum; r = (struct resource *)irq; inum = rman_get_start(r); if ((XX_IntrInfo[inum].flags & XX_INTR_FLAG_PREALLOCATED) == 0) return (E_INVALID_STATE); XX_IntrInfo[inum].flags = 0; return (XX_FreeIntr(irq)); } t_Error XX_SetIntr(uintptr_t irq, t_Isr *f_Isr, t_Handle handle) { device_t dev; struct resource *r; unsigned int flags; int err; r = (struct resource *)irq; dev = rman_get_device(r); irq = rman_get_start(r); /* Handle preallocated interrupts */ if (XX_IntrInfo[irq].flags & XX_INTR_FLAG_PREALLOCATED) { if (XX_IntrInfo[irq].handler != NULL) return (E_BUSY); XX_IntrInfo[irq].handler = f_Isr; XX_IntrInfo[irq].arg = handle; return (E_OK); } flags = INTR_TYPE_NET | INTR_MPSAFE; /* BMAN/QMAN Portal interrupts must be exlusive */ if (XX_IsPortalIntr(irq)) flags |= INTR_EXCL; err = bus_setup_intr(dev, r, flags, NULL, f_Isr, handle, &XX_IntrInfo[irq].cookie); if (err) goto finish; /* * XXX: Bind FMan IRQ to CPU0. Current interrupt subsystem directs each * interrupt to all CPUs. Race between an interrupt assertion and * masking may occur and interrupt handler may be called multiple times * per one interrupt. FMan doesn't support such a situation. Workaround * is to bind FMan interrupt to one CPU0 only. */ #ifdef SMP if (XX_FmanNeedsIntrFix(irq)) err = powerpc_bind_intr(irq, 0); #endif finish: return (err); } t_Error XX_FreeIntr(uintptr_t irq) { device_t dev; struct resource *r; r = (struct resource *)irq; dev = rman_get_device(r); irq = rman_get_start(r); /* Handle preallocated interrupts */ if (XX_IntrInfo[irq].flags & XX_INTR_FLAG_PREALLOCATED) { if (XX_IntrInfo[irq].handler == NULL) return (E_INVALID_STATE); XX_IntrInfo[irq].handler = NULL; XX_IntrInfo[irq].arg = NULL; return (E_OK); } return (bus_teardown_intr(dev, r, XX_IntrInfo[irq].cookie)); } t_Error XX_EnableIntr(uintptr_t irq) { struct resource *r; r = (struct resource *)irq; irq = rman_get_start(r); powerpc_intr_unmask(irq); return (E_OK); } t_Error XX_DisableIntr(uintptr_t irq) { struct resource *r; r = (struct resource *)irq; irq = rman_get_start(r); powerpc_intr_mask(irq); return (E_OK); } t_TaskletHandle XX_InitTasklet (void (*routine)(void *), void *data) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); return (NULL); } void XX_FreeTasklet (t_TaskletHandle h_Tasklet) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); } int XX_ScheduleTask(t_TaskletHandle h_Tasklet, int immediate) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); return (0); } void XX_FlushScheduledTasks(void) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); } int XX_TaskletIsQueued(t_TaskletHandle h_Tasklet) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); return (0); } void XX_SetTaskletData(t_TaskletHandle h_Tasklet, t_Handle data) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); } t_Handle XX_GetTaskletData(t_TaskletHandle h_Tasklet) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); return (NULL); } t_Handle XX_InitSpinlock(void) { struct mtx *m; m = malloc(sizeof(*m), M_NETCOMMSW, M_NOWAIT | M_ZERO); if (!m) return (0); mtx_init(m, "NetCommSW Lock", NULL, MTX_DEF | MTX_DUPOK); return (m); } void XX_FreeSpinlock(t_Handle h_Spinlock) { struct mtx *m; m = h_Spinlock; mtx_destroy(m); free(m, M_NETCOMMSW); } void XX_LockSpinlock(t_Handle h_Spinlock) { struct mtx *m; m = h_Spinlock; mtx_lock(m); } void XX_UnlockSpinlock(t_Handle h_Spinlock) { struct mtx *m; m = h_Spinlock; mtx_unlock(m); } uint32_t XX_LockIntrSpinlock(t_Handle h_Spinlock) { XX_LockSpinlock(h_Spinlock); return (0); } void XX_UnlockIntrSpinlock(t_Handle h_Spinlock, uint32_t intrFlags) { XX_UnlockSpinlock(h_Spinlock); } uint32_t XX_CurrentTime(void) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); return (0); } t_Handle XX_CreateTimer(void) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); return (NULL); } void XX_FreeTimer(t_Handle h_Timer) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); } void XX_StartTimer(t_Handle h_Timer, uint32_t msecs, bool periodic, void (*f_TimerExpired)(t_Handle), t_Handle h_Arg) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); } uint32_t XX_GetExpirationTime(t_Handle h_Timer) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); return (0); } void XX_StopTimer(t_Handle h_Timer) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); } void XX_ModTimer(t_Handle h_Timer, uint32_t msecs) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); } int XX_TimerIsActive(t_Handle h_Timer) { /* Not referenced */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); return (0); } uint32_t XX_Sleep(uint32_t msecs) { XX_UDelay(1000 * msecs); return (0); } void XX_UDelay(uint32_t usecs) { DELAY(usecs); } t_Error XX_IpcRegisterMsgHandler(char addr[XX_IPC_MAX_ADDR_NAME_LENGTH], t_IpcMsgHandler *f_MsgHandler, t_Handle h_Module, uint32_t replyLength) { /* * This function returns fake E_OK status and does nothing * as NetCommSW IPC is not used by FreeBSD drivers. */ return (E_OK); } t_Error XX_IpcUnregisterMsgHandler(char addr[XX_IPC_MAX_ADDR_NAME_LENGTH]) { /* * This function returns fake E_OK status and does nothing * as NetCommSW IPC is not used by FreeBSD drivers. */ return (E_OK); } t_Error XX_IpcSendMessage(t_Handle h_Session, uint8_t *p_Msg, uint32_t msgLength, uint8_t *p_Reply, uint32_t *p_ReplyLength, t_IpcMsgCompletion *f_Completion, t_Handle h_Arg) { /* Should not be called */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); return (E_OK); } t_Handle XX_IpcInitSession(char destAddr[XX_IPC_MAX_ADDR_NAME_LENGTH], char srcAddr[XX_IPC_MAX_ADDR_NAME_LENGTH]) { /* Should not be called */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); return (E_OK); } t_Error XX_IpcFreeSession(t_Handle h_Session) { /* Should not be called */ printf("NetCommSW: Unimplemented function %s() called!\n", __func__); return (E_OK); } physAddress_t XX_VirtToPhys(void *addr) { vm_paddr_t paddr; int cpu; cpu = PCPU_GET(cpuid); /* Handle NULL address */ if (addr == NULL) return (-1); + /* Check CCSR */ + if ((vm_offset_t)addr >= ccsrbar_va && + (vm_offset_t)addr < ccsrbar_va + ccsrbar_size) + return (((vm_offset_t)addr - ccsrbar_va) + ccsrbar_pa); + /* Handle BMAN mappings */ if (((vm_offset_t)addr >= XX_PInfo.portal_ce_va[BM_PORTAL]) && ((vm_offset_t)addr < XX_PInfo.portal_ce_va[BM_PORTAL] + XX_PInfo.portal_ce_size[BM_PORTAL][cpu])) return (XX_PInfo.portal_ce_pa[BM_PORTAL][cpu] + (vm_offset_t)addr - XX_PInfo.portal_ce_va[BM_PORTAL]); if (((vm_offset_t)addr >= XX_PInfo.portal_ci_va[BM_PORTAL]) && ((vm_offset_t)addr < XX_PInfo.portal_ci_va[BM_PORTAL] + XX_PInfo.portal_ci_size[BM_PORTAL][cpu])) return (XX_PInfo.portal_ci_pa[BM_PORTAL][cpu] + (vm_offset_t)addr - XX_PInfo.portal_ci_va[BM_PORTAL]); /* Handle QMAN mappings */ if (((vm_offset_t)addr >= XX_PInfo.portal_ce_va[QM_PORTAL]) && ((vm_offset_t)addr < XX_PInfo.portal_ce_va[QM_PORTAL] + XX_PInfo.portal_ce_size[QM_PORTAL][cpu])) return (XX_PInfo.portal_ce_pa[QM_PORTAL][cpu] + (vm_offset_t)addr - XX_PInfo.portal_ce_va[QM_PORTAL]); if (((vm_offset_t)addr >= XX_PInfo.portal_ci_va[QM_PORTAL]) && ((vm_offset_t)addr < XX_PInfo.portal_ci_va[QM_PORTAL] + XX_PInfo.portal_ci_size[QM_PORTAL][cpu])) return (XX_PInfo.portal_ci_pa[QM_PORTAL][cpu] + (vm_offset_t)addr - XX_PInfo.portal_ci_va[QM_PORTAL]); - paddr = XX_TrackAddress(addr); - if (paddr == -1) + paddr = pmap_kextract((vm_offset_t)addr); + if (paddr == 0) printf("NetCommSW: " "Unable to translate virtual address 0x%08X!\n", addr); + else + pmap_track_page(kernel_pmap, (vm_offset_t)addr); return (paddr); } void * XX_PhysToVirt(physAddress_t addr) { - XX_MallocTrackStruct *ts; + struct pv_entry *pv; + vm_page_t page; int cpu; + /* Check CCSR */ + if (addr >= ccsrbar_pa && addr < ccsrbar_pa + ccsrbar_size) + return ((void *)((vm_offset_t)(addr - ccsrbar_pa) + + ccsrbar_va)); + cpu = PCPU_GET(cpuid); /* Handle BMAN mappings */ if ((addr >= XX_PInfo.portal_ce_pa[BM_PORTAL][cpu]) && (addr < XX_PInfo.portal_ce_pa[BM_PORTAL][cpu] + XX_PInfo.portal_ce_size[BM_PORTAL][cpu])) return ((void *)(XX_PInfo.portal_ci_va[BM_PORTAL] + (vm_offset_t)(addr - XX_PInfo.portal_ci_pa[BM_PORTAL][cpu]))); if ((addr >= XX_PInfo.portal_ci_pa[BM_PORTAL][cpu]) && (addr < XX_PInfo.portal_ci_pa[BM_PORTAL][cpu] + XX_PInfo.portal_ci_size[BM_PORTAL][cpu])) return ((void *)(XX_PInfo.portal_ci_va[BM_PORTAL] + (vm_offset_t)(addr - XX_PInfo.portal_ci_pa[BM_PORTAL][cpu]))); /* Handle QMAN mappings */ if ((addr >= XX_PInfo.portal_ce_pa[QM_PORTAL][cpu]) && (addr < XX_PInfo.portal_ce_pa[QM_PORTAL][cpu] + XX_PInfo.portal_ce_size[QM_PORTAL][cpu])) return ((void *)(XX_PInfo.portal_ce_va[QM_PORTAL] + (vm_offset_t)(addr - XX_PInfo.portal_ce_pa[QM_PORTAL][cpu]))); if ((addr >= XX_PInfo.portal_ci_pa[QM_PORTAL][cpu]) && (addr < XX_PInfo.portal_ci_pa[QM_PORTAL][cpu] + XX_PInfo.portal_ci_size[QM_PORTAL][cpu])) return ((void *)(XX_PInfo.portal_ci_va[QM_PORTAL] + (vm_offset_t)(addr - XX_PInfo.portal_ci_pa[QM_PORTAL][cpu]))); - mtx_lock(&XX_MallocTrackLock); - ts = XX_FindTracker(addr); - mtx_unlock(&XX_MallocTrackLock); + page = PHYS_TO_VM_PAGE(addr); + pv = TAILQ_FIRST(&page->md.pv_list); - if (ts != NULL) - return ts->va; + if (pv != NULL) + return ((void *)(pv->pv_va + ((vm_offset_t)addr & PAGE_MASK))); printf("NetCommSW: " "Unable to translate physical address 0x%08llX!\n", addr); return (NULL); } void XX_PortalSetInfo(device_t dev) { char *dev_name; struct dpaa_portals_softc *sc; int i, type, len; dev_name = malloc(sizeof(*dev_name), M_TEMP, M_WAITOK | M_ZERO); len = strlen("bman-portals"); strncpy(dev_name, device_get_name(dev), len); if (strncmp(dev_name, "bman-portals", len) && strncmp(dev_name, "qman-portals", len)) goto end; if (strncmp(dev_name, "bman-portals", len) == 0) type = BM_PORTAL; else type = QM_PORTAL; sc = device_get_softc(dev); for (i = 0; sc->sc_dp[i].dp_ce_pa != 0; i++) { XX_PInfo.portal_ce_pa[type][i] = sc->sc_dp[i].dp_ce_pa; XX_PInfo.portal_ci_pa[type][i] = sc->sc_dp[i].dp_ci_pa; XX_PInfo.portal_ce_size[type][i] = sc->sc_dp[i].dp_ce_size; XX_PInfo.portal_ci_size[type][i] = sc->sc_dp[i].dp_ci_size; XX_PInfo.portal_intr[type][i] = sc->sc_dp[i].dp_intr_num; } XX_PInfo.portal_ce_va[type] = rman_get_bushandle(sc->sc_rres[0]); XX_PInfo.portal_ci_va[type] = rman_get_bushandle(sc->sc_rres[1]); end: free(dev_name, M_TEMP); -} - -static inline XX_MallocTrackStruct * -XX_FindTracker(physAddress_t pa) -{ - struct XX_MallocTrackerList *l; - XX_MallocTrackStruct *tp; - - l = &XX_MallocTracker[(pa >> XX_MALLOC_TRACK_SHIFT) & XX_MallocHashMask]; - - LIST_FOREACH(tp, l, entries) { - if (tp->pa == pa) - return tp; - } - - return NULL; -} - -void -XX_TrackInit(void) -{ - if (XX_MallocTracker == NULL) { - XX_MallocTracker = hashinit(64, M_NETCOMMSW_MT, - &XX_MallocHashMask); - } -} - -physAddress_t -XX_TrackAddress(void *addr) -{ - physAddress_t pa; - struct XX_MallocTrackerList *l; - XX_MallocTrackStruct *ts; - - pa = pmap_kextract((vm_offset_t)addr); - - l = &XX_MallocTracker[(pa >> XX_MALLOC_TRACK_SHIFT) & XX_MallocHashMask]; - - mtx_lock(&XX_MallocTrackLock); - if (XX_FindTracker(pa) == NULL) { - ts = malloc(sizeof(*ts), M_NETCOMMSW_MT, M_NOWAIT); - if (ts == NULL) - return (-1); - ts->va = addr; - ts->pa = pa; - LIST_INSERT_HEAD(l, ts, entries); - } - mtx_unlock(&XX_MallocTrackLock); - - return (pa); -} - -void -XX_UntrackAddress(void *addr) -{ - physAddress_t pa; - XX_MallocTrackStruct *ts; - - pa = pmap_kextract((vm_offset_t)addr); - - KASSERT(XX_MallocTracker != NULL, - ("Untracking an address before it's even initialized!\n")); - - mtx_lock(&XX_MallocTrackLock); - ts = XX_FindTracker(pa); - if (ts != NULL) - LIST_REMOVE(ts, entries); - mtx_unlock(&XX_MallocTrackLock); - free(ts, M_NETCOMMSW_MT); } Index: head/sys/dev/dpaa/fman.c =================================================================== --- head/sys/dev/dpaa/fman.c (revision 308717) +++ head/sys/dev/dpaa/fman.c (revision 308718) @@ -1,410 +1,408 @@ /*- * Copyright (c) 2011-2012 Semihalf. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include "opt_platform.h" #include #include #include #include #include "fman.h" static MALLOC_DEFINE(M_FMAN, "fman", "fman devices information"); /** * @group FMan private defines. * @{ */ enum fman_irq_enum { FMAN_IRQ_NUM = 0, FMAN_ERR_IRQ_NUM = 1 }; enum fman_mu_ram_map { FMAN_MURAM_OFF = 0x0, FMAN_MURAM_SIZE = 0x28000 }; struct fman_config { device_t fman_device; uintptr_t mem_base_addr; uintptr_t irq_num; uintptr_t err_irq_num; uint8_t fm_id; t_FmExceptionsCallback *exception_callback; t_FmBusErrorCallback *bus_error_callback; }; /** * @group FMan private methods/members. * @{ */ /** * Frame Manager firmware. * We use the same firmware for both P3041 and P2041 devices. */ const uint32_t fman_firmware[] = FMAN_UC_IMG; const uint32_t fman_firmware_size = sizeof(fman_firmware); static struct fman_softc *fm_sc = NULL; static t_Handle fman_init(struct fman_softc *sc, struct fman_config *cfg) { struct ofw_bus_devinfo obd; phandle_t node; t_FmParams fm_params; t_Handle muram_handle, fm_handle; t_Error error; t_FmRevisionInfo revision_info; uint16_t clock; uint32_t tmp, mod; /* MURAM configuration */ muram_handle = FM_MURAM_ConfigAndInit(cfg->mem_base_addr + FMAN_MURAM_OFF, FMAN_MURAM_SIZE); if (muram_handle == NULL) { device_printf(cfg->fman_device, "couldn't init FM MURAM module" "\n"); return (NULL); } sc->muram_handle = muram_handle; /* Fill in FM configuration */ fm_params.fmId = cfg->fm_id; /* XXX we support only one partition thus each fman has master id */ fm_params.guestId = NCSW_MASTER_ID; fm_params.baseAddr = cfg->mem_base_addr; fm_params.h_FmMuram = muram_handle; /* Get FMan clock in Hz */ if ((tmp = fman_get_clock(sc)) == 0) return (NULL); /* Convert FMan clock to MHz */ clock = (uint16_t)(tmp / 1000000); mod = tmp % 1000000; if (mod >= 500000) ++clock; fm_params.fmClkFreq = clock; fm_params.f_Exception = cfg->exception_callback; fm_params.f_BusError = cfg->bus_error_callback; fm_params.h_App = cfg->fman_device; fm_params.irq = cfg->irq_num; fm_params.errIrq = cfg->err_irq_num; fm_params.firmware.size = fman_firmware_size; fm_params.firmware.p_Code = (uint32_t*)fman_firmware; fm_handle = FM_Config(&fm_params); if (fm_handle == NULL) { device_printf(cfg->fman_device, "couldn't configure FM " "module\n"); goto err; } FM_ConfigResetOnInit(fm_handle, TRUE); error = FM_Init(fm_handle); if (error != E_OK) { device_printf(cfg->fman_device, "couldn't init FM module\n"); goto err2; } error = FM_GetRevision(fm_handle, &revision_info); if (error != E_OK) { device_printf(cfg->fman_device, "couldn't get FM revision\n"); goto err2; } device_printf(cfg->fman_device, "Hardware version: %d.%d.\n", revision_info.majorRev, revision_info.minorRev); /* Initialize the simplebus part of things */ simplebus_init(sc->sc_base.dev, 0); node = ofw_bus_get_node(sc->sc_base.dev); for (node = OF_child(node); node > 0; node = OF_peer(node)) { if (ofw_bus_gen_setup_devinfo(&obd, node) != 0) continue; simplebus_add_device(sc->sc_base.dev, node, 0, NULL, -1, NULL); } return (fm_handle); err2: FM_Free(fm_handle); err: FM_MURAM_Free(muram_handle); return (NULL); } static void fman_exception_callback(t_Handle app_handle, e_FmExceptions exception) { struct fman_softc *sc; sc = app_handle; device_printf(sc->sc_base.dev, "FMan exception occurred.\n"); } static void fman_error_callback(t_Handle app_handle, e_FmPortType port_type, uint8_t port_id, uint64_t addr, uint8_t tnum, uint16_t liodn) { struct fman_softc *sc; sc = app_handle; device_printf(sc->sc_base.dev, "FMan error occurred.\n"); } /** @} */ /** * @group FMan driver interface. * @{ */ int fman_get_handle(t_Handle *fmh) { if (fm_sc == NULL) return (ENOMEM); *fmh = fm_sc->fm_handle; return (0); } int fman_get_muram_handle(t_Handle *muramh) { if (fm_sc == NULL) return (ENOMEM); *muramh = fm_sc->muram_handle; return (0); } int fman_get_bushandle(vm_offset_t *fm_base) { if (fm_sc == NULL) return (ENOMEM); *fm_base = rman_get_bushandle(fm_sc->mem_res); return (0); } int fman_get_dev(device_t *fm_dev) { if (fm_sc == NULL) return (ENOMEM); *fm_dev = fm_sc->sc_base.dev; return (0); } int fman_attach(device_t dev) { struct fman_softc *sc; struct fman_config cfg; pcell_t qchan_range[2]; phandle_t node; sc = device_get_softc(dev); sc->sc_base.dev = dev; fm_sc = sc; /* Check if MallocSmart allocator is ready */ if (XX_MallocSmartInit() != E_OK) { device_printf(dev, "could not initialize smart allocator.\n"); return (ENXIO); } - XX_TrackInit(); - node = ofw_bus_get_node(dev); if (OF_getencprop(node, "fsl,qman-channel-range", qchan_range, sizeof(qchan_range)) <= 0) { device_printf(dev, "Missing QMan channel range property!\n"); return (ENXIO); } sc->qman_chan_base = qchan_range[0]; sc->qman_chan_count = qchan_range[1]; sc->mem_rid = 0; sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->mem_rid, RF_ACTIVE | RF_SHAREABLE); if (!sc->mem_res) { device_printf(dev, "could not allocate memory.\n"); return (ENXIO); } sc->irq_rid = 0; sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irq_rid, RF_ACTIVE); if (!sc->irq_res) { device_printf(dev, "could not allocate interrupt.\n"); goto err; } /* * XXX: Fix FMan interrupt. This is workaround for the issue with * interrupts directed to multiple CPUs by the interrupts subsystem. * Workaround is to bind the interrupt to only one CPU0. */ XX_FmanFixIntr(rman_get_start(sc->irq_res)); sc->err_irq_rid = 1; sc->err_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->err_irq_rid, RF_ACTIVE | RF_SHAREABLE); if (!sc->err_irq_res) { device_printf(dev, "could not allocate error interrupt.\n"); goto err; } /* Set FMan configuration */ cfg.fman_device = dev; cfg.fm_id = device_get_unit(dev); cfg.mem_base_addr = rman_get_bushandle(sc->mem_res); cfg.irq_num = (uintptr_t)sc->irq_res; cfg.err_irq_num = (uintptr_t)sc->err_irq_res; cfg.exception_callback = fman_exception_callback; cfg.bus_error_callback = fman_error_callback; sc->fm_handle = fman_init(sc, &cfg); if (sc->fm_handle == NULL) { device_printf(dev, "could not be configured\n"); return (ENXIO); } return (bus_generic_attach(dev)); err: fman_detach(dev); return (ENXIO); } int fman_detach(device_t dev) { struct fman_softc *sc; sc = device_get_softc(dev); if (sc->muram_handle) { FM_MURAM_Free(sc->muram_handle); } if (sc->fm_handle) { FM_Free(sc->fm_handle); } if (sc->mem_res) { bus_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid, sc->mem_res); } if (sc->irq_res) { bus_release_resource(dev, SYS_RES_IRQ, sc->irq_rid, sc->irq_res); } if (sc->irq_res) { bus_release_resource(dev, SYS_RES_IRQ, sc->err_irq_rid, sc->err_irq_res); } return (0); } int fman_suspend(device_t dev) { return (0); } int fman_resume(device_t dev) { return (0); } int fman_shutdown(device_t dev) { return (0); } int fman_qman_channel_id(device_t dev, int port) { struct fman_softc *sc; int qman_port_id[] = {0x31, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07}; int i; sc = device_get_softc(dev); for (i = 0; i < sc->qman_chan_count; i++) { if (qman_port_id[i] == port) return (sc->qman_chan_base + i); } return (0); } /** @} */ Index: head/sys/dev/dpaa/if_dtsec.c =================================================================== --- head/sys/dev/dpaa/if_dtsec.c (revision 308717) +++ head/sys/dev/dpaa/if_dtsec.c (revision 308718) @@ -1,829 +1,827 @@ /*- * Copyright (c) 2011-2012 Semihalf. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "miibus_if.h" #include #include #include #include "fman.h" #include "if_dtsec.h" #include "if_dtsec_im.h" #include "if_dtsec_rm.h" /** * @group dTSEC private defines. * @{ */ /** * dTSEC FMan MAC exceptions info struct. */ struct dtsec_fm_mac_ex_str { const int num; const char *str; }; /** @} */ /** * @group FMan MAC routines. * @{ */ #define DTSEC_MAC_EXCEPTIONS_END (-1) /** * FMan MAC exceptions. */ static const struct dtsec_fm_mac_ex_str dtsec_fm_mac_exceptions[] = { { e_FM_MAC_EX_10G_MDIO_SCAN_EVENTMDIO, "MDIO scan event" }, { e_FM_MAC_EX_10G_MDIO_CMD_CMPL, "MDIO command completion" }, { e_FM_MAC_EX_10G_REM_FAULT, "Remote fault" }, { e_FM_MAC_EX_10G_LOC_FAULT, "Local fault" }, { e_FM_MAC_EX_10G_1TX_ECC_ER, "Transmit frame ECC error" }, { e_FM_MAC_EX_10G_TX_FIFO_UNFL, "Transmit FIFO underflow" }, { e_FM_MAC_EX_10G_TX_FIFO_OVFL, "Receive FIFO overflow" }, { e_FM_MAC_EX_10G_TX_ER, "Transmit frame error" }, { e_FM_MAC_EX_10G_RX_FIFO_OVFL, "Receive FIFO overflow" }, { e_FM_MAC_EX_10G_RX_ECC_ER, "Receive frame ECC error" }, { e_FM_MAC_EX_10G_RX_JAB_FRM, "Receive jabber frame" }, { e_FM_MAC_EX_10G_RX_OVRSZ_FRM, "Receive oversized frame" }, { e_FM_MAC_EX_10G_RX_RUNT_FRM, "Receive runt frame" }, { e_FM_MAC_EX_10G_RX_FRAG_FRM, "Receive fragment frame" }, { e_FM_MAC_EX_10G_RX_LEN_ER, "Receive payload length error" }, { e_FM_MAC_EX_10G_RX_CRC_ER, "Receive CRC error" }, { e_FM_MAC_EX_10G_RX_ALIGN_ER, "Receive alignment error" }, { e_FM_MAC_EX_1G_BAB_RX, "Babbling receive error" }, { e_FM_MAC_EX_1G_RX_CTL, "Receive control (pause frame) interrupt" }, { e_FM_MAC_EX_1G_GRATEFUL_TX_STP_COMPLET, "Graceful transmit stop " "complete" }, { e_FM_MAC_EX_1G_BAB_TX, "Babbling transmit error" }, { e_FM_MAC_EX_1G_TX_CTL, "Transmit control (pause frame) interrupt" }, { e_FM_MAC_EX_1G_TX_ERR, "Transmit error" }, { e_FM_MAC_EX_1G_LATE_COL, "Late collision" }, { e_FM_MAC_EX_1G_COL_RET_LMT, "Collision retry limit" }, { e_FM_MAC_EX_1G_TX_FIFO_UNDRN, "Transmit FIFO underrun" }, { e_FM_MAC_EX_1G_MAG_PCKT, "Magic Packet detected when dTSEC is in " "Magic Packet detection mode" }, { e_FM_MAC_EX_1G_MII_MNG_RD_COMPLET, "MII management read completion" }, { e_FM_MAC_EX_1G_MII_MNG_WR_COMPLET, "MII management write completion" }, { e_FM_MAC_EX_1G_GRATEFUL_RX_STP_COMPLET, "Graceful receive stop " "complete" }, { e_FM_MAC_EX_1G_TX_DATA_ERR, "Internal data error on transmit" }, { e_FM_MAC_EX_1G_RX_DATA_ERR, "Internal data error on receive" }, { e_FM_MAC_EX_1G_1588_TS_RX_ERR, "Time-Stamp Receive Error" }, { e_FM_MAC_EX_1G_RX_MIB_CNT_OVFL, "MIB counter overflow" }, { DTSEC_MAC_EXCEPTIONS_END, "" } }; static const char * dtsec_fm_mac_ex_to_str(e_FmMacExceptions exception) { int i; for (i = 0; dtsec_fm_mac_exceptions[i].num != exception && dtsec_fm_mac_exceptions[i].num != DTSEC_MAC_EXCEPTIONS_END; ++i) ; if (dtsec_fm_mac_exceptions[i].num == DTSEC_MAC_EXCEPTIONS_END) return (""); return (dtsec_fm_mac_exceptions[i].str); } static void dtsec_fm_mac_mdio_event_callback(t_Handle h_App, e_FmMacExceptions exception) { struct dtsec_softc *sc; sc = h_App; device_printf(sc->sc_dev, "MDIO event %i: %s.\n", exception, dtsec_fm_mac_ex_to_str(exception)); } static void dtsec_fm_mac_exception_callback(t_Handle app, e_FmMacExceptions exception) { struct dtsec_softc *sc; sc = app; device_printf(sc->sc_dev, "MAC exception %i: %s.\n", exception, dtsec_fm_mac_ex_to_str(exception)); } static void dtsec_fm_mac_free(struct dtsec_softc *sc) { if (sc->sc_mach == NULL) return; FM_MAC_Disable(sc->sc_mach, e_COMM_MODE_RX_AND_TX); FM_MAC_Free(sc->sc_mach); sc->sc_mach = NULL; } static int dtsec_fm_mac_init(struct dtsec_softc *sc, uint8_t *mac) { t_FmMacParams params; t_Error error; memset(¶ms, 0, sizeof(params)); memcpy(¶ms.addr, mac, sizeof(params.addr)); params.baseAddr = sc->sc_fm_base + sc->sc_mac_mem_offset; params.enetMode = sc->sc_mac_enet_mode; params.macId = sc->sc_eth_id; params.mdioIrq = sc->sc_mac_mdio_irq; params.f_Event = dtsec_fm_mac_mdio_event_callback; params.f_Exception = dtsec_fm_mac_exception_callback; params.h_App = sc; params.h_Fm = sc->sc_fmh; sc->sc_mach = FM_MAC_Config(¶ms); if (sc->sc_mach == NULL) { device_printf(sc->sc_dev, "couldn't configure FM_MAC module.\n" ); return (ENXIO); } error = FM_MAC_ConfigResetOnInit(sc->sc_mach, TRUE); if (error != E_OK) { device_printf(sc->sc_dev, "couldn't enable reset on init " "feature.\n"); dtsec_fm_mac_free(sc); return (ENXIO); } /* Do not inform about pause frames */ error = FM_MAC_ConfigException(sc->sc_mach, e_FM_MAC_EX_1G_RX_CTL, FALSE); if (error != E_OK) { device_printf(sc->sc_dev, "couldn't disable pause frames " "exception.\n"); dtsec_fm_mac_free(sc); return (ENXIO); } error = FM_MAC_Init(sc->sc_mach); if (error != E_OK) { device_printf(sc->sc_dev, "couldn't initialize FM_MAC module." "\n"); dtsec_fm_mac_free(sc); return (ENXIO); } return (0); } /** @} */ /** * @group FMan PORT routines. * @{ */ static const char * dtsec_fm_port_ex_to_str(e_FmPortExceptions exception) { switch (exception) { case e_FM_PORT_EXCEPTION_IM_BUSY: return ("IM: RX busy"); default: return (""); } } void dtsec_fm_port_rx_exception_callback(t_Handle app, e_FmPortExceptions exception) { struct dtsec_softc *sc; sc = app; device_printf(sc->sc_dev, "RX exception: %i: %s.\n", exception, dtsec_fm_port_ex_to_str(exception)); } void dtsec_fm_port_tx_exception_callback(t_Handle app, e_FmPortExceptions exception) { struct dtsec_softc *sc; sc = app; device_printf(sc->sc_dev, "TX exception: %i: %s.\n", exception, dtsec_fm_port_ex_to_str(exception)); } e_FmPortType dtsec_fm_port_rx_type(enum eth_dev_type type) { switch (type) { case ETH_DTSEC: return (e_FM_PORT_TYPE_RX); case ETH_10GSEC: return (e_FM_PORT_TYPE_RX_10G); default: return (e_FM_PORT_TYPE_DUMMY); } } e_FmPortType dtsec_fm_port_tx_type(enum eth_dev_type type) { switch (type) { case ETH_DTSEC: return (e_FM_PORT_TYPE_TX); case ETH_10GSEC: return (e_FM_PORT_TYPE_TX_10G); default: return (e_FM_PORT_TYPE_DUMMY); } } static void dtsec_fm_port_free_both(struct dtsec_softc *sc) { if (sc->sc_rxph) { FM_PORT_Free(sc->sc_rxph); sc->sc_rxph = NULL; } if (sc->sc_txph) { FM_PORT_Free(sc->sc_txph); sc->sc_txph = NULL; } } /** @} */ /** * @group IFnet routines. * @{ */ static int dtsec_if_enable_locked(struct dtsec_softc *sc) { int error; DTSEC_LOCK_ASSERT(sc); error = FM_MAC_Enable(sc->sc_mach, e_COMM_MODE_RX_AND_TX); if (error != E_OK) return (EIO); error = FM_PORT_Enable(sc->sc_rxph); if (error != E_OK) return (EIO); error = FM_PORT_Enable(sc->sc_txph); if (error != E_OK) return (EIO); sc->sc_ifnet->if_drv_flags |= IFF_DRV_RUNNING; /* Refresh link state */ dtsec_miibus_statchg(sc->sc_dev); return (0); } static int dtsec_if_disable_locked(struct dtsec_softc *sc) { int error; DTSEC_LOCK_ASSERT(sc); error = FM_MAC_Disable(sc->sc_mach, e_COMM_MODE_RX_AND_TX); if (error != E_OK) return (EIO); error = FM_PORT_Disable(sc->sc_rxph); if (error != E_OK) return (EIO); error = FM_PORT_Disable(sc->sc_txph); if (error != E_OK) return (EIO); sc->sc_ifnet->if_drv_flags &= ~IFF_DRV_RUNNING; return (0); } static int dtsec_if_ioctl(struct ifnet *ifp, u_long command, caddr_t data) { struct dtsec_softc *sc; struct ifreq *ifr; int error; sc = ifp->if_softc; ifr = (struct ifreq *)data; error = 0; /* Basic functionality to achieve media status reports */ switch (command) { case SIOCSIFFLAGS: DTSEC_LOCK(sc); if (sc->sc_ifnet->if_flags & IFF_UP) error = dtsec_if_enable_locked(sc); else error = dtsec_if_disable_locked(sc); DTSEC_UNLOCK(sc); break; case SIOCGIFMEDIA: case SIOCSIFMEDIA: error = ifmedia_ioctl(ifp, ifr, &sc->sc_mii->mii_media, command); break; default: error = ether_ioctl(ifp, command, data); } return (error); } static void dtsec_if_tick(void *arg) { struct dtsec_softc *sc; sc = arg; /* TODO */ DTSEC_LOCK(sc); mii_tick(sc->sc_mii); callout_reset(&sc->sc_tick_callout, hz, dtsec_if_tick, sc); DTSEC_UNLOCK(sc); } static void dtsec_if_deinit_locked(struct dtsec_softc *sc) { DTSEC_LOCK_ASSERT(sc); DTSEC_UNLOCK(sc); callout_drain(&sc->sc_tick_callout); DTSEC_LOCK(sc); } static void dtsec_if_init_locked(struct dtsec_softc *sc) { int error; DTSEC_LOCK_ASSERT(sc); /* Set MAC address */ error = FM_MAC_ModifyMacAddr(sc->sc_mach, (t_EnetAddr *)IF_LLADDR(sc->sc_ifnet)); if (error != E_OK) { device_printf(sc->sc_dev, "couldn't set MAC address.\n"); goto err; } /* Start MII polling */ if (sc->sc_mii) callout_reset(&sc->sc_tick_callout, hz, dtsec_if_tick, sc); if (sc->sc_ifnet->if_flags & IFF_UP) { error = dtsec_if_enable_locked(sc); if (error != 0) goto err; } else { error = dtsec_if_disable_locked(sc); if (error != 0) goto err; } return; err: dtsec_if_deinit_locked(sc); device_printf(sc->sc_dev, "initialization error.\n"); return; } static void dtsec_if_init(void *data) { struct dtsec_softc *sc; sc = data; DTSEC_LOCK(sc); dtsec_if_init_locked(sc); DTSEC_UNLOCK(sc); } static void dtsec_if_start(struct ifnet *ifp) { struct dtsec_softc *sc; sc = ifp->if_softc; DTSEC_LOCK(sc); sc->sc_start_locked(sc); DTSEC_UNLOCK(sc); } static void dtsec_if_watchdog(struct ifnet *ifp) { /* TODO */ } /** @} */ /** * @group IFmedia routines. * @{ */ static int dtsec_ifmedia_upd(struct ifnet *ifp) { struct dtsec_softc *sc = ifp->if_softc; DTSEC_LOCK(sc); mii_mediachg(sc->sc_mii); DTSEC_UNLOCK(sc); return (0); } static void dtsec_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) { struct dtsec_softc *sc = ifp->if_softc; DTSEC_LOCK(sc); mii_pollstat(sc->sc_mii); ifmr->ifm_active = sc->sc_mii->mii_media_active; ifmr->ifm_status = sc->sc_mii->mii_media_status; DTSEC_UNLOCK(sc); } /** @} */ /** * @group dTSEC bus interface. * @{ */ static void dtsec_configure_mode(struct dtsec_softc *sc) { char tunable[64]; snprintf(tunable, sizeof(tunable), "%s.independent_mode", device_get_nameunit(sc->sc_dev)); sc->sc_mode = DTSEC_MODE_REGULAR; TUNABLE_INT_FETCH(tunable, &sc->sc_mode); if (sc->sc_mode == DTSEC_MODE_REGULAR) { sc->sc_port_rx_init = dtsec_rm_fm_port_rx_init; sc->sc_port_tx_init = dtsec_rm_fm_port_tx_init; sc->sc_start_locked = dtsec_rm_if_start_locked; } else { sc->sc_port_rx_init = dtsec_im_fm_port_rx_init; sc->sc_port_tx_init = dtsec_im_fm_port_tx_init; sc->sc_start_locked = dtsec_im_if_start_locked; } device_printf(sc->sc_dev, "Configured for %s mode.\n", (sc->sc_mode == DTSEC_MODE_REGULAR) ? "regular" : "independent"); } int dtsec_attach(device_t dev) { struct dtsec_softc *sc; int error; struct ifnet *ifp; sc = device_get_softc(dev); sc->sc_dev = dev; sc->sc_mac_mdio_irq = NO_IRQ; sc->sc_eth_id = device_get_unit(dev); /* Check if MallocSmart allocator is ready */ if (XX_MallocSmartInit() != E_OK) return (ENXIO); - XX_TrackInit(); - /* Init locks */ mtx_init(&sc->sc_lock, device_get_nameunit(dev), "DTSEC Global Lock", MTX_DEF); mtx_init(&sc->sc_mii_lock, device_get_nameunit(dev), "DTSEC MII Lock", MTX_DEF); /* Init callouts */ callout_init(&sc->sc_tick_callout, CALLOUT_MPSAFE); /* Read configuraton */ if ((error = fman_get_handle(&sc->sc_fmh)) != 0) return (error); if ((error = fman_get_muram_handle(&sc->sc_muramh)) != 0) return (error); if ((error = fman_get_bushandle(&sc->sc_fm_base)) != 0) return (error); /* Configure working mode */ dtsec_configure_mode(sc); /* If we are working in regular mode configure BMAN and QMAN */ if (sc->sc_mode == DTSEC_MODE_REGULAR) { /* Create RX buffer pool */ error = dtsec_rm_pool_rx_init(sc); if (error != 0) return (EIO); /* Create RX frame queue range */ error = dtsec_rm_fqr_rx_init(sc); if (error != 0) return (EIO); /* Create frame info pool */ error = dtsec_rm_fi_pool_init(sc); if (error != 0) return (EIO); /* Create TX frame queue range */ error = dtsec_rm_fqr_tx_init(sc); if (error != 0) return (EIO); } /* Init FMan MAC module. */ error = dtsec_fm_mac_init(sc, sc->sc_mac_addr); if (error != 0) { dtsec_detach(dev); return (ENXIO); } /* Init FMan TX port */ error = sc->sc_port_tx_init(sc, device_get_unit(sc->sc_dev)); if (error != 0) { dtsec_detach(dev); return (ENXIO); } /* Init FMan RX port */ error = sc->sc_port_rx_init(sc, device_get_unit(sc->sc_dev)); if (error != 0) { dtsec_detach(dev); return (ENXIO); } /* Create network interface for upper layers */ ifp = sc->sc_ifnet = if_alloc(IFT_ETHER); if (ifp == NULL) { device_printf(sc->sc_dev, "if_alloc() failed.\n"); dtsec_detach(dev); return (ENOMEM); } ifp->if_softc = sc; ifp->if_mtu = ETHERMTU; /* TODO: Configure */ ifp->if_flags = IFF_SIMPLEX | IFF_BROADCAST; ifp->if_init = dtsec_if_init; ifp->if_start = dtsec_if_start; ifp->if_ioctl = dtsec_if_ioctl; ifp->if_snd.ifq_maxlen = IFQ_MAXLEN; if (sc->sc_phy_addr >= 0) if_initname(ifp, device_get_name(sc->sc_dev), device_get_unit(sc->sc_dev)); else if_initname(ifp, "dtsec_phy", device_get_unit(sc->sc_dev)); /* TODO */ #if 0 IFQ_SET_MAXLEN(&ifp->if_snd, TSEC_TX_NUM_DESC - 1); ifp->if_snd.ifq_drv_maxlen = TSEC_TX_NUM_DESC - 1; IFQ_SET_READY(&ifp->if_snd); #endif ifp->if_capabilities = 0; /* TODO: Check */ ifp->if_capenable = ifp->if_capabilities; /* Attach PHY(s) */ error = mii_attach(sc->sc_dev, &sc->sc_mii_dev, ifp, dtsec_ifmedia_upd, dtsec_ifmedia_sts, BMSR_DEFCAPMASK, sc->sc_phy_addr, MII_OFFSET_ANY, 0); if (error) { device_printf(sc->sc_dev, "attaching PHYs failed: %d\n", error); dtsec_detach(sc->sc_dev); return (error); } sc->sc_mii = device_get_softc(sc->sc_mii_dev); /* Attach to stack */ ether_ifattach(ifp, sc->sc_mac_addr); return (0); } int dtsec_detach(device_t dev) { struct dtsec_softc *sc; if_t ifp; sc = device_get_softc(dev); ifp = sc->sc_ifnet; if (device_is_attached(dev)) { ether_ifdetach(ifp); /* Shutdown interface */ DTSEC_LOCK(sc); dtsec_if_deinit_locked(sc); DTSEC_UNLOCK(sc); } if (sc->sc_ifnet) { if_free(sc->sc_ifnet); sc->sc_ifnet = NULL; } if (sc->sc_mode == DTSEC_MODE_REGULAR) { /* Free RX/TX FQRs */ dtsec_rm_fqr_rx_free(sc); dtsec_rm_fqr_tx_free(sc); /* Free frame info pool */ dtsec_rm_fi_pool_free(sc); /* Free RX buffer pool */ dtsec_rm_pool_rx_free(sc); } dtsec_fm_mac_free(sc); dtsec_fm_port_free_both(sc); /* Destroy lock */ mtx_destroy(&sc->sc_lock); return (0); } int dtsec_suspend(device_t dev) { return (0); } int dtsec_resume(device_t dev) { return (0); } int dtsec_shutdown(device_t dev) { return (0); } /** @} */ /** * @group MII bus interface. * @{ */ int dtsec_miibus_readreg(device_t dev, int phy, int reg) { struct dtsec_softc *sc; sc = device_get_softc(dev); return (MIIBUS_READREG(sc->sc_mdio, phy, reg)); } int dtsec_miibus_writereg(device_t dev, int phy, int reg, int value) { struct dtsec_softc *sc; sc = device_get_softc(dev); return (MIIBUS_WRITEREG(sc->sc_mdio, phy, reg, value)); } void dtsec_miibus_statchg(device_t dev) { struct dtsec_softc *sc; e_EnetSpeed speed; bool duplex; int error; sc = device_get_softc(dev); DTSEC_LOCK_ASSERT(sc); duplex = ((sc->sc_mii->mii_media_active & IFM_GMASK) == IFM_FDX); switch (IFM_SUBTYPE(sc->sc_mii->mii_media_active)) { case IFM_1000_T: case IFM_1000_SX: speed = e_ENET_SPEED_1000; break; case IFM_100_TX: speed = e_ENET_SPEED_100; break; case IFM_10_T: speed = e_ENET_SPEED_10; break; default: speed = e_ENET_SPEED_10; } error = FM_MAC_AdjustLink(sc->sc_mach, speed, duplex); if (error != E_OK) device_printf(sc->sc_dev, "error while adjusting MAC speed.\n"); } /** @} */ Index: head/sys/dev/dpaa/if_dtsec_rm.c =================================================================== --- head/sys/dev/dpaa/if_dtsec_rm.c (revision 308717) +++ head/sys/dev/dpaa/if_dtsec_rm.c (revision 308718) @@ -1,655 +1,654 @@ /*- * Copyright (c) 2012 Semihalf. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "miibus_if.h" #include #include #include #include "fman.h" #include "bman.h" #include "qman.h" #include "if_dtsec.h" #include "if_dtsec_rm.h" /** * @group dTSEC RM private defines. * @{ */ #define DTSEC_BPOOLS_USED (1) #define DTSEC_MAX_TX_QUEUE_LEN 256 struct dtsec_rm_frame_info { struct mbuf *fi_mbuf; t_DpaaSGTE fi_sgt[DPAA_NUM_OF_SG_TABLE_ENTRY]; }; enum dtsec_rm_pool_params { DTSEC_RM_POOL_RX_LOW_MARK = 16, DTSEC_RM_POOL_RX_HIGH_MARK = 64, DTSEC_RM_POOL_RX_MAX_SIZE = 256, DTSEC_RM_POOL_FI_LOW_MARK = 16, DTSEC_RM_POOL_FI_HIGH_MARK = 64, DTSEC_RM_POOL_FI_MAX_SIZE = 256, }; enum dtsec_rm_fqr_params { DTSEC_RM_FQR_RX_CHANNEL = e_QM_FQ_CHANNEL_POOL1, DTSEC_RM_FQR_RX_WQ = 1, DTSEC_RM_FQR_TX_CONF_CHANNEL = e_QM_FQ_CHANNEL_SWPORTAL0, DTSEC_RM_FQR_TX_WQ = 1, DTSEC_RM_FQR_TX_CONF_WQ = 1 }; /** @} */ /** * @group dTSEC Frame Info routines. * @{ */ void dtsec_rm_fi_pool_free(struct dtsec_softc *sc) { if (sc->sc_fi_zone != NULL) uma_zdestroy(sc->sc_fi_zone); } int dtsec_rm_fi_pool_init(struct dtsec_softc *sc) { snprintf(sc->sc_fi_zname, sizeof(sc->sc_fi_zname), "%s: Frame Info", device_get_nameunit(sc->sc_dev)); sc->sc_fi_zone = uma_zcreate(sc->sc_fi_zname, sizeof(struct dtsec_rm_frame_info), NULL, NULL, NULL, NULL, sizeof(void *), 0); if (sc->sc_fi_zone == NULL) return (EIO); return (0); } static struct dtsec_rm_frame_info * dtsec_rm_fi_alloc(struct dtsec_softc *sc) { struct dtsec_rm_frame_info *fi; fi = uma_zalloc(sc->sc_fi_zone, M_NOWAIT); return (fi); } static void dtsec_rm_fi_free(struct dtsec_softc *sc, struct dtsec_rm_frame_info *fi) { - XX_UntrackAddress(fi); uma_zfree(sc->sc_fi_zone, fi); } /** @} */ /** * @group dTSEC FMan PORT routines. * @{ */ int dtsec_rm_fm_port_rx_init(struct dtsec_softc *sc, int unit) { t_FmPortParams params; t_FmPortRxParams *rx_params; t_FmPortExtPools *pool_params; t_Error error; memset(¶ms, 0, sizeof(params)); params.baseAddr = sc->sc_fm_base + sc->sc_port_rx_hw_id; params.h_Fm = sc->sc_fmh; params.portType = dtsec_fm_port_rx_type(sc->sc_eth_dev_type); params.portId = sc->sc_eth_id; params.independentModeEnable = FALSE; params.liodnBase = FM_PORT_LIODN_BASE; params.f_Exception = dtsec_fm_port_rx_exception_callback; params.h_App = sc; rx_params = ¶ms.specificParams.rxParams; rx_params->errFqid = sc->sc_rx_fqid; rx_params->dfltFqid = sc->sc_rx_fqid; rx_params->liodnOffset = 0; pool_params = &rx_params->extBufPools; pool_params->numOfPoolsUsed = DTSEC_BPOOLS_USED; pool_params->extBufPool->id = sc->sc_rx_bpid; pool_params->extBufPool->size = FM_PORT_BUFFER_SIZE; sc->sc_rxph = FM_PORT_Config(¶ms); if (sc->sc_rxph == NULL) { device_printf(sc->sc_dev, "couldn't configure FM Port RX.\n"); return (ENXIO); } error = FM_PORT_Init(sc->sc_rxph); if (error != E_OK) { device_printf(sc->sc_dev, "couldn't initialize FM Port RX.\n"); FM_PORT_Free(sc->sc_rxph); return (ENXIO); } if (bootverbose) device_printf(sc->sc_dev, "RX hw port 0x%02x initialized.\n", sc->sc_port_rx_hw_id); return (0); } int dtsec_rm_fm_port_tx_init(struct dtsec_softc *sc, int unit) { t_FmPortParams params; t_FmPortNonRxParams *tx_params; t_Error error; memset(¶ms, 0, sizeof(params)); params.baseAddr = sc->sc_fm_base + sc->sc_port_tx_hw_id; params.h_Fm = sc->sc_fmh; params.portType = dtsec_fm_port_tx_type(sc->sc_eth_dev_type); params.portId = sc->sc_eth_id; params.independentModeEnable = FALSE; params.liodnBase = FM_PORT_LIODN_BASE; params.f_Exception = dtsec_fm_port_tx_exception_callback; params.h_App = sc; tx_params = ¶ms.specificParams.nonRxParams; tx_params->errFqid = sc->sc_tx_conf_fqid; tx_params->dfltFqid = sc->sc_tx_conf_fqid; tx_params->qmChannel = sc->sc_port_tx_qman_chan; #ifdef FM_OP_PARTITION_ERRATA_FMANx8 tx_params->opLiodnOffset = 0; #endif sc->sc_txph = FM_PORT_Config(¶ms); if (sc->sc_txph == NULL) { device_printf(sc->sc_dev, "couldn't configure FM Port TX.\n"); return (ENXIO); } error = FM_PORT_Init(sc->sc_txph); if (error != E_OK) { device_printf(sc->sc_dev, "couldn't initialize FM Port TX.\n"); FM_PORT_Free(sc->sc_txph); return (ENXIO); } if (bootverbose) device_printf(sc->sc_dev, "TX hw port 0x%02x initialized.\n", sc->sc_port_tx_hw_id); return (0); } /** @} */ /** * @group dTSEC buffer pools routines. * @{ */ static t_Error dtsec_rm_pool_rx_put_buffer(t_Handle h_BufferPool, uint8_t *buffer, t_Handle context) { struct dtsec_softc *sc; sc = h_BufferPool; uma_zfree(sc->sc_rx_zone, buffer); return (E_OK); } static uint8_t * dtsec_rm_pool_rx_get_buffer(t_Handle h_BufferPool, t_Handle *context) { struct dtsec_softc *sc; uint8_t *buffer; sc = h_BufferPool; buffer = uma_zalloc(sc->sc_rx_zone, M_NOWAIT); return (buffer); } static void dtsec_rm_pool_rx_depleted(t_Handle h_App, bool in) { struct dtsec_softc *sc; unsigned int count; sc = h_App; if (!in) return; while (1) { count = bman_count(sc->sc_rx_pool); if (count > DTSEC_RM_POOL_RX_HIGH_MARK) return; bman_pool_fill(sc->sc_rx_pool, DTSEC_RM_POOL_RX_HIGH_MARK); } } void dtsec_rm_pool_rx_free(struct dtsec_softc *sc) { if (sc->sc_rx_pool != NULL) bman_pool_destroy(sc->sc_rx_pool); if (sc->sc_rx_zone != NULL) uma_zdestroy(sc->sc_rx_zone); } int dtsec_rm_pool_rx_init(struct dtsec_softc *sc) { /* FM_PORT_BUFFER_SIZE must be less than PAGE_SIZE */ CTASSERT(FM_PORT_BUFFER_SIZE < PAGE_SIZE); snprintf(sc->sc_rx_zname, sizeof(sc->sc_rx_zname), "%s: RX Buffers", device_get_nameunit(sc->sc_dev)); sc->sc_rx_zone = uma_zcreate(sc->sc_rx_zname, FM_PORT_BUFFER_SIZE, NULL, NULL, NULL, NULL, FM_PORT_BUFFER_SIZE, 0); if (sc->sc_rx_zone == NULL) return (EIO); sc->sc_rx_pool = bman_pool_create(&sc->sc_rx_bpid, FM_PORT_BUFFER_SIZE, 0, 0, DTSEC_RM_POOL_RX_MAX_SIZE, dtsec_rm_pool_rx_get_buffer, dtsec_rm_pool_rx_put_buffer, DTSEC_RM_POOL_RX_LOW_MARK, DTSEC_RM_POOL_RX_HIGH_MARK, 0, 0, dtsec_rm_pool_rx_depleted, sc, NULL, NULL); if (sc->sc_rx_pool == NULL) { device_printf(sc->sc_dev, "NULL rx pool somehow\n"); dtsec_rm_pool_rx_free(sc); return (EIO); } return (0); } /** @} */ /** * @group dTSEC Frame Queue Range routines. * @{ */ static void dtsec_rm_fqr_mext_free(struct mbuf *m, void *buffer, void *arg) { struct dtsec_softc *sc; sc = arg; if (bman_count(sc->sc_rx_pool) <= DTSEC_RM_POOL_RX_MAX_SIZE) bman_put_buffer(sc->sc_rx_pool, buffer); else dtsec_rm_pool_rx_put_buffer(arg, buffer, NULL); } static e_RxStoreResponse dtsec_rm_fqr_rx_callback(t_Handle app, t_Handle fqr, t_Handle portal, uint32_t fqid_off, t_DpaaFD *frame) { struct dtsec_softc *sc; struct mbuf *m; m = NULL; sc = app; KASSERT(DPAA_FD_GET_FORMAT(frame) == e_DPAA_FD_FORMAT_TYPE_SHORT_SBSF, ("%s(): Got unsupported frame format 0x%02X!", __func__, DPAA_FD_GET_FORMAT(frame))); KASSERT(DPAA_FD_GET_OFFSET(frame) == 0, ("%s(): Only offset 0 is supported!", __func__)); if (DPAA_FD_GET_STATUS(frame) != 0) { device_printf(sc->sc_dev, "RX error: 0x%08X\n", DPAA_FD_GET_STATUS(frame)); goto err; } m = m_gethdr(M_NOWAIT, MT_HEADER); if (m == NULL) goto err; m_extadd(m, DPAA_FD_GET_ADDR(frame), FM_PORT_BUFFER_SIZE, dtsec_rm_fqr_mext_free, DPAA_FD_GET_ADDR(frame), sc, 0, EXT_NET_DRV); m->m_pkthdr.rcvif = sc->sc_ifnet; m->m_len = DPAA_FD_GET_LENGTH(frame); m_fixhdr(m); (*sc->sc_ifnet->if_input)(sc->sc_ifnet, m); return (e_RX_STORE_RESPONSE_CONTINUE); err: bman_put_buffer(sc->sc_rx_pool, DPAA_FD_GET_ADDR(frame)); if (m != NULL) m_freem(m); return (e_RX_STORE_RESPONSE_CONTINUE); } static e_RxStoreResponse dtsec_rm_fqr_tx_confirm_callback(t_Handle app, t_Handle fqr, t_Handle portal, uint32_t fqid_off, t_DpaaFD *frame) { struct dtsec_rm_frame_info *fi; struct dtsec_softc *sc; unsigned int qlen; t_DpaaSGTE *sgt0; sc = app; if (DPAA_FD_GET_STATUS(frame) != 0) device_printf(sc->sc_dev, "TX error: 0x%08X\n", DPAA_FD_GET_STATUS(frame)); /* * We are storing struct dtsec_rm_frame_info in first entry * of scatter-gather table. */ sgt0 = DPAA_FD_GET_ADDR(frame); fi = DPAA_SGTE_GET_ADDR(sgt0); /* Free transmitted frame */ m_freem(fi->fi_mbuf); dtsec_rm_fi_free(sc, fi); qlen = qman_fqr_get_counter(sc->sc_tx_conf_fqr, 0, e_QM_FQR_COUNTERS_FRAME); if (qlen == 0) { DTSEC_LOCK(sc); if (sc->sc_tx_fqr_full) { sc->sc_tx_fqr_full = 0; dtsec_rm_if_start_locked(sc); } DTSEC_UNLOCK(sc); } return (e_RX_STORE_RESPONSE_CONTINUE); } void dtsec_rm_fqr_rx_free(struct dtsec_softc *sc) { if (sc->sc_rx_fqr) qman_fqr_free(sc->sc_rx_fqr); } int dtsec_rm_fqr_rx_init(struct dtsec_softc *sc) { t_Error error; t_Handle fqr; /* Default Frame Queue */ fqr = qman_fqr_create(1, DTSEC_RM_FQR_RX_CHANNEL, DTSEC_RM_FQR_RX_WQ, FALSE, 0, FALSE, FALSE, TRUE, FALSE, 0, 0, 0); if (fqr == NULL) { device_printf(sc->sc_dev, "could not create default RX queue" "\n"); return (EIO); } sc->sc_rx_fqr = fqr; sc->sc_rx_fqid = qman_fqr_get_base_fqid(fqr); error = qman_fqr_register_cb(fqr, dtsec_rm_fqr_rx_callback, sc); if (error != E_OK) { device_printf(sc->sc_dev, "could not register RX callback\n"); dtsec_rm_fqr_rx_free(sc); return (EIO); } return (0); } void dtsec_rm_fqr_tx_free(struct dtsec_softc *sc) { if (sc->sc_tx_fqr) qman_fqr_free(sc->sc_tx_fqr); if (sc->sc_tx_conf_fqr) qman_fqr_free(sc->sc_tx_conf_fqr); } int dtsec_rm_fqr_tx_init(struct dtsec_softc *sc) { t_Error error; t_Handle fqr; /* TX Frame Queue */ fqr = qman_fqr_create(1, sc->sc_port_tx_qman_chan, DTSEC_RM_FQR_TX_WQ, FALSE, 0, FALSE, FALSE, TRUE, FALSE, 0, 0, 0); if (fqr == NULL) { device_printf(sc->sc_dev, "could not create default TX queue" "\n"); return (EIO); } sc->sc_tx_fqr = fqr; /* TX Confirmation Frame Queue */ fqr = qman_fqr_create(1, DTSEC_RM_FQR_TX_CONF_CHANNEL, DTSEC_RM_FQR_TX_CONF_WQ, FALSE, 0, FALSE, FALSE, TRUE, FALSE, 0, 0, 0); if (fqr == NULL) { device_printf(sc->sc_dev, "could not create TX confirmation " "queue\n"); dtsec_rm_fqr_tx_free(sc); return (EIO); } sc->sc_tx_conf_fqr = fqr; sc->sc_tx_conf_fqid = qman_fqr_get_base_fqid(fqr); error = qman_fqr_register_cb(fqr, dtsec_rm_fqr_tx_confirm_callback, sc); if (error != E_OK) { device_printf(sc->sc_dev, "could not register TX confirmation " "callback\n"); dtsec_rm_fqr_tx_free(sc); return (EIO); } return (0); } /** @} */ /** * @group dTSEC IFnet routines. * @{ */ void dtsec_rm_if_start_locked(struct dtsec_softc *sc) { vm_size_t dsize, psize, ssize; struct dtsec_rm_frame_info *fi; unsigned int qlen, i; struct mbuf *m0, *m; vm_offset_t vaddr; vm_paddr_t paddr; t_DpaaFD fd; DTSEC_LOCK_ASSERT(sc); /* TODO: IFF_DRV_OACTIVE */ if ((sc->sc_mii->mii_media_status & IFM_ACTIVE) == 0) return; if ((sc->sc_ifnet->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) return; while (!IFQ_DRV_IS_EMPTY(&sc->sc_ifnet->if_snd)) { /* Check length of the TX queue */ qlen = qman_fqr_get_counter(sc->sc_tx_fqr, 0, e_QM_FQR_COUNTERS_FRAME); if (qlen >= DTSEC_MAX_TX_QUEUE_LEN) { sc->sc_tx_fqr_full = 1; return; } fi = dtsec_rm_fi_alloc(sc); if (fi == NULL) return; IFQ_DRV_DEQUEUE(&sc->sc_ifnet->if_snd, m0); if (m0 == NULL) { dtsec_rm_fi_free(sc, fi); return; } i = 0; m = m0; psize = 0; dsize = 0; fi->fi_mbuf = m0; while (m && i < DPAA_NUM_OF_SG_TABLE_ENTRY) { if (m->m_len == 0) continue; /* * First entry in scatter-gather table is used to keep * pointer to frame info structure. */ DPAA_SGTE_SET_ADDR(&fi->fi_sgt[i], (void *)fi); DPAA_SGTE_SET_LENGTH(&fi->fi_sgt[i], 0); DPAA_SGTE_SET_EXTENSION(&fi->fi_sgt[i], 0); DPAA_SGTE_SET_FINAL(&fi->fi_sgt[i], 0); DPAA_SGTE_SET_BPID(&fi->fi_sgt[i], 0); DPAA_SGTE_SET_OFFSET(&fi->fi_sgt[i], 0); i++; dsize = m->m_len; vaddr = (vm_offset_t)m->m_data; while (dsize > 0 && i < DPAA_NUM_OF_SG_TABLE_ENTRY) { paddr = XX_VirtToPhys((void *)vaddr); ssize = PAGE_SIZE - (paddr & PAGE_MASK); if (m->m_len < ssize) ssize = m->m_len; DPAA_SGTE_SET_ADDR(&fi->fi_sgt[i], (void *)vaddr); DPAA_SGTE_SET_LENGTH(&fi->fi_sgt[i], ssize); DPAA_SGTE_SET_EXTENSION(&fi->fi_sgt[i], 0); DPAA_SGTE_SET_FINAL(&fi->fi_sgt[i], 0); DPAA_SGTE_SET_BPID(&fi->fi_sgt[i], 0); DPAA_SGTE_SET_OFFSET(&fi->fi_sgt[i], 0); dsize -= ssize; vaddr += ssize; psize += ssize; i++; } if (dsize > 0) break; m = m->m_next; } /* Check if SG table was constructed properly */ if (m != NULL || dsize != 0) { dtsec_rm_fi_free(sc, fi); m_freem(m0); continue; } DPAA_SGTE_SET_FINAL(&fi->fi_sgt[i-1], 1); DPAA_FD_SET_ADDR(&fd, fi->fi_sgt); DPAA_FD_SET_LENGTH(&fd, psize); DPAA_FD_SET_FORMAT(&fd, e_DPAA_FD_FORMAT_TYPE_SHORT_MBSF); DPAA_FD_SET_DD(&fd, 0); DPAA_FD_SET_PID(&fd, 0); DPAA_FD_SET_BPID(&fd, 0); DPAA_FD_SET_OFFSET(&fd, 0); DPAA_FD_SET_STATUS(&fd, 0); DTSEC_UNLOCK(sc); if (qman_fqr_enqueue(sc->sc_tx_fqr, 0, &fd) != E_OK) { dtsec_rm_fi_free(sc, fi); m_freem(m0); } DTSEC_LOCK(sc); } } /** @} */ Index: head/sys/powerpc/booke/pmap.c =================================================================== --- head/sys/powerpc/booke/pmap.c (revision 308717) +++ head/sys/powerpc/booke/pmap.c (revision 308718) @@ -1,3607 +1,3644 @@ /*- * Copyright (C) 2007-2009 Semihalf, Rafal Jaworowski * Copyright (C) 2006 Semihalf, Marian Balakowicz * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Some hw specific parts of this pmap were derived or influenced * by NetBSD's ibm4xx pmap module. More generic code is shared with * a few other pmap modules from the FreeBSD tree. */ /* * VM layout notes: * * Kernel and user threads run within one common virtual address space * defined by AS=0. * * Virtual address space layout: * ----------------------------- * 0x0000_0000 - 0xafff_ffff : user process * 0xb000_0000 - 0xbfff_ffff : pmap_mapdev()-ed area (PCI/PCIE etc.) * 0xc000_0000 - 0xc0ff_ffff : kernel reserved * 0xc000_0000 - data_end : kernel code+data, env, metadata etc. * 0xc100_0000 - 0xfeef_ffff : KVA * 0xc100_0000 - 0xc100_3fff : reserved for page zero/copy * 0xc100_4000 - 0xc200_3fff : reserved for ptbl bufs * 0xc200_4000 - 0xc200_8fff : guard page + kstack0 * 0xc200_9000 - 0xfeef_ffff : actual free KVA space * 0xfef0_0000 - 0xffff_ffff : I/O devices region */ #include __FBSDID("$FreeBSD$"); #include "opt_kstack_pages.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "mmu_if.h" #define SPARSE_MAPDEV #ifdef DEBUG #define debugf(fmt, args...) printf(fmt, ##args) #else #define debugf(fmt, args...) #endif #define TODO panic("%s: not implemented", __func__); extern unsigned char _etext[]; extern unsigned char _end[]; extern uint32_t *bootinfo; vm_paddr_t kernload; vm_offset_t kernstart; vm_size_t kernsize; /* Message buffer and tables. */ static vm_offset_t data_start; static vm_size_t data_end; /* Phys/avail memory regions. */ static struct mem_region *availmem_regions; static int availmem_regions_sz; static struct mem_region *physmem_regions; static int physmem_regions_sz; /* Reserved KVA space and mutex for mmu_booke_zero_page. */ static vm_offset_t zero_page_va; static struct mtx zero_page_mutex; static struct mtx tlbivax_mutex; /* Reserved KVA space and mutex for mmu_booke_copy_page. */ static vm_offset_t copy_page_src_va; static vm_offset_t copy_page_dst_va; static struct mtx copy_page_mutex; /**************************************************************************/ /* PMAP */ /**************************************************************************/ static int mmu_booke_enter_locked(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int flags, int8_t psind); unsigned int kptbl_min; /* Index of the first kernel ptbl. */ unsigned int kernel_ptbls; /* Number of KVA ptbls. */ /* * If user pmap is processed with mmu_booke_remove and the resident count * drops to 0, there are no more pages to remove, so we need not continue. */ #define PMAP_REMOVE_DONE(pmap) \ ((pmap) != kernel_pmap && (pmap)->pm_stats.resident_count == 0) extern int elf32_nxstack; /**************************************************************************/ /* TLB and TID handling */ /**************************************************************************/ /* Translation ID busy table */ static volatile pmap_t tidbusy[MAXCPU][TID_MAX + 1]; /* * TLB0 capabilities (entry, way numbers etc.). These can vary between e500 * core revisions and should be read from h/w registers during early config. */ uint32_t tlb0_entries; uint32_t tlb0_ways; uint32_t tlb0_entries_per_way; uint32_t tlb1_entries; #define TLB0_ENTRIES (tlb0_entries) #define TLB0_WAYS (tlb0_ways) #define TLB0_ENTRIES_PER_WAY (tlb0_entries_per_way) #define TLB1_ENTRIES (tlb1_entries) #define TLB1_MAXENTRIES 64 static vm_offset_t tlb1_map_base = VM_MAXUSER_ADDRESS + PAGE_SIZE; static tlbtid_t tid_alloc(struct pmap *); static void tid_flush(tlbtid_t tid); static void tlb_print_entry(int, uint32_t, uint32_t, uint32_t, uint32_t); static void tlb1_read_entry(tlb_entry_t *, unsigned int); static void tlb1_write_entry(tlb_entry_t *, unsigned int); static int tlb1_iomapped(int, vm_paddr_t, vm_size_t, vm_offset_t *); static vm_size_t tlb1_mapin_region(vm_offset_t, vm_paddr_t, vm_size_t); static vm_size_t tsize2size(unsigned int); static unsigned int size2tsize(vm_size_t); static unsigned int ilog2(unsigned int); static void set_mas4_defaults(void); static inline void tlb0_flush_entry(vm_offset_t); static inline unsigned int tlb0_tableidx(vm_offset_t, unsigned int); /**************************************************************************/ /* Page table management */ /**************************************************************************/ static struct rwlock_padalign pvh_global_lock; /* Data for the pv entry allocation mechanism */ static uma_zone_t pvzone; static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; #define PV_ENTRY_ZONE_MIN 2048 /* min pv entries in uma zone */ #ifndef PMAP_SHPGPERPROC #define PMAP_SHPGPERPROC 200 #endif static void ptbl_init(void); static struct ptbl_buf *ptbl_buf_alloc(void); static void ptbl_buf_free(struct ptbl_buf *); static void ptbl_free_pmap_ptbl(pmap_t, pte_t *); static pte_t *ptbl_alloc(mmu_t, pmap_t, unsigned int, boolean_t); static void ptbl_free(mmu_t, pmap_t, unsigned int); static void ptbl_hold(mmu_t, pmap_t, unsigned int); static int ptbl_unhold(mmu_t, pmap_t, unsigned int); static vm_paddr_t pte_vatopa(mmu_t, pmap_t, vm_offset_t); static pte_t *pte_find(mmu_t, pmap_t, vm_offset_t); static int pte_enter(mmu_t, pmap_t, vm_page_t, vm_offset_t, uint32_t, boolean_t); static int pte_remove(mmu_t, pmap_t, vm_offset_t, uint8_t); static void kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr, vm_offset_t pdir); static pv_entry_t pv_alloc(void); static void pv_free(pv_entry_t); static void pv_insert(pmap_t, vm_offset_t, vm_page_t); static void pv_remove(pmap_t, vm_offset_t, vm_page_t); static void booke_pmap_init_qpages(void); /* Number of kva ptbl buffers, each covering one ptbl (PTBL_PAGES). */ #define PTBL_BUFS (128 * 16) struct ptbl_buf { TAILQ_ENTRY(ptbl_buf) link; /* list link */ vm_offset_t kva; /* va of mapping */ }; /* ptbl free list and a lock used for access synchronization. */ static TAILQ_HEAD(, ptbl_buf) ptbl_buf_freelist; static struct mtx ptbl_buf_freelist_lock; /* Base address of kva space allocated fot ptbl bufs. */ static vm_offset_t ptbl_buf_pool_vabase; /* Pointer to ptbl_buf structures. */ static struct ptbl_buf *ptbl_bufs; #ifdef SMP extern tlb_entry_t __boot_tlb1[]; void pmap_bootstrap_ap(volatile uint32_t *); #endif /* * Kernel MMU interface */ static void mmu_booke_clear_modify(mmu_t, vm_page_t); static void mmu_booke_copy(mmu_t, pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t); static void mmu_booke_copy_page(mmu_t, vm_page_t, vm_page_t); static void mmu_booke_copy_pages(mmu_t, vm_page_t *, vm_offset_t, vm_page_t *, vm_offset_t, int); static int mmu_booke_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int flags, int8_t psind); static void mmu_booke_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_page_t, vm_prot_t); static void mmu_booke_enter_quick(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t); static vm_paddr_t mmu_booke_extract(mmu_t, pmap_t, vm_offset_t); static vm_page_t mmu_booke_extract_and_hold(mmu_t, pmap_t, vm_offset_t, vm_prot_t); static void mmu_booke_init(mmu_t); static boolean_t mmu_booke_is_modified(mmu_t, vm_page_t); static boolean_t mmu_booke_is_prefaultable(mmu_t, pmap_t, vm_offset_t); static boolean_t mmu_booke_is_referenced(mmu_t, vm_page_t); static int mmu_booke_ts_referenced(mmu_t, vm_page_t); static vm_offset_t mmu_booke_map(mmu_t, vm_offset_t *, vm_paddr_t, vm_paddr_t, int); static int mmu_booke_mincore(mmu_t, pmap_t, vm_offset_t, vm_paddr_t *); static void mmu_booke_object_init_pt(mmu_t, pmap_t, vm_offset_t, vm_object_t, vm_pindex_t, vm_size_t); static boolean_t mmu_booke_page_exists_quick(mmu_t, pmap_t, vm_page_t); static void mmu_booke_page_init(mmu_t, vm_page_t); static int mmu_booke_page_wired_mappings(mmu_t, vm_page_t); static void mmu_booke_pinit(mmu_t, pmap_t); static void mmu_booke_pinit0(mmu_t, pmap_t); static void mmu_booke_protect(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_prot_t); static void mmu_booke_qenter(mmu_t, vm_offset_t, vm_page_t *, int); static void mmu_booke_qremove(mmu_t, vm_offset_t, int); static void mmu_booke_release(mmu_t, pmap_t); static void mmu_booke_remove(mmu_t, pmap_t, vm_offset_t, vm_offset_t); static void mmu_booke_remove_all(mmu_t, vm_page_t); static void mmu_booke_remove_write(mmu_t, vm_page_t); static void mmu_booke_unwire(mmu_t, pmap_t, vm_offset_t, vm_offset_t); static void mmu_booke_zero_page(mmu_t, vm_page_t); static void mmu_booke_zero_page_area(mmu_t, vm_page_t, int, int); static void mmu_booke_activate(mmu_t, struct thread *); static void mmu_booke_deactivate(mmu_t, struct thread *); static void mmu_booke_bootstrap(mmu_t, vm_offset_t, vm_offset_t); static void *mmu_booke_mapdev(mmu_t, vm_paddr_t, vm_size_t); static void *mmu_booke_mapdev_attr(mmu_t, vm_paddr_t, vm_size_t, vm_memattr_t); static void mmu_booke_unmapdev(mmu_t, vm_offset_t, vm_size_t); static vm_paddr_t mmu_booke_kextract(mmu_t, vm_offset_t); static void mmu_booke_kenter(mmu_t, vm_offset_t, vm_paddr_t); static void mmu_booke_kenter_attr(mmu_t, vm_offset_t, vm_paddr_t, vm_memattr_t); static void mmu_booke_kremove(mmu_t, vm_offset_t); static boolean_t mmu_booke_dev_direct_mapped(mmu_t, vm_paddr_t, vm_size_t); static void mmu_booke_sync_icache(mmu_t, pmap_t, vm_offset_t, vm_size_t); static void mmu_booke_dumpsys_map(mmu_t, vm_paddr_t pa, size_t, void **); static void mmu_booke_dumpsys_unmap(mmu_t, vm_paddr_t pa, size_t, void *); static void mmu_booke_scan_init(mmu_t); static vm_offset_t mmu_booke_quick_enter_page(mmu_t mmu, vm_page_t m); static void mmu_booke_quick_remove_page(mmu_t mmu, vm_offset_t addr); static int mmu_booke_change_attr(mmu_t mmu, vm_offset_t addr, vm_size_t sz, vm_memattr_t mode); static mmu_method_t mmu_booke_methods[] = { /* pmap dispatcher interface */ MMUMETHOD(mmu_clear_modify, mmu_booke_clear_modify), MMUMETHOD(mmu_copy, mmu_booke_copy), MMUMETHOD(mmu_copy_page, mmu_booke_copy_page), MMUMETHOD(mmu_copy_pages, mmu_booke_copy_pages), MMUMETHOD(mmu_enter, mmu_booke_enter), MMUMETHOD(mmu_enter_object, mmu_booke_enter_object), MMUMETHOD(mmu_enter_quick, mmu_booke_enter_quick), MMUMETHOD(mmu_extract, mmu_booke_extract), MMUMETHOD(mmu_extract_and_hold, mmu_booke_extract_and_hold), MMUMETHOD(mmu_init, mmu_booke_init), MMUMETHOD(mmu_is_modified, mmu_booke_is_modified), MMUMETHOD(mmu_is_prefaultable, mmu_booke_is_prefaultable), MMUMETHOD(mmu_is_referenced, mmu_booke_is_referenced), MMUMETHOD(mmu_ts_referenced, mmu_booke_ts_referenced), MMUMETHOD(mmu_map, mmu_booke_map), MMUMETHOD(mmu_mincore, mmu_booke_mincore), MMUMETHOD(mmu_object_init_pt, mmu_booke_object_init_pt), MMUMETHOD(mmu_page_exists_quick,mmu_booke_page_exists_quick), MMUMETHOD(mmu_page_init, mmu_booke_page_init), MMUMETHOD(mmu_page_wired_mappings, mmu_booke_page_wired_mappings), MMUMETHOD(mmu_pinit, mmu_booke_pinit), MMUMETHOD(mmu_pinit0, mmu_booke_pinit0), MMUMETHOD(mmu_protect, mmu_booke_protect), MMUMETHOD(mmu_qenter, mmu_booke_qenter), MMUMETHOD(mmu_qremove, mmu_booke_qremove), MMUMETHOD(mmu_release, mmu_booke_release), MMUMETHOD(mmu_remove, mmu_booke_remove), MMUMETHOD(mmu_remove_all, mmu_booke_remove_all), MMUMETHOD(mmu_remove_write, mmu_booke_remove_write), MMUMETHOD(mmu_sync_icache, mmu_booke_sync_icache), MMUMETHOD(mmu_unwire, mmu_booke_unwire), MMUMETHOD(mmu_zero_page, mmu_booke_zero_page), MMUMETHOD(mmu_zero_page_area, mmu_booke_zero_page_area), MMUMETHOD(mmu_activate, mmu_booke_activate), MMUMETHOD(mmu_deactivate, mmu_booke_deactivate), MMUMETHOD(mmu_quick_enter_page, mmu_booke_quick_enter_page), MMUMETHOD(mmu_quick_remove_page, mmu_booke_quick_remove_page), /* Internal interfaces */ MMUMETHOD(mmu_bootstrap, mmu_booke_bootstrap), MMUMETHOD(mmu_dev_direct_mapped,mmu_booke_dev_direct_mapped), MMUMETHOD(mmu_mapdev, mmu_booke_mapdev), MMUMETHOD(mmu_mapdev_attr, mmu_booke_mapdev_attr), MMUMETHOD(mmu_kenter, mmu_booke_kenter), MMUMETHOD(mmu_kenter_attr, mmu_booke_kenter_attr), MMUMETHOD(mmu_kextract, mmu_booke_kextract), MMUMETHOD(mmu_kremove, mmu_booke_kremove), MMUMETHOD(mmu_unmapdev, mmu_booke_unmapdev), MMUMETHOD(mmu_change_attr, mmu_booke_change_attr), /* dumpsys() support */ MMUMETHOD(mmu_dumpsys_map, mmu_booke_dumpsys_map), MMUMETHOD(mmu_dumpsys_unmap, mmu_booke_dumpsys_unmap), MMUMETHOD(mmu_scan_init, mmu_booke_scan_init), { 0, 0 } }; MMU_DEF(booke_mmu, MMU_TYPE_BOOKE, mmu_booke_methods, 0); static __inline uint32_t tlb_calc_wimg(vm_paddr_t pa, vm_memattr_t ma) { uint32_t attrib; int i; if (ma != VM_MEMATTR_DEFAULT) { switch (ma) { case VM_MEMATTR_UNCACHEABLE: return (MAS2_I | MAS2_G); case VM_MEMATTR_WRITE_COMBINING: case VM_MEMATTR_WRITE_BACK: case VM_MEMATTR_PREFETCHABLE: return (MAS2_I); case VM_MEMATTR_WRITE_THROUGH: return (MAS2_W | MAS2_M); case VM_MEMATTR_CACHEABLE: return (MAS2_M); } } /* * Assume the page is cache inhibited and access is guarded unless * it's in our available memory array. */ attrib = _TLB_ENTRY_IO; for (i = 0; i < physmem_regions_sz; i++) { if ((pa >= physmem_regions[i].mr_start) && (pa < (physmem_regions[i].mr_start + physmem_regions[i].mr_size))) { attrib = _TLB_ENTRY_MEM; break; } } return (attrib); } static inline void tlb_miss_lock(void) { #ifdef SMP struct pcpu *pc; if (!smp_started) return; STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { if (pc != pcpup) { CTR3(KTR_PMAP, "%s: tlb miss LOCK of CPU=%d, " "tlb_lock=%p", __func__, pc->pc_cpuid, pc->pc_booke_tlb_lock); KASSERT((pc->pc_cpuid != PCPU_GET(cpuid)), ("tlb_miss_lock: tried to lock self")); tlb_lock(pc->pc_booke_tlb_lock); CTR1(KTR_PMAP, "%s: locked", __func__); } } #endif } static inline void tlb_miss_unlock(void) { #ifdef SMP struct pcpu *pc; if (!smp_started) return; STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { if (pc != pcpup) { CTR2(KTR_PMAP, "%s: tlb miss UNLOCK of CPU=%d", __func__, pc->pc_cpuid); tlb_unlock(pc->pc_booke_tlb_lock); CTR1(KTR_PMAP, "%s: unlocked", __func__); } } #endif } /* Return number of entries in TLB0. */ static __inline void tlb0_get_tlbconf(void) { uint32_t tlb0_cfg; tlb0_cfg = mfspr(SPR_TLB0CFG); tlb0_entries = tlb0_cfg & TLBCFG_NENTRY_MASK; tlb0_ways = (tlb0_cfg & TLBCFG_ASSOC_MASK) >> TLBCFG_ASSOC_SHIFT; tlb0_entries_per_way = tlb0_entries / tlb0_ways; } /* Return number of entries in TLB1. */ static __inline void tlb1_get_tlbconf(void) { uint32_t tlb1_cfg; tlb1_cfg = mfspr(SPR_TLB1CFG); tlb1_entries = tlb1_cfg & TLBCFG_NENTRY_MASK; } /**************************************************************************/ /* Page table related */ /**************************************************************************/ /* Initialize pool of kva ptbl buffers. */ static void ptbl_init(void) { int i; CTR3(KTR_PMAP, "%s: s (ptbl_bufs = 0x%08x size 0x%08x)", __func__, (uint32_t)ptbl_bufs, sizeof(struct ptbl_buf) * PTBL_BUFS); CTR3(KTR_PMAP, "%s: s (ptbl_buf_pool_vabase = 0x%08x size = 0x%08x)", __func__, ptbl_buf_pool_vabase, PTBL_BUFS * PTBL_PAGES * PAGE_SIZE); mtx_init(&ptbl_buf_freelist_lock, "ptbl bufs lock", NULL, MTX_DEF); TAILQ_INIT(&ptbl_buf_freelist); for (i = 0; i < PTBL_BUFS; i++) { ptbl_bufs[i].kva = ptbl_buf_pool_vabase + i * PTBL_PAGES * PAGE_SIZE; TAILQ_INSERT_TAIL(&ptbl_buf_freelist, &ptbl_bufs[i], link); } } /* Get a ptbl_buf from the freelist. */ static struct ptbl_buf * ptbl_buf_alloc(void) { struct ptbl_buf *buf; mtx_lock(&ptbl_buf_freelist_lock); buf = TAILQ_FIRST(&ptbl_buf_freelist); if (buf != NULL) TAILQ_REMOVE(&ptbl_buf_freelist, buf, link); mtx_unlock(&ptbl_buf_freelist_lock); CTR2(KTR_PMAP, "%s: buf = %p", __func__, buf); return (buf); } /* Return ptbl buff to free pool. */ static void ptbl_buf_free(struct ptbl_buf *buf) { CTR2(KTR_PMAP, "%s: buf = %p", __func__, buf); mtx_lock(&ptbl_buf_freelist_lock); TAILQ_INSERT_TAIL(&ptbl_buf_freelist, buf, link); mtx_unlock(&ptbl_buf_freelist_lock); } /* * Search the list of allocated ptbl bufs and find on list of allocated ptbls */ static void ptbl_free_pmap_ptbl(pmap_t pmap, pte_t *ptbl) { struct ptbl_buf *pbuf; CTR2(KTR_PMAP, "%s: ptbl = %p", __func__, ptbl); PMAP_LOCK_ASSERT(pmap, MA_OWNED); TAILQ_FOREACH(pbuf, &pmap->pm_ptbl_list, link) if (pbuf->kva == (vm_offset_t)ptbl) { /* Remove from pmap ptbl buf list. */ TAILQ_REMOVE(&pmap->pm_ptbl_list, pbuf, link); /* Free corresponding ptbl buf. */ ptbl_buf_free(pbuf); break; } } /* Allocate page table. */ static pte_t * ptbl_alloc(mmu_t mmu, pmap_t pmap, unsigned int pdir_idx, boolean_t nosleep) { vm_page_t mtbl[PTBL_PAGES]; vm_page_t m; struct ptbl_buf *pbuf; unsigned int pidx; pte_t *ptbl; int i, j; CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap, (pmap == kernel_pmap), pdir_idx); KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)), ("ptbl_alloc: invalid pdir_idx")); KASSERT((pmap->pm_pdir[pdir_idx] == NULL), ("pte_alloc: valid ptbl entry exists!")); pbuf = ptbl_buf_alloc(); if (pbuf == NULL) panic("pte_alloc: couldn't alloc kernel virtual memory"); ptbl = (pte_t *)pbuf->kva; CTR2(KTR_PMAP, "%s: ptbl kva = %p", __func__, ptbl); /* Allocate ptbl pages, this will sleep! */ for (i = 0; i < PTBL_PAGES; i++) { pidx = (PTBL_PAGES * pdir_idx) + i; while ((m = vm_page_alloc(NULL, pidx, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { PMAP_UNLOCK(pmap); rw_wunlock(&pvh_global_lock); if (nosleep) { ptbl_free_pmap_ptbl(pmap, ptbl); for (j = 0; j < i; j++) vm_page_free(mtbl[j]); atomic_subtract_int(&vm_cnt.v_wire_count, i); return (NULL); } VM_WAIT; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); } mtbl[i] = m; } /* Map allocated pages into kernel_pmap. */ mmu_booke_qenter(mmu, (vm_offset_t)ptbl, mtbl, PTBL_PAGES); /* Zero whole ptbl. */ bzero((caddr_t)ptbl, PTBL_PAGES * PAGE_SIZE); /* Add pbuf to the pmap ptbl bufs list. */ TAILQ_INSERT_TAIL(&pmap->pm_ptbl_list, pbuf, link); return (ptbl); } /* Free ptbl pages and invalidate pdir entry. */ static void ptbl_free(mmu_t mmu, pmap_t pmap, unsigned int pdir_idx) { pte_t *ptbl; vm_paddr_t pa; vm_offset_t va; vm_page_t m; int i; CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap, (pmap == kernel_pmap), pdir_idx); KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)), ("ptbl_free: invalid pdir_idx")); ptbl = pmap->pm_pdir[pdir_idx]; CTR2(KTR_PMAP, "%s: ptbl = %p", __func__, ptbl); KASSERT((ptbl != NULL), ("ptbl_free: null ptbl")); /* * Invalidate the pdir entry as soon as possible, so that other CPUs * don't attempt to look up the page tables we are releasing. */ mtx_lock_spin(&tlbivax_mutex); tlb_miss_lock(); pmap->pm_pdir[pdir_idx] = NULL; tlb_miss_unlock(); mtx_unlock_spin(&tlbivax_mutex); for (i = 0; i < PTBL_PAGES; i++) { va = ((vm_offset_t)ptbl + (i * PAGE_SIZE)); pa = pte_vatopa(mmu, kernel_pmap, va); m = PHYS_TO_VM_PAGE(pa); vm_page_free_zero(m); atomic_subtract_int(&vm_cnt.v_wire_count, 1); mmu_booke_kremove(mmu, va); } ptbl_free_pmap_ptbl(pmap, ptbl); } /* * Decrement ptbl pages hold count and attempt to free ptbl pages. * Called when removing pte entry from ptbl. * * Return 1 if ptbl pages were freed. */ static int ptbl_unhold(mmu_t mmu, pmap_t pmap, unsigned int pdir_idx) { pte_t *ptbl; vm_paddr_t pa; vm_page_t m; int i; CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap, (pmap == kernel_pmap), pdir_idx); KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)), ("ptbl_unhold: invalid pdir_idx")); KASSERT((pmap != kernel_pmap), ("ptbl_unhold: unholding kernel ptbl!")); ptbl = pmap->pm_pdir[pdir_idx]; //debugf("ptbl_unhold: ptbl = 0x%08x\n", (u_int32_t)ptbl); KASSERT(((vm_offset_t)ptbl >= VM_MIN_KERNEL_ADDRESS), ("ptbl_unhold: non kva ptbl")); /* decrement hold count */ for (i = 0; i < PTBL_PAGES; i++) { pa = pte_vatopa(mmu, kernel_pmap, (vm_offset_t)ptbl + (i * PAGE_SIZE)); m = PHYS_TO_VM_PAGE(pa); m->wire_count--; } /* * Free ptbl pages if there are no pte etries in this ptbl. * wire_count has the same value for all ptbl pages, so check the last * page. */ if (m->wire_count == 0) { ptbl_free(mmu, pmap, pdir_idx); //debugf("ptbl_unhold: e (freed ptbl)\n"); return (1); } return (0); } /* * Increment hold count for ptbl pages. This routine is used when a new pte * entry is being inserted into the ptbl. */ static void ptbl_hold(mmu_t mmu, pmap_t pmap, unsigned int pdir_idx) { vm_paddr_t pa; pte_t *ptbl; vm_page_t m; int i; CTR3(KTR_PMAP, "%s: pmap = %p pdir_idx = %d", __func__, pmap, pdir_idx); KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)), ("ptbl_hold: invalid pdir_idx")); KASSERT((pmap != kernel_pmap), ("ptbl_hold: holding kernel ptbl!")); ptbl = pmap->pm_pdir[pdir_idx]; KASSERT((ptbl != NULL), ("ptbl_hold: null ptbl")); for (i = 0; i < PTBL_PAGES; i++) { pa = pte_vatopa(mmu, kernel_pmap, (vm_offset_t)ptbl + (i * PAGE_SIZE)); m = PHYS_TO_VM_PAGE(pa); m->wire_count++; } } /* Allocate pv_entry structure. */ pv_entry_t pv_alloc(void) { pv_entry_t pv; pv_entry_count++; if (pv_entry_count > pv_entry_high_water) pagedaemon_wakeup(); pv = uma_zalloc(pvzone, M_NOWAIT); return (pv); } /* Free pv_entry structure. */ static __inline void pv_free(pv_entry_t pve) { pv_entry_count--; uma_zfree(pvzone, pve); } /* Allocate and initialize pv_entry structure. */ static void pv_insert(pmap_t pmap, vm_offset_t va, vm_page_t m) { pv_entry_t pve; //int su = (pmap == kernel_pmap); //debugf("pv_insert: s (su = %d pmap = 0x%08x va = 0x%08x m = 0x%08x)\n", su, // (u_int32_t)pmap, va, (u_int32_t)m); pve = pv_alloc(); if (pve == NULL) panic("pv_insert: no pv entries!"); pve->pv_pmap = pmap; pve->pv_va = va; /* add to pv_list */ PMAP_LOCK_ASSERT(pmap, MA_OWNED); rw_assert(&pvh_global_lock, RA_WLOCKED); TAILQ_INSERT_TAIL(&m->md.pv_list, pve, pv_link); //debugf("pv_insert: e\n"); } /* Destroy pv entry. */ static void pv_remove(pmap_t pmap, vm_offset_t va, vm_page_t m) { pv_entry_t pve; //int su = (pmap == kernel_pmap); //debugf("pv_remove: s (su = %d pmap = 0x%08x va = 0x%08x)\n", su, (u_int32_t)pmap, va); PMAP_LOCK_ASSERT(pmap, MA_OWNED); rw_assert(&pvh_global_lock, RA_WLOCKED); /* find pv entry */ TAILQ_FOREACH(pve, &m->md.pv_list, pv_link) { if ((pmap == pve->pv_pmap) && (va == pve->pv_va)) { /* remove from pv_list */ TAILQ_REMOVE(&m->md.pv_list, pve, pv_link); if (TAILQ_EMPTY(&m->md.pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); /* free pv entry struct */ pv_free(pve); break; } } //debugf("pv_remove: e\n"); } /* * Clean pte entry, try to free page table page if requested. * * Return 1 if ptbl pages were freed, otherwise return 0. */ static int pte_remove(mmu_t mmu, pmap_t pmap, vm_offset_t va, uint8_t flags) { unsigned int pdir_idx = PDIR_IDX(va); unsigned int ptbl_idx = PTBL_IDX(va); vm_page_t m; pte_t *ptbl; pte_t *pte; //int su = (pmap == kernel_pmap); //debugf("pte_remove: s (su = %d pmap = 0x%08x va = 0x%08x flags = %d)\n", // su, (u_int32_t)pmap, va, flags); ptbl = pmap->pm_pdir[pdir_idx]; KASSERT(ptbl, ("pte_remove: null ptbl")); pte = &ptbl[ptbl_idx]; if (pte == NULL || !PTE_ISVALID(pte)) return (0); if (PTE_ISWIRED(pte)) pmap->pm_stats.wired_count--; + /* Get vm_page_t for mapped pte. */ + m = PHYS_TO_VM_PAGE(PTE_PA(pte)); + /* Handle managed entry. */ if (PTE_ISMANAGED(pte)) { - /* Get vm_page_t for mapped pte. */ - m = PHYS_TO_VM_PAGE(PTE_PA(pte)); if (PTE_ISMODIFIED(pte)) vm_page_dirty(m); if (PTE_ISREFERENCED(pte)) vm_page_aflag_set(m, PGA_REFERENCED); pv_remove(pmap, va, m); + } else if (m->md.pv_tracked) { + /* + * Always pv_insert()/pv_remove() on MPC85XX, in case DPAA is + * used. This is needed by the NCSW support code for fast + * VA<->PA translation. + */ + pv_remove(pmap, va, m); + if (TAILQ_EMPTY(&m->md.pv_list)) + m->md.pv_tracked = false; } mtx_lock_spin(&tlbivax_mutex); tlb_miss_lock(); tlb0_flush_entry(va); *pte = 0; tlb_miss_unlock(); mtx_unlock_spin(&tlbivax_mutex); pmap->pm_stats.resident_count--; if (flags & PTBL_UNHOLD) { //debugf("pte_remove: e (unhold)\n"); return (ptbl_unhold(mmu, pmap, pdir_idx)); } //debugf("pte_remove: e\n"); return (0); } /* * Insert PTE for a given page and virtual address. */ static int pte_enter(mmu_t mmu, pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags, boolean_t nosleep) { unsigned int pdir_idx = PDIR_IDX(va); unsigned int ptbl_idx = PTBL_IDX(va); pte_t *ptbl, *pte; CTR4(KTR_PMAP, "%s: su = %d pmap = %p va = %p", __func__, pmap == kernel_pmap, pmap, va); /* Get the page table pointer. */ ptbl = pmap->pm_pdir[pdir_idx]; if (ptbl == NULL) { /* Allocate page table pages. */ ptbl = ptbl_alloc(mmu, pmap, pdir_idx, nosleep); if (ptbl == NULL) { KASSERT(nosleep, ("nosleep and NULL ptbl")); return (ENOMEM); } } else { /* * Check if there is valid mapping for requested * va, if there is, remove it. */ pte = &pmap->pm_pdir[pdir_idx][ptbl_idx]; if (PTE_ISVALID(pte)) { pte_remove(mmu, pmap, va, PTBL_HOLD); } else { /* * pte is not used, increment hold count * for ptbl pages. */ if (pmap != kernel_pmap) ptbl_hold(mmu, pmap, pdir_idx); } } /* * Insert pv_entry into pv_list for mapped page if part of managed * memory. */ if ((m->oflags & VPO_UNMANAGED) == 0) { flags |= PTE_MANAGED; /* Create and insert pv entry. */ pv_insert(pmap, va, m); } pmap->pm_stats.resident_count++; mtx_lock_spin(&tlbivax_mutex); tlb_miss_lock(); tlb0_flush_entry(va); if (pmap->pm_pdir[pdir_idx] == NULL) { /* * If we just allocated a new page table, hook it in * the pdir. */ pmap->pm_pdir[pdir_idx] = ptbl; } pte = &(pmap->pm_pdir[pdir_idx][ptbl_idx]); *pte = PTE_RPN_FROM_PA(VM_PAGE_TO_PHYS(m)); *pte |= (PTE_VALID | flags | PTE_PS_4KB); /* 4KB pages only */ tlb_miss_unlock(); mtx_unlock_spin(&tlbivax_mutex); return (0); } /* Return the pa for the given pmap/va. */ static vm_paddr_t pte_vatopa(mmu_t mmu, pmap_t pmap, vm_offset_t va) { vm_paddr_t pa = 0; pte_t *pte; pte = pte_find(mmu, pmap, va); if ((pte != NULL) && PTE_ISVALID(pte)) pa = (PTE_PA(pte) | (va & PTE_PA_MASK)); return (pa); } /* Get a pointer to a PTE in a page table. */ static pte_t * pte_find(mmu_t mmu, pmap_t pmap, vm_offset_t va) { unsigned int pdir_idx = PDIR_IDX(va); unsigned int ptbl_idx = PTBL_IDX(va); KASSERT((pmap != NULL), ("pte_find: invalid pmap")); if (pmap->pm_pdir[pdir_idx]) return (&(pmap->pm_pdir[pdir_idx][ptbl_idx])); return (NULL); } /* Set up kernel page tables. */ static void kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr, vm_offset_t pdir) { int i; vm_offset_t va; pte_t *pte; /* Initialize kernel pdir */ for (i = 0; i < kernel_ptbls; i++) kernel_pmap->pm_pdir[kptbl_min + i] = (pte_t *)(pdir + (i * PAGE_SIZE * PTBL_PAGES)); /* * Fill in PTEs covering kernel code and data. They are not required * for address translation, as this area is covered by static TLB1 * entries, but for pte_vatopa() to work correctly with kernel area * addresses. */ for (va = addr; va < data_end; va += PAGE_SIZE) { pte = &(kernel_pmap->pm_pdir[PDIR_IDX(va)][PTBL_IDX(va)]); *pte = PTE_RPN_FROM_PA(kernload + (va - kernstart)); *pte |= PTE_M | PTE_SR | PTE_SW | PTE_SX | PTE_WIRED | PTE_VALID | PTE_PS_4KB; } } /**************************************************************************/ /* PMAP related */ /**************************************************************************/ /* * This is called during booke_init, before the system is really initialized. */ static void mmu_booke_bootstrap(mmu_t mmu, vm_offset_t start, vm_offset_t kernelend) { vm_paddr_t phys_kernelend; struct mem_region *mp, *mp1; int cnt, i, j; vm_paddr_t s, e, sz; vm_paddr_t physsz, hwphyssz; u_int phys_avail_count; vm_size_t kstack0_sz; vm_offset_t kernel_pdir, kstack0; vm_paddr_t kstack0_phys; void *dpcpu; debugf("mmu_booke_bootstrap: entered\n"); /* Set interesting system properties */ hw_direct_map = 0; elf32_nxstack = 1; /* Initialize invalidation mutex */ mtx_init(&tlbivax_mutex, "tlbivax", NULL, MTX_SPIN); /* Read TLB0 size and associativity. */ tlb0_get_tlbconf(); /* * Align kernel start and end address (kernel image). * Note that kernel end does not necessarily relate to kernsize. * kernsize is the size of the kernel that is actually mapped. */ kernstart = trunc_page(start); data_start = round_page(kernelend); data_end = data_start; /* * Addresses of preloaded modules (like file systems) use * physical addresses. Make sure we relocate those into * virtual addresses. */ preload_addr_relocate = kernstart - kernload; /* Allocate the dynamic per-cpu area. */ dpcpu = (void *)data_end; data_end += DPCPU_SIZE; /* Allocate space for the message buffer. */ msgbufp = (struct msgbuf *)data_end; data_end += msgbufsize; debugf(" msgbufp at 0x%08x end = 0x%08x\n", (uint32_t)msgbufp, data_end); data_end = round_page(data_end); /* Allocate space for ptbl_bufs. */ ptbl_bufs = (struct ptbl_buf *)data_end; data_end += sizeof(struct ptbl_buf) * PTBL_BUFS; debugf(" ptbl_bufs at 0x%08x end = 0x%08x\n", (uint32_t)ptbl_bufs, data_end); data_end = round_page(data_end); /* Allocate PTE tables for kernel KVA. */ kernel_pdir = data_end; kernel_ptbls = howmany(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS, PDIR_SIZE); data_end += kernel_ptbls * PTBL_PAGES * PAGE_SIZE; debugf(" kernel ptbls: %d\n", kernel_ptbls); debugf(" kernel pdir at 0x%08x end = 0x%08x\n", kernel_pdir, data_end); debugf(" data_end: 0x%08x\n", data_end); if (data_end - kernstart > kernsize) { kernsize += tlb1_mapin_region(kernstart + kernsize, kernload + kernsize, (data_end - kernstart) - kernsize); } data_end = kernstart + kernsize; debugf(" updated data_end: 0x%08x\n", data_end); /* * Clear the structures - note we can only do it safely after the * possible additional TLB1 translations are in place (above) so that * all range up to the currently calculated 'data_end' is covered. */ dpcpu_init(dpcpu, 0); memset((void *)ptbl_bufs, 0, sizeof(struct ptbl_buf) * PTBL_SIZE); memset((void *)kernel_pdir, 0, kernel_ptbls * PTBL_PAGES * PAGE_SIZE); /*******************************************************/ /* Set the start and end of kva. */ /*******************************************************/ virtual_avail = round_page(data_end); virtual_end = VM_MAX_KERNEL_ADDRESS; /* Allocate KVA space for page zero/copy operations. */ zero_page_va = virtual_avail; virtual_avail += PAGE_SIZE; copy_page_src_va = virtual_avail; virtual_avail += PAGE_SIZE; copy_page_dst_va = virtual_avail; virtual_avail += PAGE_SIZE; debugf("zero_page_va = 0x%08x\n", zero_page_va); debugf("copy_page_src_va = 0x%08x\n", copy_page_src_va); debugf("copy_page_dst_va = 0x%08x\n", copy_page_dst_va); /* Initialize page zero/copy mutexes. */ mtx_init(&zero_page_mutex, "mmu_booke_zero_page", NULL, MTX_DEF); mtx_init(©_page_mutex, "mmu_booke_copy_page", NULL, MTX_DEF); /* Allocate KVA space for ptbl bufs. */ ptbl_buf_pool_vabase = virtual_avail; virtual_avail += PTBL_BUFS * PTBL_PAGES * PAGE_SIZE; debugf("ptbl_buf_pool_vabase = 0x%08x end = 0x%08x\n", ptbl_buf_pool_vabase, virtual_avail); /* Calculate corresponding physical addresses for the kernel region. */ phys_kernelend = kernload + kernsize; debugf("kernel image and allocated data:\n"); debugf(" kernload = 0x%09llx\n", (uint64_t)kernload); debugf(" kernstart = 0x%08x\n", kernstart); debugf(" kernsize = 0x%08x\n", kernsize); if (sizeof(phys_avail) / sizeof(phys_avail[0]) < availmem_regions_sz) panic("mmu_booke_bootstrap: phys_avail too small"); /* * Remove kernel physical address range from avail regions list. Page * align all regions. Non-page aligned memory isn't very interesting * to us. Also, sort the entries for ascending addresses. */ /* Retrieve phys/avail mem regions */ mem_regions(&physmem_regions, &physmem_regions_sz, &availmem_regions, &availmem_regions_sz); sz = 0; cnt = availmem_regions_sz; debugf("processing avail regions:\n"); for (mp = availmem_regions; mp->mr_size; mp++) { s = mp->mr_start; e = mp->mr_start + mp->mr_size; debugf(" %09jx-%09jx -> ", (uintmax_t)s, (uintmax_t)e); /* Check whether this region holds all of the kernel. */ if (s < kernload && e > phys_kernelend) { availmem_regions[cnt].mr_start = phys_kernelend; availmem_regions[cnt++].mr_size = e - phys_kernelend; e = kernload; } /* Look whether this regions starts within the kernel. */ if (s >= kernload && s < phys_kernelend) { if (e <= phys_kernelend) goto empty; s = phys_kernelend; } /* Now look whether this region ends within the kernel. */ if (e > kernload && e <= phys_kernelend) { if (s >= kernload) goto empty; e = kernload; } /* Now page align the start and size of the region. */ s = round_page(s); e = trunc_page(e); if (e < s) e = s; sz = e - s; debugf("%09jx-%09jx = %jx\n", (uintmax_t)s, (uintmax_t)e, (uintmax_t)sz); /* Check whether some memory is left here. */ if (sz == 0) { empty: memmove(mp, mp + 1, (cnt - (mp - availmem_regions)) * sizeof(*mp)); cnt--; mp--; continue; } /* Do an insertion sort. */ for (mp1 = availmem_regions; mp1 < mp; mp1++) if (s < mp1->mr_start) break; if (mp1 < mp) { memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1); mp1->mr_start = s; mp1->mr_size = sz; } else { mp->mr_start = s; mp->mr_size = sz; } } availmem_regions_sz = cnt; /*******************************************************/ /* Steal physical memory for kernel stack from the end */ /* of the first avail region */ /*******************************************************/ kstack0_sz = kstack_pages * PAGE_SIZE; kstack0_phys = availmem_regions[0].mr_start + availmem_regions[0].mr_size; kstack0_phys -= kstack0_sz; availmem_regions[0].mr_size -= kstack0_sz; /*******************************************************/ /* Fill in phys_avail table, based on availmem_regions */ /*******************************************************/ phys_avail_count = 0; physsz = 0; hwphyssz = 0; TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz); debugf("fill in phys_avail:\n"); for (i = 0, j = 0; i < availmem_regions_sz; i++, j += 2) { debugf(" region: 0x%jx - 0x%jx (0x%jx)\n", (uintmax_t)availmem_regions[i].mr_start, (uintmax_t)availmem_regions[i].mr_start + availmem_regions[i].mr_size, (uintmax_t)availmem_regions[i].mr_size); if (hwphyssz != 0 && (physsz + availmem_regions[i].mr_size) >= hwphyssz) { debugf(" hw.physmem adjust\n"); if (physsz < hwphyssz) { phys_avail[j] = availmem_regions[i].mr_start; phys_avail[j + 1] = availmem_regions[i].mr_start + hwphyssz - physsz; physsz = hwphyssz; phys_avail_count++; } break; } phys_avail[j] = availmem_regions[i].mr_start; phys_avail[j + 1] = availmem_regions[i].mr_start + availmem_regions[i].mr_size; phys_avail_count++; physsz += availmem_regions[i].mr_size; } physmem = btoc(physsz); /* Calculate the last available physical address. */ for (i = 0; phys_avail[i + 2] != 0; i += 2) ; Maxmem = powerpc_btop(phys_avail[i + 1]); debugf("Maxmem = 0x%08lx\n", Maxmem); debugf("phys_avail_count = %d\n", phys_avail_count); debugf("physsz = 0x%09jx physmem = %jd (0x%09jx)\n", (uintmax_t)physsz, (uintmax_t)physmem, (uintmax_t)physmem); /*******************************************************/ /* Initialize (statically allocated) kernel pmap. */ /*******************************************************/ PMAP_LOCK_INIT(kernel_pmap); kptbl_min = VM_MIN_KERNEL_ADDRESS / PDIR_SIZE; debugf("kernel_pmap = 0x%08x\n", (uint32_t)kernel_pmap); debugf("kptbl_min = %d, kernel_ptbls = %d\n", kptbl_min, kernel_ptbls); debugf("kernel pdir range: 0x%08x - 0x%08x\n", kptbl_min * PDIR_SIZE, (kptbl_min + kernel_ptbls) * PDIR_SIZE - 1); kernel_pte_alloc(data_end, kernstart, kernel_pdir); for (i = 0; i < MAXCPU; i++) { kernel_pmap->pm_tid[i] = TID_KERNEL; /* Initialize each CPU's tidbusy entry 0 with kernel_pmap */ tidbusy[i][TID_KERNEL] = kernel_pmap; } /* Mark kernel_pmap active on all CPUs */ CPU_FILL(&kernel_pmap->pm_active); /* * Initialize the global pv list lock. */ rw_init(&pvh_global_lock, "pmap pv global"); /*******************************************************/ /* Final setup */ /*******************************************************/ /* Enter kstack0 into kernel map, provide guard page */ kstack0 = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; thread0.td_kstack = kstack0; thread0.td_kstack_pages = kstack_pages; debugf("kstack_sz = 0x%08x\n", kstack0_sz); debugf("kstack0_phys at 0x%09llx - 0x%09llx\n", kstack0_phys, kstack0_phys + kstack0_sz); debugf("kstack0 at 0x%08x - 0x%08x\n", kstack0, kstack0 + kstack0_sz); virtual_avail += KSTACK_GUARD_PAGES * PAGE_SIZE + kstack0_sz; for (i = 0; i < kstack_pages; i++) { mmu_booke_kenter(mmu, kstack0, kstack0_phys); kstack0 += PAGE_SIZE; kstack0_phys += PAGE_SIZE; } pmap_bootstrapped = 1; debugf("virtual_avail = %08x\n", virtual_avail); debugf("virtual_end = %08x\n", virtual_end); debugf("mmu_booke_bootstrap: exit\n"); } #ifdef SMP void tlb1_ap_prep(void) { tlb_entry_t *e, tmp; unsigned int i; /* Prepare TLB1 image for AP processors */ e = __boot_tlb1; for (i = 0; i < TLB1_ENTRIES; i++) { tlb1_read_entry(&tmp, i); if ((tmp.mas1 & MAS1_VALID) && (tmp.mas2 & _TLB_ENTRY_SHARED)) memcpy(e++, &tmp, sizeof(tmp)); } } void pmap_bootstrap_ap(volatile uint32_t *trcp __unused) { int i; /* * Finish TLB1 configuration: the BSP already set up its TLB1 and we * have the snapshot of its contents in the s/w __boot_tlb1[] table * created by tlb1_ap_prep(), so use these values directly to * (re)program AP's TLB1 hardware. * * Start at index 1 because index 0 has the kernel map. */ for (i = 1; i < TLB1_ENTRIES; i++) { if (__boot_tlb1[i].mas1 & MAS1_VALID) tlb1_write_entry(&__boot_tlb1[i], i); } set_mas4_defaults(); } #endif static void booke_pmap_init_qpages(void) { struct pcpu *pc; int i; CPU_FOREACH(i) { pc = pcpu_find(i); pc->pc_qmap_addr = kva_alloc(PAGE_SIZE); if (pc->pc_qmap_addr == 0) panic("pmap_init_qpages: unable to allocate KVA"); } } SYSINIT(qpages_init, SI_SUB_CPU, SI_ORDER_ANY, booke_pmap_init_qpages, NULL); /* * Get the physical page address for the given pmap/virtual address. */ static vm_paddr_t mmu_booke_extract(mmu_t mmu, pmap_t pmap, vm_offset_t va) { vm_paddr_t pa; PMAP_LOCK(pmap); pa = pte_vatopa(mmu, pmap, va); PMAP_UNLOCK(pmap); return (pa); } /* * Extract the physical page address associated with the given * kernel virtual address. */ static vm_paddr_t mmu_booke_kextract(mmu_t mmu, vm_offset_t va) { tlb_entry_t e; int i; /* Check TLB1 mappings */ for (i = 0; i < TLB1_ENTRIES; i++) { tlb1_read_entry(&e, i); if (!(e.mas1 & MAS1_VALID)) continue; if (va >= e.virt && va < e.virt + e.size) return (e.phys + (va - e.virt)); } return (pte_vatopa(mmu, kernel_pmap, va)); } /* * Initialize the pmap module. * Called by vm_init, to initialize any structures that the pmap * system needs to map virtual memory. */ static void mmu_booke_init(mmu_t mmu) { int shpgperproc = PMAP_SHPGPERPROC; /* * Initialize the address space (zone) for the pv entries. Set a * high water mark so that the system can recover from excessive * numbers of pv entries. */ pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); pv_entry_max = shpgperproc * maxproc + vm_cnt.v_page_count; TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max); pv_entry_high_water = 9 * (pv_entry_max / 10); uma_zone_reserve_kva(pvzone, pv_entry_max); /* Pre-fill pvzone with initial number of pv entries. */ uma_prealloc(pvzone, PV_ENTRY_ZONE_MIN); /* Initialize ptbl allocation. */ ptbl_init(); } /* * Map a list of wired pages into kernel virtual address space. This is * intended for temporary mappings which do not need page modification or * references recorded. Existing mappings in the region are overwritten. */ static void mmu_booke_qenter(mmu_t mmu, vm_offset_t sva, vm_page_t *m, int count) { vm_offset_t va; va = sva; while (count-- > 0) { mmu_booke_kenter(mmu, va, VM_PAGE_TO_PHYS(*m)); va += PAGE_SIZE; m++; } } /* * Remove page mappings from kernel virtual address space. Intended for * temporary mappings entered by mmu_booke_qenter. */ static void mmu_booke_qremove(mmu_t mmu, vm_offset_t sva, int count) { vm_offset_t va; va = sva; while (count-- > 0) { mmu_booke_kremove(mmu, va); va += PAGE_SIZE; } } /* * Map a wired page into kernel virtual address space. */ static void mmu_booke_kenter(mmu_t mmu, vm_offset_t va, vm_paddr_t pa) { mmu_booke_kenter_attr(mmu, va, pa, VM_MEMATTR_DEFAULT); } static void mmu_booke_kenter_attr(mmu_t mmu, vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma) { uint32_t flags; pte_t *pte; KASSERT(((va >= VM_MIN_KERNEL_ADDRESS) && (va <= VM_MAX_KERNEL_ADDRESS)), ("mmu_booke_kenter: invalid va")); flags = PTE_SR | PTE_SW | PTE_SX | PTE_WIRED | PTE_VALID; flags |= tlb_calc_wimg(pa, ma) << PTE_MAS2_SHIFT; flags |= PTE_PS_4KB; pte = pte_find(mmu, kernel_pmap, va); mtx_lock_spin(&tlbivax_mutex); tlb_miss_lock(); if (PTE_ISVALID(pte)) { CTR1(KTR_PMAP, "%s: replacing entry!", __func__); /* Flush entry from TLB0 */ tlb0_flush_entry(va); } *pte = PTE_RPN_FROM_PA(pa) | flags; //debugf("mmu_booke_kenter: pdir_idx = %d ptbl_idx = %d va=0x%08x " // "pa=0x%08x rpn=0x%08x flags=0x%08x\n", // pdir_idx, ptbl_idx, va, pa, pte->rpn, pte->flags); /* Flush the real memory from the instruction cache. */ if ((flags & (PTE_I | PTE_G)) == 0) __syncicache((void *)va, PAGE_SIZE); tlb_miss_unlock(); mtx_unlock_spin(&tlbivax_mutex); } /* * Remove a page from kernel page table. */ static void mmu_booke_kremove(mmu_t mmu, vm_offset_t va) { pte_t *pte; CTR2(KTR_PMAP,"%s: s (va = 0x%08x)\n", __func__, va); KASSERT(((va >= VM_MIN_KERNEL_ADDRESS) && (va <= VM_MAX_KERNEL_ADDRESS)), ("mmu_booke_kremove: invalid va")); pte = pte_find(mmu, kernel_pmap, va); if (!PTE_ISVALID(pte)) { CTR1(KTR_PMAP, "%s: invalid pte", __func__); return; } mtx_lock_spin(&tlbivax_mutex); tlb_miss_lock(); /* Invalidate entry in TLB0, update PTE. */ tlb0_flush_entry(va); *pte = 0; tlb_miss_unlock(); mtx_unlock_spin(&tlbivax_mutex); } /* * Initialize pmap associated with process 0. */ static void mmu_booke_pinit0(mmu_t mmu, pmap_t pmap) { PMAP_LOCK_INIT(pmap); mmu_booke_pinit(mmu, pmap); PCPU_SET(curpmap, pmap); } /* * Initialize a preallocated and zeroed pmap structure, * such as one in a vmspace structure. */ static void mmu_booke_pinit(mmu_t mmu, pmap_t pmap) { int i; CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap, curthread->td_proc->p_pid, curthread->td_proc->p_comm); KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap")); for (i = 0; i < MAXCPU; i++) pmap->pm_tid[i] = TID_NONE; CPU_ZERO(&kernel_pmap->pm_active); bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); bzero(&pmap->pm_pdir, sizeof(pte_t *) * PDIR_NENTRIES); TAILQ_INIT(&pmap->pm_ptbl_list); } /* * Release any resources held by the given physical map. * Called when a pmap initialized by mmu_booke_pinit is being released. * Should only be called if the map contains no valid mappings. */ static void mmu_booke_release(mmu_t mmu, pmap_t pmap) { KASSERT(pmap->pm_stats.resident_count == 0, ("pmap_release: pmap resident count %ld != 0", pmap->pm_stats.resident_count)); } /* * Insert the given physical page at the specified virtual address in the * target physical map with the protection requested. If specified the page * will be wired down. */ static int mmu_booke_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind) { int error; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); error = mmu_booke_enter_locked(mmu, pmap, va, m, prot, flags, psind); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); return (error); } static int mmu_booke_enter_locked(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int pmap_flags, int8_t psind __unused) { pte_t *pte; vm_paddr_t pa; uint32_t flags; int error, su, sync; pa = VM_PAGE_TO_PHYS(m); su = (pmap == kernel_pmap); sync = 0; //debugf("mmu_booke_enter_locked: s (pmap=0x%08x su=%d tid=%d m=0x%08x va=0x%08x " // "pa=0x%08x prot=0x%08x flags=%#x)\n", // (u_int32_t)pmap, su, pmap->pm_tid, // (u_int32_t)m, va, pa, prot, flags); if (su) { KASSERT(((va >= virtual_avail) && (va <= VM_MAX_KERNEL_ADDRESS)), ("mmu_booke_enter_locked: kernel pmap, non kernel va")); } else { KASSERT((va <= VM_MAXUSER_ADDRESS), ("mmu_booke_enter_locked: user pmap, non user va")); } if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) VM_OBJECT_ASSERT_LOCKED(m->object); PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * If there is an existing mapping, and the physical address has not * changed, must be protection or wiring change. */ if (((pte = pte_find(mmu, pmap, va)) != NULL) && (PTE_ISVALID(pte)) && (PTE_PA(pte) == pa)) { /* * Before actually updating pte->flags we calculate and * prepare its new value in a helper var. */ flags = *pte; flags &= ~(PTE_UW | PTE_UX | PTE_SW | PTE_SX | PTE_MODIFIED); /* Wiring change, just update stats. */ if ((pmap_flags & PMAP_ENTER_WIRED) != 0) { if (!PTE_ISWIRED(pte)) { flags |= PTE_WIRED; pmap->pm_stats.wired_count++; } } else { if (PTE_ISWIRED(pte)) { flags &= ~PTE_WIRED; pmap->pm_stats.wired_count--; } } if (prot & VM_PROT_WRITE) { /* Add write permissions. */ flags |= PTE_SW; if (!su) flags |= PTE_UW; if ((flags & PTE_MANAGED) != 0) vm_page_aflag_set(m, PGA_WRITEABLE); } else { /* Handle modified pages, sense modify status. */ /* * The PTE_MODIFIED flag could be set by underlying * TLB misses since we last read it (above), possibly * other CPUs could update it so we check in the PTE * directly rather than rely on that saved local flags * copy. */ if (PTE_ISMODIFIED(pte)) vm_page_dirty(m); } if (prot & VM_PROT_EXECUTE) { flags |= PTE_SX; if (!su) flags |= PTE_UX; /* * Check existing flags for execute permissions: if we * are turning execute permissions on, icache should * be flushed. */ if ((*pte & (PTE_UX | PTE_SX)) == 0) sync++; } flags &= ~PTE_REFERENCED; /* * The new flags value is all calculated -- only now actually * update the PTE. */ mtx_lock_spin(&tlbivax_mutex); tlb_miss_lock(); tlb0_flush_entry(va); *pte &= ~PTE_FLAGS_MASK; *pte |= flags; tlb_miss_unlock(); mtx_unlock_spin(&tlbivax_mutex); } else { /* * If there is an existing mapping, but it's for a different * physical address, pte_enter() will delete the old mapping. */ //if ((pte != NULL) && PTE_ISVALID(pte)) // debugf("mmu_booke_enter_locked: replace\n"); //else // debugf("mmu_booke_enter_locked: new\n"); /* Now set up the flags and install the new mapping. */ flags = (PTE_SR | PTE_VALID); flags |= PTE_M; if (!su) flags |= PTE_UR; if (prot & VM_PROT_WRITE) { flags |= PTE_SW; if (!su) flags |= PTE_UW; if ((m->oflags & VPO_UNMANAGED) == 0) vm_page_aflag_set(m, PGA_WRITEABLE); } if (prot & VM_PROT_EXECUTE) { flags |= PTE_SX; if (!su) flags |= PTE_UX; } /* If its wired update stats. */ if ((pmap_flags & PMAP_ENTER_WIRED) != 0) flags |= PTE_WIRED; error = pte_enter(mmu, pmap, m, va, flags, (pmap_flags & PMAP_ENTER_NOSLEEP) != 0); if (error != 0) return (KERN_RESOURCE_SHORTAGE); if ((flags & PMAP_ENTER_WIRED) != 0) pmap->pm_stats.wired_count++; /* Flush the real memory from the instruction cache. */ if (prot & VM_PROT_EXECUTE) sync++; } if (sync && (su || pmap == PCPU_GET(curpmap))) { __syncicache((void *)va, PAGE_SIZE); sync = 0; } return (KERN_SUCCESS); } /* * Maps a sequence of resident pages belonging to the same object. * The sequence begins with the given page m_start. This page is * mapped at the given virtual address start. Each subsequent page is * mapped at a virtual address that is offset from start by the same * amount as the page is offset from m_start within the object. The * last page in the sequence is the page with the largest offset from * m_start that can be mapped at a virtual address less than the given * virtual address end. Not every virtual page between start and end * is mapped; only those for which a resident page exists with the * corresponding offset from m_start are mapped. */ static void mmu_booke_enter_object(mmu_t mmu, pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot) { vm_page_t m; vm_pindex_t diff, psize; VM_OBJECT_ASSERT_LOCKED(m_start->object); psize = atop(end - start); m = m_start; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { mmu_booke_enter_locked(mmu, pmap, start + ptoa(diff), m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), PMAP_ENTER_NOSLEEP, 0); m = TAILQ_NEXT(m, listq); } rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } static void mmu_booke_enter_quick(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); mmu_booke_enter_locked(mmu, pmap, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), PMAP_ENTER_NOSLEEP, 0); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * Remove the given range of addresses from the specified map. * * It is assumed that the start and end are properly rounded to the page size. */ static void mmu_booke_remove(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_offset_t endva) { pte_t *pte; uint8_t hold_flag; int su = (pmap == kernel_pmap); //debugf("mmu_booke_remove: s (su = %d pmap=0x%08x tid=%d va=0x%08x endva=0x%08x)\n", // su, (u_int32_t)pmap, pmap->pm_tid, va, endva); if (su) { KASSERT(((va >= virtual_avail) && (va <= VM_MAX_KERNEL_ADDRESS)), ("mmu_booke_remove: kernel pmap, non kernel va")); } else { KASSERT((va <= VM_MAXUSER_ADDRESS), ("mmu_booke_remove: user pmap, non user va")); } if (PMAP_REMOVE_DONE(pmap)) { //debugf("mmu_booke_remove: e (empty)\n"); return; } hold_flag = PTBL_HOLD_FLAG(pmap); //debugf("mmu_booke_remove: hold_flag = %d\n", hold_flag); rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); for (; va < endva; va += PAGE_SIZE) { pte = pte_find(mmu, pmap, va); if ((pte != NULL) && PTE_ISVALID(pte)) pte_remove(mmu, pmap, va, hold_flag); } PMAP_UNLOCK(pmap); rw_wunlock(&pvh_global_lock); //debugf("mmu_booke_remove: e\n"); } /* * Remove physical page from all pmaps in which it resides. */ static void mmu_booke_remove_all(mmu_t mmu, vm_page_t m) { pv_entry_t pv, pvn; uint8_t hold_flag; rw_wlock(&pvh_global_lock); for (pv = TAILQ_FIRST(&m->md.pv_list); pv != NULL; pv = pvn) { pvn = TAILQ_NEXT(pv, pv_link); PMAP_LOCK(pv->pv_pmap); hold_flag = PTBL_HOLD_FLAG(pv->pv_pmap); pte_remove(mmu, pv->pv_pmap, pv->pv_va, hold_flag); PMAP_UNLOCK(pv->pv_pmap); } vm_page_aflag_clear(m, PGA_WRITEABLE); rw_wunlock(&pvh_global_lock); } /* * Map a range of physical addresses into kernel virtual address space. */ static vm_offset_t mmu_booke_map(mmu_t mmu, vm_offset_t *virt, vm_paddr_t pa_start, vm_paddr_t pa_end, int prot) { vm_offset_t sva = *virt; vm_offset_t va = sva; //debugf("mmu_booke_map: s (sva = 0x%08x pa_start = 0x%08x pa_end = 0x%08x)\n", // sva, pa_start, pa_end); while (pa_start < pa_end) { mmu_booke_kenter(mmu, va, pa_start); va += PAGE_SIZE; pa_start += PAGE_SIZE; } *virt = va; //debugf("mmu_booke_map: e (va = 0x%08x)\n", va); return (sva); } /* * The pmap must be activated before it's address space can be accessed in any * way. */ static void mmu_booke_activate(mmu_t mmu, struct thread *td) { pmap_t pmap; u_int cpuid; pmap = &td->td_proc->p_vmspace->vm_pmap; CTR5(KTR_PMAP, "%s: s (td = %p, proc = '%s', id = %d, pmap = 0x%08x)", __func__, td, td->td_proc->p_comm, td->td_proc->p_pid, pmap); KASSERT((pmap != kernel_pmap), ("mmu_booke_activate: kernel_pmap!")); sched_pin(); cpuid = PCPU_GET(cpuid); CPU_SET_ATOMIC(cpuid, &pmap->pm_active); PCPU_SET(curpmap, pmap); if (pmap->pm_tid[cpuid] == TID_NONE) tid_alloc(pmap); /* Load PID0 register with pmap tid value. */ mtspr(SPR_PID0, pmap->pm_tid[cpuid]); __asm __volatile("isync"); mtspr(SPR_DBCR0, td->td_pcb->pcb_cpu.booke.dbcr0); sched_unpin(); CTR3(KTR_PMAP, "%s: e (tid = %d for '%s')", __func__, pmap->pm_tid[PCPU_GET(cpuid)], td->td_proc->p_comm); } /* * Deactivate the specified process's address space. */ static void mmu_booke_deactivate(mmu_t mmu, struct thread *td) { pmap_t pmap; pmap = &td->td_proc->p_vmspace->vm_pmap; CTR5(KTR_PMAP, "%s: td=%p, proc = '%s', id = %d, pmap = 0x%08x", __func__, td, td->td_proc->p_comm, td->td_proc->p_pid, pmap); td->td_pcb->pcb_cpu.booke.dbcr0 = mfspr(SPR_DBCR0); CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmap->pm_active); PCPU_SET(curpmap, NULL); } /* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len * in the destination map. * * This routine is only advisory and need not do anything. */ static void mmu_booke_copy(mmu_t mmu, pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { } /* * Set the physical protection on the specified range of this map as requested. */ static void mmu_booke_protect(mmu_t mmu, pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { vm_offset_t va; vm_page_t m; pte_t *pte; if ((prot & VM_PROT_READ) == VM_PROT_NONE) { mmu_booke_remove(mmu, pmap, sva, eva); return; } if (prot & VM_PROT_WRITE) return; PMAP_LOCK(pmap); for (va = sva; va < eva; va += PAGE_SIZE) { if ((pte = pte_find(mmu, pmap, va)) != NULL) { if (PTE_ISVALID(pte)) { m = PHYS_TO_VM_PAGE(PTE_PA(pte)); mtx_lock_spin(&tlbivax_mutex); tlb_miss_lock(); /* Handle modified pages. */ if (PTE_ISMODIFIED(pte) && PTE_ISMANAGED(pte)) vm_page_dirty(m); tlb0_flush_entry(va); *pte &= ~(PTE_UW | PTE_SW | PTE_MODIFIED); tlb_miss_unlock(); mtx_unlock_spin(&tlbivax_mutex); } } } PMAP_UNLOCK(pmap); } /* * Clear the write and modified bits in each of the given page's mappings. */ static void mmu_booke_remove_write(mmu_t mmu, vm_page_t m) { pv_entry_t pv; pte_t *pte; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("mmu_booke_remove_write: page %p is not managed", m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * set by another thread while the object is locked. Thus, * if PGA_WRITEABLE is clear, no page table entries need updating. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return; rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { PMAP_LOCK(pv->pv_pmap); if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL) { if (PTE_ISVALID(pte)) { m = PHYS_TO_VM_PAGE(PTE_PA(pte)); mtx_lock_spin(&tlbivax_mutex); tlb_miss_lock(); /* Handle modified pages. */ if (PTE_ISMODIFIED(pte)) vm_page_dirty(m); /* Flush mapping from TLB0. */ *pte &= ~(PTE_UW | PTE_SW | PTE_MODIFIED); tlb_miss_unlock(); mtx_unlock_spin(&tlbivax_mutex); } } PMAP_UNLOCK(pv->pv_pmap); } vm_page_aflag_clear(m, PGA_WRITEABLE); rw_wunlock(&pvh_global_lock); } static void mmu_booke_sync_icache(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_size_t sz) { pte_t *pte; pmap_t pmap; vm_page_t m; vm_offset_t addr; vm_paddr_t pa = 0; int active, valid; va = trunc_page(va); sz = round_page(sz); rw_wlock(&pvh_global_lock); pmap = PCPU_GET(curpmap); active = (pm == kernel_pmap || pm == pmap) ? 1 : 0; while (sz > 0) { PMAP_LOCK(pm); pte = pte_find(mmu, pm, va); valid = (pte != NULL && PTE_ISVALID(pte)) ? 1 : 0; if (valid) pa = PTE_PA(pte); PMAP_UNLOCK(pm); if (valid) { if (!active) { /* Create a mapping in the active pmap. */ addr = 0; m = PHYS_TO_VM_PAGE(pa); PMAP_LOCK(pmap); pte_enter(mmu, pmap, m, addr, PTE_SR | PTE_VALID | PTE_UR, FALSE); __syncicache((void *)addr, PAGE_SIZE); pte_remove(mmu, pmap, addr, PTBL_UNHOLD); PMAP_UNLOCK(pmap); } else __syncicache((void *)va, PAGE_SIZE); } va += PAGE_SIZE; sz -= PAGE_SIZE; } rw_wunlock(&pvh_global_lock); } /* * Atomically extract and hold the physical page with the given * pmap and virtual address pair if that mapping permits the given * protection. */ static vm_page_t mmu_booke_extract_and_hold(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_prot_t prot) { pte_t *pte; vm_page_t m; uint32_t pte_wbit; vm_paddr_t pa; m = NULL; pa = 0; PMAP_LOCK(pmap); retry: pte = pte_find(mmu, pmap, va); if ((pte != NULL) && PTE_ISVALID(pte)) { if (pmap == kernel_pmap) pte_wbit = PTE_SW; else pte_wbit = PTE_UW; if ((*pte & pte_wbit) || ((prot & VM_PROT_WRITE) == 0)) { if (vm_page_pa_tryrelock(pmap, PTE_PA(pte), &pa)) goto retry; m = PHYS_TO_VM_PAGE(PTE_PA(pte)); vm_page_hold(m); } } PA_UNLOCK_COND(pa); PMAP_UNLOCK(pmap); return (m); } /* * Initialize a vm_page's machine-dependent fields. */ static void mmu_booke_page_init(mmu_t mmu, vm_page_t m) { TAILQ_INIT(&m->md.pv_list); } /* * mmu_booke_zero_page_area zeros the specified hardware page by * mapping it into virtual memory and using bzero to clear * its contents. * * off and size must reside within a single page. */ static void mmu_booke_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size) { vm_offset_t va; /* XXX KASSERT off and size are within a single page? */ mtx_lock(&zero_page_mutex); va = zero_page_va; mmu_booke_kenter(mmu, va, VM_PAGE_TO_PHYS(m)); bzero((caddr_t)va + off, size); mmu_booke_kremove(mmu, va); mtx_unlock(&zero_page_mutex); } /* * mmu_booke_zero_page zeros the specified hardware page. */ static void mmu_booke_zero_page(mmu_t mmu, vm_page_t m) { vm_offset_t off, va; mtx_lock(&zero_page_mutex); va = zero_page_va; mmu_booke_kenter(mmu, va, VM_PAGE_TO_PHYS(m)); for (off = 0; off < PAGE_SIZE; off += cacheline_size) __asm __volatile("dcbz 0,%0" :: "r"(va + off)); mmu_booke_kremove(mmu, va); mtx_unlock(&zero_page_mutex); } /* * mmu_booke_copy_page copies the specified (machine independent) page by * mapping the page into virtual memory and using memcopy to copy the page, * one machine dependent page at a time. */ static void mmu_booke_copy_page(mmu_t mmu, vm_page_t sm, vm_page_t dm) { vm_offset_t sva, dva; sva = copy_page_src_va; dva = copy_page_dst_va; mtx_lock(©_page_mutex); mmu_booke_kenter(mmu, sva, VM_PAGE_TO_PHYS(sm)); mmu_booke_kenter(mmu, dva, VM_PAGE_TO_PHYS(dm)); memcpy((caddr_t)dva, (caddr_t)sva, PAGE_SIZE); mmu_booke_kremove(mmu, dva); mmu_booke_kremove(mmu, sva); mtx_unlock(©_page_mutex); } static inline void mmu_booke_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, vm_page_t *mb, vm_offset_t b_offset, int xfersize) { void *a_cp, *b_cp; vm_offset_t a_pg_offset, b_pg_offset; int cnt; mtx_lock(©_page_mutex); while (xfersize > 0) { a_pg_offset = a_offset & PAGE_MASK; cnt = min(xfersize, PAGE_SIZE - a_pg_offset); mmu_booke_kenter(mmu, copy_page_src_va, VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])); a_cp = (char *)copy_page_src_va + a_pg_offset; b_pg_offset = b_offset & PAGE_MASK; cnt = min(cnt, PAGE_SIZE - b_pg_offset); mmu_booke_kenter(mmu, copy_page_dst_va, VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])); b_cp = (char *)copy_page_dst_va + b_pg_offset; bcopy(a_cp, b_cp, cnt); mmu_booke_kremove(mmu, copy_page_dst_va); mmu_booke_kremove(mmu, copy_page_src_va); a_offset += cnt; b_offset += cnt; xfersize -= cnt; } mtx_unlock(©_page_mutex); } static vm_offset_t mmu_booke_quick_enter_page(mmu_t mmu, vm_page_t m) { vm_paddr_t paddr; vm_offset_t qaddr; uint32_t flags; pte_t *pte; paddr = VM_PAGE_TO_PHYS(m); flags = PTE_SR | PTE_SW | PTE_SX | PTE_WIRED | PTE_VALID; flags |= tlb_calc_wimg(paddr, pmap_page_get_memattr(m)) << PTE_MAS2_SHIFT; flags |= PTE_PS_4KB; critical_enter(); qaddr = PCPU_GET(qmap_addr); pte = pte_find(mmu, kernel_pmap, qaddr); KASSERT(*pte == 0, ("mmu_booke_quick_enter_page: PTE busy")); /* * XXX: tlbivax is broadcast to other cores, but qaddr should * not be present in other TLBs. Is there a better instruction * sequence to use? Or just forget it & use mmu_booke_kenter()... */ __asm __volatile("tlbivax 0, %0" :: "r"(qaddr & MAS2_EPN_MASK)); __asm __volatile("isync; msync"); *pte = PTE_RPN_FROM_PA(paddr) | flags; /* Flush the real memory from the instruction cache. */ if ((flags & (PTE_I | PTE_G)) == 0) __syncicache((void *)qaddr, PAGE_SIZE); return (qaddr); } static void mmu_booke_quick_remove_page(mmu_t mmu, vm_offset_t addr) { pte_t *pte; pte = pte_find(mmu, kernel_pmap, addr); KASSERT(PCPU_GET(qmap_addr) == addr, ("mmu_booke_quick_remove_page: invalid address")); KASSERT(*pte != 0, ("mmu_booke_quick_remove_page: PTE not in use")); *pte = 0; critical_exit(); } /* * Return whether or not the specified physical page was modified * in any of physical maps. */ static boolean_t mmu_booke_is_modified(mmu_t mmu, vm_page_t m) { pte_t *pte; pv_entry_t pv; boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("mmu_booke_is_modified: page %p is not managed", m)); rv = FALSE; /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTEs can be modified. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return (rv); rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { PMAP_LOCK(pv->pv_pmap); if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL && PTE_ISVALID(pte)) { if (PTE_ISMODIFIED(pte)) rv = TRUE; } PMAP_UNLOCK(pv->pv_pmap); if (rv) break; } rw_wunlock(&pvh_global_lock); return (rv); } /* * Return whether or not the specified virtual address is eligible * for prefault. */ static boolean_t mmu_booke_is_prefaultable(mmu_t mmu, pmap_t pmap, vm_offset_t addr) { return (FALSE); } /* * Return whether or not the specified physical page was referenced * in any physical maps. */ static boolean_t mmu_booke_is_referenced(mmu_t mmu, vm_page_t m) { pte_t *pte; pv_entry_t pv; boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("mmu_booke_is_referenced: page %p is not managed", m)); rv = FALSE; rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { PMAP_LOCK(pv->pv_pmap); if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL && PTE_ISVALID(pte)) { if (PTE_ISREFERENCED(pte)) rv = TRUE; } PMAP_UNLOCK(pv->pv_pmap); if (rv) break; } rw_wunlock(&pvh_global_lock); return (rv); } /* * Clear the modify bits on the specified physical page. */ static void mmu_booke_clear_modify(mmu_t mmu, vm_page_t m) { pte_t *pte; pv_entry_t pv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("mmu_booke_clear_modify: page %p is not managed", m)); VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(!vm_page_xbusied(m), ("mmu_booke_clear_modify: page %p is exclusive busied", m)); /* * If the page is not PG_AWRITEABLE, then no PTEs can be modified. * If the object containing the page is locked and the page is not * exclusive busied, then PG_AWRITEABLE cannot be concurrently set. */ if ((m->aflags & PGA_WRITEABLE) == 0) return; rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { PMAP_LOCK(pv->pv_pmap); if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL && PTE_ISVALID(pte)) { mtx_lock_spin(&tlbivax_mutex); tlb_miss_lock(); if (*pte & (PTE_SW | PTE_UW | PTE_MODIFIED)) { tlb0_flush_entry(pv->pv_va); *pte &= ~(PTE_SW | PTE_UW | PTE_MODIFIED | PTE_REFERENCED); } tlb_miss_unlock(); mtx_unlock_spin(&tlbivax_mutex); } PMAP_UNLOCK(pv->pv_pmap); } rw_wunlock(&pvh_global_lock); } /* * Return a count of reference bits for a page, clearing those bits. * It is not necessary for every reference bit to be cleared, but it * is necessary that 0 only be returned when there are truly no * reference bits set. * * As an optimization, update the page's dirty field if a modified bit is * found while counting reference bits. This opportunistic update can be * performed at low cost and can eliminate the need for some future calls * to pmap_is_modified(). However, since this function stops after * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some * dirty pages. Those dirty pages will only be detected by a future call * to pmap_is_modified(). */ static int mmu_booke_ts_referenced(mmu_t mmu, vm_page_t m) { pte_t *pte; pv_entry_t pv; int count; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("mmu_booke_ts_referenced: page %p is not managed", m)); count = 0; rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { PMAP_LOCK(pv->pv_pmap); if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL && PTE_ISVALID(pte)) { if (PTE_ISMODIFIED(pte)) vm_page_dirty(m); if (PTE_ISREFERENCED(pte)) { mtx_lock_spin(&tlbivax_mutex); tlb_miss_lock(); tlb0_flush_entry(pv->pv_va); *pte &= ~PTE_REFERENCED; tlb_miss_unlock(); mtx_unlock_spin(&tlbivax_mutex); if (++count >= PMAP_TS_REFERENCED_MAX) { PMAP_UNLOCK(pv->pv_pmap); break; } } } PMAP_UNLOCK(pv->pv_pmap); } rw_wunlock(&pvh_global_lock); return (count); } /* * Clear the wired attribute from the mappings for the specified range of * addresses in the given pmap. Every valid mapping within that range must * have the wired attribute set. In contrast, invalid mappings cannot have * the wired attribute set, so they are ignored. * * The wired attribute of the page table entry is not a hardware feature, so * there is no need to invalidate any TLB entries. */ static void mmu_booke_unwire(mmu_t mmu, pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { vm_offset_t va; pte_t *pte; PMAP_LOCK(pmap); for (va = sva; va < eva; va += PAGE_SIZE) { if ((pte = pte_find(mmu, pmap, va)) != NULL && PTE_ISVALID(pte)) { if (!PTE_ISWIRED(pte)) panic("mmu_booke_unwire: pte %p isn't wired", pte); *pte &= ~PTE_WIRED; pmap->pm_stats.wired_count--; } } PMAP_UNLOCK(pmap); } /* * Return true if the pmap's pv is one of the first 16 pvs linked to from this * page. This count may be changed upwards or downwards in the future; it is * only necessary that true be returned for a small subset of pmaps for proper * page aging. */ static boolean_t mmu_booke_page_exists_quick(mmu_t mmu, pmap_t pmap, vm_page_t m) { pv_entry_t pv; int loops; boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("mmu_booke_page_exists_quick: page %p is not managed", m)); loops = 0; rv = FALSE; rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { if (pv->pv_pmap == pmap) { rv = TRUE; break; } if (++loops >= 16) break; } rw_wunlock(&pvh_global_lock); return (rv); } /* * Return the number of managed mappings to the given physical page that are * wired. */ static int mmu_booke_page_wired_mappings(mmu_t mmu, vm_page_t m) { pv_entry_t pv; pte_t *pte; int count = 0; if ((m->oflags & VPO_UNMANAGED) != 0) return (count); rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { PMAP_LOCK(pv->pv_pmap); if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL) if (PTE_ISVALID(pte) && PTE_ISWIRED(pte)) count++; PMAP_UNLOCK(pv->pv_pmap); } rw_wunlock(&pvh_global_lock); return (count); } static int mmu_booke_dev_direct_mapped(mmu_t mmu, vm_paddr_t pa, vm_size_t size) { int i; vm_offset_t va; /* * This currently does not work for entries that * overlap TLB1 entries. */ for (i = 0; i < TLB1_ENTRIES; i ++) { if (tlb1_iomapped(i, pa, size, &va) == 0) return (0); } return (EFAULT); } void mmu_booke_dumpsys_map(mmu_t mmu, vm_paddr_t pa, size_t sz, void **va) { vm_paddr_t ppa; vm_offset_t ofs; vm_size_t gran; /* Minidumps are based on virtual memory addresses. */ if (do_minidump) { *va = (void *)(vm_offset_t)pa; return; } /* Raw physical memory dumps don't have a virtual address. */ /* We always map a 256MB page at 256M. */ gran = 256 * 1024 * 1024; ppa = rounddown2(pa, gran); ofs = pa - ppa; *va = (void *)gran; tlb1_set_entry((vm_offset_t)va, ppa, gran, _TLB_ENTRY_IO); if (sz > (gran - ofs)) tlb1_set_entry((vm_offset_t)(va + gran), ppa + gran, gran, _TLB_ENTRY_IO); } void mmu_booke_dumpsys_unmap(mmu_t mmu, vm_paddr_t pa, size_t sz, void *va) { vm_paddr_t ppa; vm_offset_t ofs; vm_size_t gran; tlb_entry_t e; int i; /* Minidumps are based on virtual memory addresses. */ /* Nothing to do... */ if (do_minidump) return; for (i = 0; i < TLB1_ENTRIES; i++) { tlb1_read_entry(&e, i); if (!(e.mas1 & MAS1_VALID)) break; } /* Raw physical memory dumps don't have a virtual address. */ i--; e.mas1 = 0; e.mas2 = 0; e.mas3 = 0; tlb1_write_entry(&e, i); gran = 256 * 1024 * 1024; ppa = rounddown2(pa, gran); ofs = pa - ppa; if (sz > (gran - ofs)) { i--; e.mas1 = 0; e.mas2 = 0; e.mas3 = 0; tlb1_write_entry(&e, i); } } extern struct dump_pa dump_map[PHYS_AVAIL_SZ + 1]; void mmu_booke_scan_init(mmu_t mmu) { vm_offset_t va; pte_t *pte; int i; if (!do_minidump) { /* Initialize phys. segments for dumpsys(). */ memset(&dump_map, 0, sizeof(dump_map)); mem_regions(&physmem_regions, &physmem_regions_sz, &availmem_regions, &availmem_regions_sz); for (i = 0; i < physmem_regions_sz; i++) { dump_map[i].pa_start = physmem_regions[i].mr_start; dump_map[i].pa_size = physmem_regions[i].mr_size; } return; } /* Virtual segments for minidumps: */ memset(&dump_map, 0, sizeof(dump_map)); /* 1st: kernel .data and .bss. */ dump_map[0].pa_start = trunc_page((uintptr_t)_etext); dump_map[0].pa_size = round_page((uintptr_t)_end) - dump_map[0].pa_start; /* 2nd: msgbuf and tables (see pmap_bootstrap()). */ dump_map[1].pa_start = data_start; dump_map[1].pa_size = data_end - data_start; /* 3rd: kernel VM. */ va = dump_map[1].pa_start + dump_map[1].pa_size; /* Find start of next chunk (from va). */ while (va < virtual_end) { /* Don't dump the buffer cache. */ if (va >= kmi.buffer_sva && va < kmi.buffer_eva) { va = kmi.buffer_eva; continue; } pte = pte_find(mmu, kernel_pmap, va); if (pte != NULL && PTE_ISVALID(pte)) break; va += PAGE_SIZE; } if (va < virtual_end) { dump_map[2].pa_start = va; va += PAGE_SIZE; /* Find last page in chunk. */ while (va < virtual_end) { /* Don't run into the buffer cache. */ if (va == kmi.buffer_sva) break; pte = pte_find(mmu, kernel_pmap, va); if (pte == NULL || !PTE_ISVALID(pte)) break; va += PAGE_SIZE; } dump_map[2].pa_size = va - dump_map[2].pa_start; } } /* * Map a set of physical memory pages into the kernel virtual address space. * Return a pointer to where it is mapped. This routine is intended to be used * for mapping device memory, NOT real memory. */ static void * mmu_booke_mapdev(mmu_t mmu, vm_paddr_t pa, vm_size_t size) { return (mmu_booke_mapdev_attr(mmu, pa, size, VM_MEMATTR_DEFAULT)); } static void * mmu_booke_mapdev_attr(mmu_t mmu, vm_paddr_t pa, vm_size_t size, vm_memattr_t ma) { tlb_entry_t e; void *res; uintptr_t va, tmpva; vm_size_t sz; int i; /* * Check if this is premapped in TLB1. Note: this should probably also * check whether a sequence of TLB1 entries exist that match the * requirement, but now only checks the easy case. */ if (ma == VM_MEMATTR_DEFAULT) { for (i = 0; i < TLB1_ENTRIES; i++) { tlb1_read_entry(&e, i); if (!(e.mas1 & MAS1_VALID)) continue; if (pa >= e.phys && (pa + size) <= (e.phys + e.size)) return (void *)(e.virt + (vm_offset_t)(pa - e.phys)); } } size = roundup(size, PAGE_SIZE); /* * The device mapping area is between VM_MAXUSER_ADDRESS and * VM_MIN_KERNEL_ADDRESS. This gives 1GB of device addressing. */ #ifdef SPARSE_MAPDEV /* * With a sparse mapdev, align to the largest starting region. This * could feasibly be optimized for a 'best-fit' alignment, but that * calculation could be very costly. */ do { tmpva = tlb1_map_base; va = roundup(tlb1_map_base, 1 << flsl(size)); } while (!atomic_cmpset_int(&tlb1_map_base, tmpva, va + size)); #else va = atomic_fetchadd_int(&tlb1_map_base, size); #endif res = (void *)va; do { sz = 1 << (ilog2(size) & ~1); if (va % sz != 0) { do { sz >>= 2; } while (va % sz != 0); } if (bootverbose) printf("Wiring VA=%x to PA=%jx (size=%x)\n", va, (uintmax_t)pa, sz); tlb1_set_entry(va, pa, sz, _TLB_ENTRY_SHARED | tlb_calc_wimg(pa, ma)); size -= sz; pa += sz; va += sz; } while (size > 0); return (res); } /* * 'Unmap' a range mapped by mmu_booke_mapdev(). */ static void mmu_booke_unmapdev(mmu_t mmu, vm_offset_t va, vm_size_t size) { #ifdef SUPPORTS_SHRINKING_TLB1 vm_offset_t base, offset; /* * Unmap only if this is inside kernel virtual space. */ if ((va >= VM_MIN_KERNEL_ADDRESS) && (va <= VM_MAX_KERNEL_ADDRESS)) { base = trunc_page(va); offset = va & PAGE_MASK; size = roundup(offset + size, PAGE_SIZE); kva_free(base, size); } #endif } /* * mmu_booke_object_init_pt preloads the ptes for a given object into the * specified pmap. This eliminates the blast of soft faults on process startup * and immediately after an mmap. */ static void mmu_booke_object_init_pt(mmu_t mmu, pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("mmu_booke_object_init_pt: non-device object")); } /* * Perform the pmap work for mincore. */ static int mmu_booke_mincore(mmu_t mmu, pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) { /* XXX: this should be implemented at some point */ return (0); } static int mmu_booke_change_attr(mmu_t mmu, vm_offset_t addr, vm_size_t sz, vm_memattr_t mode) { vm_offset_t va; pte_t *pte; int i, j; tlb_entry_t e; /* Check TLB1 mappings */ for (i = 0; i < TLB1_ENTRIES; i++) { tlb1_read_entry(&e, i); if (!(e.mas1 & MAS1_VALID)) continue; if (addr >= e.virt && addr < e.virt + e.size) break; } if (i < TLB1_ENTRIES) { /* Only allow full mappings to be modified for now. */ /* Validate the range. */ for (j = i, va = addr; va < addr + sz; va += e.size, j++) { tlb1_read_entry(&e, j); if (va != e.virt || (sz - (va - addr) < e.size)) return (EINVAL); } for (va = addr; va < addr + sz; va += e.size, i++) { tlb1_read_entry(&e, i); e.mas2 &= ~MAS2_WIMGE_MASK; e.mas2 |= tlb_calc_wimg(e.phys, mode); /* * Write it out to the TLB. Should really re-sync with other * cores. */ tlb1_write_entry(&e, i); } return (0); } /* Not in TLB1, try through pmap */ /* First validate the range. */ for (va = addr; va < addr + sz; va += PAGE_SIZE) { pte = pte_find(mmu, kernel_pmap, va); if (pte == NULL || !PTE_ISVALID(pte)) return (EINVAL); } mtx_lock_spin(&tlbivax_mutex); tlb_miss_lock(); for (va = addr; va < addr + sz; va += PAGE_SIZE) { pte = pte_find(mmu, kernel_pmap, va); *pte &= ~(PTE_MAS2_MASK << PTE_MAS2_SHIFT); *pte |= tlb_calc_wimg(PTE_PA(pte), mode << PTE_MAS2_SHIFT); tlb0_flush_entry(va); } tlb_miss_unlock(); mtx_unlock_spin(&tlbivax_mutex); return (pte_vatopa(mmu, kernel_pmap, va)); } /**************************************************************************/ /* TID handling */ /**************************************************************************/ /* * Allocate a TID. If necessary, steal one from someone else. * The new TID is flushed from the TLB before returning. */ static tlbtid_t tid_alloc(pmap_t pmap) { tlbtid_t tid; int thiscpu; KASSERT((pmap != kernel_pmap), ("tid_alloc: kernel pmap")); CTR2(KTR_PMAP, "%s: s (pmap = %p)", __func__, pmap); thiscpu = PCPU_GET(cpuid); tid = PCPU_GET(tid_next); if (tid > TID_MAX) tid = TID_MIN; PCPU_SET(tid_next, tid + 1); /* If we are stealing TID then clear the relevant pmap's field */ if (tidbusy[thiscpu][tid] != NULL) { CTR2(KTR_PMAP, "%s: warning: stealing tid %d", __func__, tid); tidbusy[thiscpu][tid]->pm_tid[thiscpu] = TID_NONE; /* Flush all entries from TLB0 matching this TID. */ tid_flush(tid); } tidbusy[thiscpu][tid] = pmap; pmap->pm_tid[thiscpu] = tid; __asm __volatile("msync; isync"); CTR3(KTR_PMAP, "%s: e (%02d next = %02d)", __func__, tid, PCPU_GET(tid_next)); return (tid); } /**************************************************************************/ /* TLB0 handling */ /**************************************************************************/ static void tlb_print_entry(int i, uint32_t mas1, uint32_t mas2, uint32_t mas3, uint32_t mas7) { int as; char desc[3]; tlbtid_t tid; vm_size_t size; unsigned int tsize; desc[2] = '\0'; if (mas1 & MAS1_VALID) desc[0] = 'V'; else desc[0] = ' '; if (mas1 & MAS1_IPROT) desc[1] = 'P'; else desc[1] = ' '; as = (mas1 & MAS1_TS_MASK) ? 1 : 0; tid = MAS1_GETTID(mas1); tsize = (mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT; size = 0; if (tsize) size = tsize2size(tsize); debugf("%3d: (%s) [AS=%d] " "sz = 0x%08x tsz = %d tid = %d mas1 = 0x%08x " "mas2(va) = 0x%08x mas3(pa) = 0x%08x mas7 = 0x%08x\n", i, desc, as, size, tsize, tid, mas1, mas2, mas3, mas7); } /* Convert TLB0 va and way number to tlb0[] table index. */ static inline unsigned int tlb0_tableidx(vm_offset_t va, unsigned int way) { unsigned int idx; idx = (way * TLB0_ENTRIES_PER_WAY); idx += (va & MAS2_TLB0_ENTRY_IDX_MASK) >> MAS2_TLB0_ENTRY_IDX_SHIFT; return (idx); } /* * Invalidate TLB0 entry. */ static inline void tlb0_flush_entry(vm_offset_t va) { CTR2(KTR_PMAP, "%s: s va=0x%08x", __func__, va); mtx_assert(&tlbivax_mutex, MA_OWNED); __asm __volatile("tlbivax 0, %0" :: "r"(va & MAS2_EPN_MASK)); __asm __volatile("isync; msync"); __asm __volatile("tlbsync; msync"); CTR1(KTR_PMAP, "%s: e", __func__); } /* Print out contents of the MAS registers for each TLB0 entry */ void tlb0_print_tlbentries(void) { uint32_t mas0, mas1, mas2, mas3, mas7; int entryidx, way, idx; debugf("TLB0 entries:\n"); for (way = 0; way < TLB0_WAYS; way ++) for (entryidx = 0; entryidx < TLB0_ENTRIES_PER_WAY; entryidx++) { mas0 = MAS0_TLBSEL(0) | MAS0_ESEL(way); mtspr(SPR_MAS0, mas0); __asm __volatile("isync"); mas2 = entryidx << MAS2_TLB0_ENTRY_IDX_SHIFT; mtspr(SPR_MAS2, mas2); __asm __volatile("isync; tlbre"); mas1 = mfspr(SPR_MAS1); mas2 = mfspr(SPR_MAS2); mas3 = mfspr(SPR_MAS3); mas7 = mfspr(SPR_MAS7); idx = tlb0_tableidx(mas2, way); tlb_print_entry(idx, mas1, mas2, mas3, mas7); } } /**************************************************************************/ /* TLB1 handling */ /**************************************************************************/ /* * TLB1 mapping notes: * * TLB1[0] Kernel text and data. * TLB1[1-15] Additional kernel text and data mappings (if required), PCI * windows, other devices mappings. */ /* * Read an entry from given TLB1 slot. */ void tlb1_read_entry(tlb_entry_t *entry, unsigned int slot) { uint32_t mas0; KASSERT((entry != NULL), ("%s(): Entry is NULL!", __func__)); mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(slot); mtspr(SPR_MAS0, mas0); __asm __volatile("isync; tlbre"); entry->mas1 = mfspr(SPR_MAS1); entry->mas2 = mfspr(SPR_MAS2); entry->mas3 = mfspr(SPR_MAS3); switch ((mfpvr() >> 16) & 0xFFFF) { case FSL_E500v2: case FSL_E500mc: case FSL_E5500: case FSL_E6500: entry->mas7 = mfspr(SPR_MAS7); break; default: entry->mas7 = 0; break; } entry->virt = entry->mas2 & MAS2_EPN_MASK; entry->phys = ((vm_paddr_t)(entry->mas7 & MAS7_RPN) << 32) | (entry->mas3 & MAS3_RPN); entry->size = tsize2size((entry->mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT); } /* * Write given entry to TLB1 hardware. * Use 32 bit pa, clear 4 high-order bits of RPN (mas7). */ static void tlb1_write_entry(tlb_entry_t *e, unsigned int idx) { uint32_t mas0; //debugf("tlb1_write_entry: s\n"); /* Select entry */ mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(idx); //debugf("tlb1_write_entry: mas0 = 0x%08x\n", mas0); mtspr(SPR_MAS0, mas0); __asm __volatile("isync"); mtspr(SPR_MAS1, e->mas1); __asm __volatile("isync"); mtspr(SPR_MAS2, e->mas2); __asm __volatile("isync"); mtspr(SPR_MAS3, e->mas3); __asm __volatile("isync"); switch ((mfpvr() >> 16) & 0xFFFF) { case FSL_E500mc: case FSL_E5500: case FSL_E6500: mtspr(SPR_MAS8, 0); __asm __volatile("isync"); /* FALLTHROUGH */ case FSL_E500v2: mtspr(SPR_MAS7, e->mas7); __asm __volatile("isync"); break; default: break; } __asm __volatile("tlbwe; isync; msync"); //debugf("tlb1_write_entry: e\n"); } /* * Return the largest uint value log such that 2^log <= num. */ static unsigned int ilog2(unsigned int num) { int lz; __asm ("cntlzw %0, %1" : "=r" (lz) : "r" (num)); return (31 - lz); } /* * Convert TLB TSIZE value to mapped region size. */ static vm_size_t tsize2size(unsigned int tsize) { /* * size = 4^tsize KB * size = 4^tsize * 2^10 = 2^(2 * tsize - 10) */ return ((1 << (2 * tsize)) * 1024); } /* * Convert region size (must be power of 4) to TLB TSIZE value. */ static unsigned int size2tsize(vm_size_t size) { return (ilog2(size) / 2 - 5); } /* * Register permanent kernel mapping in TLB1. * * Entries are created starting from index 0 (current free entry is * kept in tlb1_idx) and are not supposed to be invalidated. */ int tlb1_set_entry(vm_offset_t va, vm_paddr_t pa, vm_size_t size, uint32_t flags) { tlb_entry_t e; uint32_t ts, tid; int tsize, index; for (index = 0; index < TLB1_ENTRIES; index++) { tlb1_read_entry(&e, index); if ((e.mas1 & MAS1_VALID) == 0) break; /* Check if we're just updating the flags, and update them. */ if (e.phys == pa && e.virt == va && e.size == size) { e.mas2 = (va & MAS2_EPN_MASK) | flags; tlb1_write_entry(&e, index); return (0); } } if (index >= TLB1_ENTRIES) { printf("tlb1_set_entry: TLB1 full!\n"); return (-1); } /* Convert size to TSIZE */ tsize = size2tsize(size); tid = (TID_KERNEL << MAS1_TID_SHIFT) & MAS1_TID_MASK; /* XXX TS is hard coded to 0 for now as we only use single address space */ ts = (0 << MAS1_TS_SHIFT) & MAS1_TS_MASK; e.phys = pa; e.virt = va; e.size = size; e.mas1 = MAS1_VALID | MAS1_IPROT | ts | tid; e.mas1 |= ((tsize << MAS1_TSIZE_SHIFT) & MAS1_TSIZE_MASK); e.mas2 = (va & MAS2_EPN_MASK) | flags; /* Set supervisor RWX permission bits */ e.mas3 = (pa & MAS3_RPN) | MAS3_SR | MAS3_SW | MAS3_SX; e.mas7 = (pa >> 32) & MAS7_RPN; tlb1_write_entry(&e, index); /* * XXX in general TLB1 updates should be propagated between CPUs, * since current design assumes to have the same TLB1 set-up on all * cores. */ return (0); } /* * Map in contiguous RAM region into the TLB1 using maximum of * KERNEL_REGION_MAX_TLB_ENTRIES entries. * * If necessary round up last entry size and return total size * used by all allocated entries. */ vm_size_t tlb1_mapin_region(vm_offset_t va, vm_paddr_t pa, vm_size_t size) { vm_size_t pgs[KERNEL_REGION_MAX_TLB_ENTRIES]; vm_size_t mapped, pgsz, base, mask; int idx, nents; /* Round up to the next 1M */ size = roundup2(size, 1 << 20); mapped = 0; idx = 0; base = va; pgsz = 64*1024*1024; while (mapped < size) { while (mapped < size && idx < KERNEL_REGION_MAX_TLB_ENTRIES) { while (pgsz > (size - mapped)) pgsz >>= 2; pgs[idx++] = pgsz; mapped += pgsz; } /* We under-map. Correct for this. */ if (mapped < size) { while (pgs[idx - 1] == pgsz) { idx--; mapped -= pgsz; } /* XXX We may increase beyond out starting point. */ pgsz <<= 2; pgs[idx++] = pgsz; mapped += pgsz; } } nents = idx; mask = pgs[0] - 1; /* Align address to the boundary */ if (va & mask) { va = (va + mask) & ~mask; pa = (pa + mask) & ~mask; } for (idx = 0; idx < nents; idx++) { pgsz = pgs[idx]; debugf("%u: %llx -> %x, size=%x\n", idx, pa, va, pgsz); tlb1_set_entry(va, pa, pgsz, _TLB_ENTRY_SHARED | _TLB_ENTRY_MEM); pa += pgsz; va += pgsz; } mapped = (va - base); #ifdef __powerpc64__ printf("mapped size 0x%016lx (wasted space 0x%16lx)\n", #else printf("mapped size 0x%08x (wasted space 0x%08x)\n", #endif mapped, mapped - size); return (mapped); } /* * TLB1 initialization routine, to be called after the very first * assembler level setup done in locore.S. */ void tlb1_init() { uint32_t mas0, mas1, mas2, mas3, mas7; uint32_t tsz; tlb1_get_tlbconf(); mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(0); mtspr(SPR_MAS0, mas0); __asm __volatile("isync; tlbre"); mas1 = mfspr(SPR_MAS1); mas2 = mfspr(SPR_MAS2); mas3 = mfspr(SPR_MAS3); mas7 = mfspr(SPR_MAS7); kernload = ((vm_paddr_t)(mas7 & MAS7_RPN) << 32) | (mas3 & MAS3_RPN); tsz = (mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT; kernsize += (tsz > 0) ? tsize2size(tsz) : 0; /* Setup TLB miss defaults */ set_mas4_defaults(); } /* * pmap_early_io_unmap() should be used in short conjunction with * pmap_early_io_map(), as in the following snippet: * * x = pmap_early_io_map(...); * * pmap_early_io_unmap(x, size); * * And avoiding more allocations between. */ void pmap_early_io_unmap(vm_offset_t va, vm_size_t size) { int i; tlb_entry_t e; vm_size_t isize; size = roundup(size, PAGE_SIZE); isize = size; for (i = 0; i < TLB1_ENTRIES && size > 0; i++) { tlb1_read_entry(&e, i); if (!(e.mas1 & MAS1_VALID)) continue; if (va <= e.virt && (va + isize) >= (e.virt + e.size)) { size -= e.size; e.mas1 &= ~MAS1_VALID; tlb1_write_entry(&e, i); } } if (tlb1_map_base == va + isize) tlb1_map_base -= isize; } vm_offset_t pmap_early_io_map(vm_paddr_t pa, vm_size_t size) { vm_paddr_t pa_base; vm_offset_t va, sz; int i; tlb_entry_t e; KASSERT(!pmap_bootstrapped, ("Do not use after PMAP is up!")); for (i = 0; i < TLB1_ENTRIES; i++) { tlb1_read_entry(&e, i); if (!(e.mas1 & MAS1_VALID)) continue; if (pa >= e.phys && (pa + size) <= (e.phys + e.size)) return (e.virt + (pa - e.phys)); } pa_base = rounddown(pa, PAGE_SIZE); size = roundup(size + (pa - pa_base), PAGE_SIZE); tlb1_map_base = roundup2(tlb1_map_base, 1 << (ilog2(size) & ~1)); va = tlb1_map_base + (pa - pa_base); do { sz = 1 << (ilog2(size) & ~1); tlb1_set_entry(tlb1_map_base, pa_base, sz, _TLB_ENTRY_SHARED | _TLB_ENTRY_IO); size -= sz; pa_base += sz; tlb1_map_base += sz; } while (size > 0); return (va); } + +void +pmap_track_page(pmap_t pmap, vm_offset_t va) +{ + vm_paddr_t pa; + vm_page_t page; + struct pv_entry *pve; + + va &= ~PAGE_MASK; + pa = pmap_kextract(va); + + rw_wlock(&pvh_global_lock); + PMAP_LOCK(pmap); + page = PHYS_TO_VM_PAGE(pa); + + TAILQ_FOREACH(pve, &page->md.pv_list, pv_link) { + if ((pmap == pve->pv_pmap) && (va == pve->pv_va)) { + goto out; + } + } + page->md.pv_tracked = true; + pv_insert(pmap, va, page); +out: + PMAP_UNLOCK(pmap); + rw_wunlock(&pvh_global_lock); +} + /* * Setup MAS4 defaults. * These values are loaded to MAS0-2 on a TLB miss. */ static void set_mas4_defaults(void) { uint32_t mas4; /* Defaults: TLB0, PID0, TSIZED=4K */ mas4 = MAS4_TLBSELD0; mas4 |= (TLB_SIZE_4K << MAS4_TSIZED_SHIFT) & MAS4_TSIZED_MASK; #ifdef SMP mas4 |= MAS4_MD; #endif mtspr(SPR_MAS4, mas4); __asm __volatile("isync"); } /* * Print out contents of the MAS registers for each TLB1 entry */ void tlb1_print_tlbentries(void) { uint32_t mas0, mas1, mas2, mas3, mas7; int i; debugf("TLB1 entries:\n"); for (i = 0; i < TLB1_ENTRIES; i++) { mas0 = MAS0_TLBSEL(1) | MAS0_ESEL(i); mtspr(SPR_MAS0, mas0); __asm __volatile("isync; tlbre"); mas1 = mfspr(SPR_MAS1); mas2 = mfspr(SPR_MAS2); mas3 = mfspr(SPR_MAS3); mas7 = mfspr(SPR_MAS7); tlb_print_entry(i, mas1, mas2, mas3, mas7); } } /* * Return 0 if the physical IO range is encompassed by one of the * the TLB1 entries, otherwise return related error code. */ static int tlb1_iomapped(int i, vm_paddr_t pa, vm_size_t size, vm_offset_t *va) { uint32_t prot; vm_paddr_t pa_start; vm_paddr_t pa_end; unsigned int entry_tsize; vm_size_t entry_size; tlb_entry_t e; *va = (vm_offset_t)NULL; tlb1_read_entry(&e, i); /* Skip invalid entries */ if (!(e.mas1 & MAS1_VALID)) return (EINVAL); /* * The entry must be cache-inhibited, guarded, and r/w * so it can function as an i/o page */ prot = e.mas2 & (MAS2_I | MAS2_G); if (prot != (MAS2_I | MAS2_G)) return (EPERM); prot = e.mas3 & (MAS3_SR | MAS3_SW); if (prot != (MAS3_SR | MAS3_SW)) return (EPERM); /* The address should be within the entry range. */ entry_tsize = (e.mas1 & MAS1_TSIZE_MASK) >> MAS1_TSIZE_SHIFT; KASSERT((entry_tsize), ("tlb1_iomapped: invalid entry tsize")); entry_size = tsize2size(entry_tsize); pa_start = (((vm_paddr_t)e.mas7 & MAS7_RPN) << 32) | (e.mas3 & MAS3_RPN); pa_end = pa_start + entry_size; if ((pa < pa_start) || ((pa + size) > pa_end)) return (ERANGE); /* Return virtual address of this mapping. */ *va = (e.mas2 & MAS2_EPN_MASK) + (pa - pa_start); return (0); } /* * Invalidate all TLB0 entries which match the given TID. Note this is * dedicated for cases when invalidations should NOT be propagated to other * CPUs. */ static void tid_flush(tlbtid_t tid) { register_t msr; uint32_t mas0, mas1, mas2; int entry, way; /* Don't evict kernel translations */ if (tid == TID_KERNEL) return; msr = mfmsr(); __asm __volatile("wrteei 0"); for (way = 0; way < TLB0_WAYS; way++) for (entry = 0; entry < TLB0_ENTRIES_PER_WAY; entry++) { mas0 = MAS0_TLBSEL(0) | MAS0_ESEL(way); mtspr(SPR_MAS0, mas0); __asm __volatile("isync"); mas2 = entry << MAS2_TLB0_ENTRY_IDX_SHIFT; mtspr(SPR_MAS2, mas2); __asm __volatile("isync; tlbre"); mas1 = mfspr(SPR_MAS1); if (!(mas1 & MAS1_VALID)) continue; if (((mas1 & MAS1_TID_MASK) >> MAS1_TID_SHIFT) != tid) continue; mas1 &= ~MAS1_VALID; mtspr(SPR_MAS1, mas1); __asm __volatile("isync; tlbwe; isync; msync"); } mtmsr(msr); } Index: head/sys/powerpc/include/pmap.h =================================================================== --- head/sys/powerpc/include/pmap.h (revision 308717) +++ head/sys/powerpc/include/pmap.h (revision 308718) @@ -1,267 +1,269 @@ /*- * Copyright (C) 2006 Semihalf, Marian Balakowicz * All rights reserved. * * Adapted for Freescale's e500 core CPUs. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ /*- * Copyright (C) 1995, 1996 Wolfgang Solfrank. * Copyright (C) 1995, 1996 TooLs GmbH. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * from: $NetBSD: pmap.h,v 1.17 2000/03/30 16:18:24 jdolecek Exp $ */ #ifndef _MACHINE_PMAP_H_ #define _MACHINE_PMAP_H_ #include #include #include #include #include #include #include #include #include #if defined(AIM) #if !defined(NPMAPS) #define NPMAPS 32768 #endif /* !defined(NPMAPS) */ struct slbtnode; struct pmap; typedef struct pmap *pmap_t; struct pvo_entry { LIST_ENTRY(pvo_entry) pvo_vlink; /* Link to common virt page */ #ifndef __powerpc64__ LIST_ENTRY(pvo_entry) pvo_olink; /* Link to overflow entry */ #endif RB_ENTRY(pvo_entry) pvo_plink; /* Link to pmap entries */ struct { #ifndef __powerpc64__ /* 32-bit fields */ struct pte pte; #endif /* 64-bit fields */ uintptr_t slot; vm_paddr_t pa; vm_prot_t prot; } pvo_pte; pmap_t pvo_pmap; /* Owning pmap */ vm_offset_t pvo_vaddr; /* VA of entry */ uint64_t pvo_vpn; /* Virtual page number */ }; LIST_HEAD(pvo_head, pvo_entry); RB_HEAD(pvo_tree, pvo_entry); int pvo_vaddr_compare(struct pvo_entry *, struct pvo_entry *); RB_PROTOTYPE(pvo_tree, pvo_entry, pvo_plink, pvo_vaddr_compare); /* Used by 32-bit PMAP */ #define PVO_PTEGIDX_MASK 0x007UL /* which PTEG slot */ #define PVO_PTEGIDX_VALID 0x008UL /* slot is valid */ /* Used by 64-bit PMAP */ #define PVO_HID 0x008UL /* PVO entry in alternate hash*/ /* Used by both */ #define PVO_WIRED 0x010UL /* PVO entry is wired */ #define PVO_MANAGED 0x020UL /* PVO entry is managed */ #define PVO_BOOTSTRAP 0x080UL /* PVO entry allocated during bootstrap */ #define PVO_DEAD 0x100UL /* waiting to be deleted */ #define PVO_LARGE 0x200UL /* large page */ #define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF) #define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK) #define PVO_PTEGIDX_ISSET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID) #define PVO_PTEGIDX_CLR(pvo) \ ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK))) #define PVO_PTEGIDX_SET(pvo, i) \ ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID)) #define PVO_VSID(pvo) ((pvo)->pvo_vpn >> 16) struct pmap { struct mtx pm_mtx; #ifdef __powerpc64__ struct slbtnode *pm_slb_tree_root; struct slb **pm_slb; int pm_slb_len; #else register_t pm_sr[16]; #endif cpuset_t pm_active; struct pmap *pmap_phys; struct pmap_statistics pm_stats; struct pvo_tree pmap_pvo; }; struct md_page { volatile int32_t mdpg_attrs; vm_memattr_t mdpg_cache_attrs; struct pvo_head mdpg_pvoh; }; #define pmap_page_get_memattr(m) ((m)->md.mdpg_cache_attrs) #define pmap_page_is_mapped(m) (!LIST_EMPTY(&(m)->md.mdpg_pvoh)) /* * Return the VSID corresponding to a given virtual address. * If no VSID is currently defined, it will allocate one, and add * it to a free slot if available. * * NB: The PMAP MUST be locked already. */ uint64_t va_to_vsid(pmap_t pm, vm_offset_t va); /* Lock-free, non-allocating lookup routines */ uint64_t kernel_va_to_slbv(vm_offset_t va); struct slb *user_va_to_slb_entry(pmap_t pm, vm_offset_t va); uint64_t allocate_user_vsid(pmap_t pm, uint64_t esid, int large); void free_vsid(pmap_t pm, uint64_t esid, int large); void slb_insert_user(pmap_t pm, struct slb *slb); void slb_insert_kernel(uint64_t slbe, uint64_t slbv); struct slbtnode *slb_alloc_tree(void); void slb_free_tree(pmap_t pm); struct slb **slb_alloc_user_cache(void); void slb_free_user_cache(struct slb **); #else struct pmap { struct mtx pm_mtx; /* pmap mutex */ tlbtid_t pm_tid[MAXCPU]; /* TID to identify this pmap entries in TLB */ cpuset_t pm_active; /* active on cpus */ struct pmap_statistics pm_stats; /* pmap statistics */ /* Page table directory, array of pointers to page tables. */ pte_t *pm_pdir[PDIR_NENTRIES]; /* List of allocated ptbl bufs (ptbl kva regions). */ TAILQ_HEAD(, ptbl_buf) pm_ptbl_list; }; typedef struct pmap *pmap_t; struct pv_entry { pmap_t pv_pmap; vm_offset_t pv_va; TAILQ_ENTRY(pv_entry) pv_link; }; typedef struct pv_entry *pv_entry_t; struct md_page { TAILQ_HEAD(, pv_entry) pv_list; + bool pv_tracked; }; #define pmap_page_get_memattr(m) VM_MEMATTR_DEFAULT #define pmap_page_is_mapped(m) (!TAILQ_EMPTY(&(m)->md.pv_list)) #endif /* AIM */ extern struct pmap kernel_pmap_store; #define kernel_pmap (&kernel_pmap_store) #ifdef _KERNEL #define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx) #define PMAP_LOCK_ASSERT(pmap, type) \ mtx_assert(&(pmap)->pm_mtx, (type)) #define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx) #define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, \ (pmap == kernel_pmap) ? "kernelpmap" : \ "pmap", NULL, MTX_DEF) #define PMAP_LOCKED(pmap) mtx_owned(&(pmap)->pm_mtx) #define PMAP_MTX(pmap) (&(pmap)->pm_mtx) #define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx) #define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx) #define pmap_page_is_write_mapped(m) (((m)->aflags & PGA_WRITEABLE) != 0) void pmap_bootstrap(vm_offset_t, vm_offset_t); void pmap_kenter(vm_offset_t va, vm_paddr_t pa); void pmap_kenter_attr(vm_offset_t va, vm_offset_t pa, vm_memattr_t); void pmap_kremove(vm_offset_t); void *pmap_mapdev(vm_paddr_t, vm_size_t); void *pmap_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t); void pmap_unmapdev(vm_offset_t, vm_size_t); void pmap_page_set_memattr(vm_page_t, vm_memattr_t); int pmap_change_attr(vm_offset_t, vm_size_t, vm_memattr_t); void pmap_deactivate(struct thread *); vm_paddr_t pmap_kextract(vm_offset_t); int pmap_dev_direct_mapped(vm_paddr_t, vm_size_t); boolean_t pmap_mmu_install(char *name, int prio); #define vtophys(va) pmap_kextract((vm_offset_t)(va)) #define PHYS_AVAIL_SZ 256 /* Allows up to 16GB Ram on pSeries with * logical memory block size of 64MB. * For more Ram increase the lmb or this value. */ extern vm_paddr_t phys_avail[PHYS_AVAIL_SZ]; extern vm_offset_t virtual_avail; extern vm_offset_t virtual_end; extern vm_offset_t msgbuf_phys; extern int pmap_bootstrapped; vm_offset_t pmap_early_io_map(vm_paddr_t pa, vm_size_t size); void pmap_early_io_unmap(vm_offset_t va, vm_size_t size); +void pmap_track_page(pmap_t pmap, vm_offset_t va); #endif #endif /* !_MACHINE_PMAP_H_ */ Index: head/sys/powerpc/mpc85xx/mpc85xx.h =================================================================== --- head/sys/powerpc/mpc85xx/mpc85xx.h (revision 308717) +++ head/sys/powerpc/mpc85xx/mpc85xx.h (revision 308718) @@ -1,173 +1,175 @@ /*- * Copyright (C) 2008 Semihalf, Rafal Jaworowski * Copyright 2006 by Juniper Networks. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MPC85XX_H_ #define _MPC85XX_H_ #include /* * Configuration control and status registers */ extern vm_offset_t ccsrbar_va; +extern vm_paddr_t ccsrbar_pa; +extern vm_size_t ccsrbar_size; #define CCSRBAR_VA ccsrbar_va #define OCP85XX_CCSRBAR (CCSRBAR_VA + 0x0) #define OCP85XX_BPTR (CCSRBAR_VA + 0x20) #define OCP85XX_BSTRH (CCSRBAR_VA + 0x20) #define OCP85XX_BSTRL (CCSRBAR_VA + 0x24) #define OCP85XX_BSTAR (CCSRBAR_VA + 0x28) #define OCP85XX_COREDISR (CCSRBAR_VA + 0xE0094) #define OCP85XX_BRR (CCSRBAR_VA + 0xE00E4) /* * Run Control and Power Management registers */ #define CCSR_CTBENR (CCSRBAR_VA + 0xE2084) #define CCSR_CTBCKSELR (CCSRBAR_VA + 0xE208C) #define CCSR_CTBCHLTCR (CCSRBAR_VA + 0xE2094) /* * DDR Memory controller. */ #define OCP85XX_DDR1_CS0_CONFIG (CCSRBAR_VA + 0x8080) /* * E500 Coherency Module registers */ #define OCP85XX_EEBPCR (CCSRBAR_VA + 0x1010) /* * Local access registers */ /* Write order: OCP_LAWBARH -> OCP_LAWBARL -> OCP_LAWSR */ #define OCP85XX_LAWBARH(n) (CCSRBAR_VA + 0xc00 + 0x10 * (n)) #define OCP85XX_LAWBARL(n) (CCSRBAR_VA + 0xc04 + 0x10 * (n)) #define OCP85XX_LAWSR_QORIQ(n) (CCSRBAR_VA + 0xc08 + 0x10 * (n)) #define OCP85XX_LAWBAR(n) (CCSRBAR_VA + 0xc08 + 0x10 * (n)) #define OCP85XX_LAWSR_85XX(n) (CCSRBAR_VA + 0xc10 + 0x10 * (n)) #define OCP85XX_LAWSR(n) (mpc85xx_is_qoriq() ? OCP85XX_LAWSR_QORIQ(n) : \ OCP85XX_LAWSR_85XX(n)) /* Attribute register */ #define OCP85XX_ENA_MASK 0x80000000 #define OCP85XX_DIS_MASK 0x7fffffff #define OCP85XX_TGTIF_LBC_QORIQ 0x1f #define OCP85XX_TGTIF_RAM_INTL_QORIQ 0x14 #define OCP85XX_TGTIF_RAM1_QORIQ 0x10 #define OCP85XX_TGTIF_RAM2_QORIQ 0x11 #define OCP85XX_TGTIF_BMAN 0x18 #define OCP85XX_TGTIF_DCSR 0x1D #define OCP85XX_TGTIF_QMAN 0x3C #define OCP85XX_TRGT_SHIFT_QORIQ 20 #define OCP85XX_TGTIF_LBC_85XX 0x04 #define OCP85XX_TGTIF_RAM_INTL_85XX 0x0b #define OCP85XX_TGTIF_RIO_85XX 0x0c #define OCP85XX_TGTIF_RAM1_85XX 0x0f #define OCP85XX_TGTIF_RAM2_85XX 0x16 #define OCP85XX_TGTIF_LBC \ (mpc85xx_is_qoriq() ? OCP85XX_TGTIF_LBC_QORIQ : OCP85XX_TGTIF_LBC_85XX) #define OCP85XX_TGTIF_RAM_INTL \ (mpc85xx_is_qoriq() ? OCP85XX_TGTIF_RAM_INTL_QORIQ : OCP85XX_TGTIF_RAM_INTL_85XX) #define OCP85XX_TGTIF_RIO \ (mpc85xx_is_qoriq() ? OCP85XX_TGTIF_RIO_QORIQ : OCP85XX_TGTIF_RIO_85XX) #define OCP85XX_TGTIF_RAM1 \ (mpc85xx_is_qoriq() ? OCP85XX_TGTIF_RAM1_QORIQ : OCP85XX_TGTIF_RAM1_85XX) #define OCP85XX_TGTIF_RAM2 \ (mpc85xx_is_qoriq() ? OCP85XX_TGTIF_RAM2_QORIQ : OCP85XX_TGTIF_RAM2_85XX) /* * L2 cache registers */ #define OCP85XX_L2CTL (CCSRBAR_VA + 0x20000) /* * L3 CoreNet platform cache (CPC) registers */ #define OCP85XX_CPC_CSR0 (CCSRBAR_VA + 0x10000) #define OCP85XX_CPC_CSR0_CE 0x80000000 #define OCP85XX_CPC_CSR0_PE 0x40000000 #define OCP85XX_CPC_CSR0_FI 0x00200000 #define OCP85XX_CPC_CSR0_WT 0x00080000 #define OCP85XX_CPC_CSR0_FL 0x00000800 #define OCP85XX_CPC_CSR0_LFC 0x00000400 #define OCP85XX_CPC_CFG0 (CCSRBAR_VA + 0x10008) #define OCP85XX_CPC_CFG_SZ_MASK 0x00003fff #define OCP85XX_CPC_CFG0_SZ_K(x) (((x) & OCP85XX_CPC_CFG_SZ_MASK) << 6) /* * Power-On Reset configuration */ #define OCP85XX_PORDEVSR (CCSRBAR_VA + 0xe000c) #define OCP85XX_PORDEVSR_IO_SEL 0x00780000 #define OCP85XX_PORDEVSR_IO_SEL_SHIFT 19 #define OCP85XX_PORDEVSR2 (CCSRBAR_VA + 0xe0014) /* * Status Registers. */ #define OCP85XX_RSTCR (CCSRBAR_VA + 0xe00b0) #define OCP85XX_CLKDVDR (CCSRBAR_VA + 0xe0800) #define OCP85XX_CLKDVDR_PXCKEN 0x80000000 #define OCP85XX_CLKDVDR_SSICKEN 0x20000000 #define OCP85XX_CLKDVDR_PXCKINV 0x10000000 #define OCP85XX_CLKDVDR_PXCLK_MASK 0x00FF0000 #define OCP85XX_CLKDVDR_SSICLK_MASK 0x000000FF /* * Run Control/Power Management Registers. */ #define OCP85XX_RCPM_CDOZSR (CCSRBAR_VA + 0xe2004) #define OCP85XX_RCPM_CDOZCR (CCSRBAR_VA + 0xe200c) /* * Prototypes. */ uint32_t ccsr_read4(uintptr_t addr); void ccsr_write4(uintptr_t addr, uint32_t val); int law_enable(int trgt, uint64_t bar, uint32_t size); int law_disable(int trgt, uint64_t bar, uint32_t size); int law_getmax(void); int law_pci_target(struct resource *, int *, int *); DECLARE_CLASS(mpc85xx_platform); int mpc85xx_attach(platform_t); void mpc85xx_enable_l3_cache(void); void mpc85xx_fix_errata(vm_offset_t); void dataloss_erratum_access(vm_offset_t, uint32_t); int mpc85xx_is_qoriq(void); #endif /* _MPC85XX_H_ */ Index: head/sys/powerpc/mpc85xx/platform_mpc85xx.c =================================================================== --- head/sys/powerpc/mpc85xx/platform_mpc85xx.c (revision 308717) +++ head/sys/powerpc/mpc85xx/platform_mpc85xx.c (revision 308718) @@ -1,551 +1,555 @@ /*- * Copyright (c) 2008-2012 Semihalf. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "platform_if.h" #ifdef SMP extern void *ap_pcpu; extern vm_paddr_t kernload; /* Kernel physical load address */ extern uint8_t __boot_page[]; /* Boot page body */ extern uint32_t bp_kernload; struct cpu_release { uint32_t entry_h; uint32_t entry_l; uint32_t r3_h; uint32_t r3_l; uint32_t reserved; uint32_t pir; }; #endif extern uint32_t *bootinfo; +vm_paddr_t ccsrbar_pa; vm_offset_t ccsrbar_va; +vm_size_t ccsrbar_size; static int cpu, maxcpu; static int mpc85xx_probe(platform_t); static void mpc85xx_mem_regions(platform_t, struct mem_region *phys, int *physsz, struct mem_region *avail, int *availsz); static u_long mpc85xx_timebase_freq(platform_t, struct cpuref *cpuref); static int mpc85xx_smp_first_cpu(platform_t, struct cpuref *cpuref); static int mpc85xx_smp_next_cpu(platform_t, struct cpuref *cpuref); static int mpc85xx_smp_get_bsp(platform_t, struct cpuref *cpuref); static int mpc85xx_smp_start_cpu(platform_t, struct pcpu *cpu); static void mpc85xx_idle(platform_t, int cpu); static int mpc85xx_idle_wakeup(platform_t plat, int cpu); static void mpc85xx_reset(platform_t); static platform_method_t mpc85xx_methods[] = { PLATFORMMETHOD(platform_probe, mpc85xx_probe), PLATFORMMETHOD(platform_attach, mpc85xx_attach), PLATFORMMETHOD(platform_mem_regions, mpc85xx_mem_regions), PLATFORMMETHOD(platform_timebase_freq, mpc85xx_timebase_freq), PLATFORMMETHOD(platform_smp_first_cpu, mpc85xx_smp_first_cpu), PLATFORMMETHOD(platform_smp_next_cpu, mpc85xx_smp_next_cpu), PLATFORMMETHOD(platform_smp_get_bsp, mpc85xx_smp_get_bsp), PLATFORMMETHOD(platform_smp_start_cpu, mpc85xx_smp_start_cpu), PLATFORMMETHOD(platform_reset, mpc85xx_reset), PLATFORMMETHOD(platform_idle, mpc85xx_idle), PLATFORMMETHOD(platform_idle_wakeup, mpc85xx_idle_wakeup), PLATFORMMETHOD_END }; DEFINE_CLASS_0(mpc85xx, mpc85xx_platform, mpc85xx_methods, 0); PLATFORM_DEF(mpc85xx_platform); static int mpc85xx_probe(platform_t plat) { u_int pvr = (mfpvr() >> 16) & 0xFFFF; switch (pvr) { case FSL_E500v1: case FSL_E500v2: case FSL_E500mc: case FSL_E5500: case FSL_E6500: return (BUS_PROBE_DEFAULT); } return (ENXIO); } int mpc85xx_attach(platform_t plat) { phandle_t cpus, child, ccsr; const char *soc_name_guesses[] = {"/soc", "soc", NULL}; const char **name; pcell_t ranges[6], acells, pacells, scells; uint64_t ccsrbar, ccsrsize; int i; if ((cpus = OF_finddevice("/cpus")) != -1) { for (maxcpu = 0, child = OF_child(cpus); child != 0; child = OF_peer(child), maxcpu++) ; } else maxcpu = 1; /* * Locate CCSR region. Irritatingly, there is no way to find it * unless you already know where it is. Try to infer its location * from the device tree. */ ccsr = -1; for (name = soc_name_guesses; *name != NULL && ccsr == -1; name++) ccsr = OF_finddevice(*name); if (ccsr == -1) { char type[64]; /* That didn't work. Search for devices of type "soc" */ child = OF_child(OF_peer(0)); for (OF_child(child); child != 0; child = OF_peer(child)) { if (OF_getprop(child, "device_type", type, sizeof(type)) <= 0) continue; if (strcmp(type, "soc") == 0) { ccsr = child; break; } } } if (ccsr == -1) panic("Could not locate CCSR window!"); OF_getprop(ccsr, "#size-cells", &scells, sizeof(scells)); OF_getprop(ccsr, "#address-cells", &acells, sizeof(acells)); OF_searchprop(OF_parent(ccsr), "#address-cells", &pacells, sizeof(pacells)); OF_getprop(ccsr, "ranges", ranges, sizeof(ranges)); ccsrbar = ccsrsize = 0; for (i = acells; i < acells + pacells; i++) { ccsrbar <<= 32; ccsrbar |= ranges[i]; } for (i = acells + pacells; i < acells + pacells + scells; i++) { ccsrsize <<= 32; ccsrsize |= ranges[i]; } ccsrbar_va = pmap_early_io_map(ccsrbar, ccsrsize); + ccsrbar_pa = ccsrbar; + ccsrbar_size = ccsrsize; #if 0 mpc85xx_fix_errata(ccsrbar_va); #endif mpc85xx_enable_l3_cache(); return (0); } void mpc85xx_mem_regions(platform_t plat, struct mem_region *phys, int *physsz, struct mem_region *avail, int *availsz) { ofw_mem_regions(phys, physsz, avail, availsz); } static u_long mpc85xx_timebase_freq(platform_t plat, struct cpuref *cpuref) { u_long ticks; phandle_t cpus, child; pcell_t freq; if (bootinfo != NULL) { if (bootinfo[0] == 1) { /* Backward compatibility. See 8-STABLE. */ ticks = bootinfo[3] >> 3; } else { /* Compatibility with Juniper's loader. */ ticks = bootinfo[5] >> 3; } } else ticks = 0; if ((cpus = OF_finddevice("/cpus")) == -1) goto out; if ((child = OF_child(cpus)) == 0) goto out; switch (OF_getproplen(child, "timebase-frequency")) { case 4: { uint32_t tbase; OF_getprop(child, "timebase-frequency", &tbase, sizeof(tbase)); ticks = tbase; return (ticks); } case 8: { uint64_t tbase; OF_getprop(child, "timebase-frequency", &tbase, sizeof(tbase)); ticks = tbase; return (ticks); } default: break; } freq = 0; if (OF_getprop(child, "bus-frequency", (void *)&freq, sizeof(freq)) <= 0) goto out; if (freq == 0) goto out; /* * Time Base and Decrementer are updated every 8 CCB bus clocks. * HID0[SEL_TBCLK] = 0 */ if (mpc85xx_is_qoriq()) ticks = freq / 32; else ticks = freq / 8; out: if (ticks <= 0) panic("Unable to determine timebase frequency!"); return (ticks); } static int mpc85xx_smp_first_cpu(platform_t plat, struct cpuref *cpuref) { cpu = 0; cpuref->cr_cpuid = cpu; cpuref->cr_hwref = cpuref->cr_cpuid; if (bootverbose) printf("powerpc_smp_first_cpu: cpuid %d\n", cpuref->cr_cpuid); cpu++; return (0); } static int mpc85xx_smp_next_cpu(platform_t plat, struct cpuref *cpuref) { if (cpu >= maxcpu) return (ENOENT); cpuref->cr_cpuid = cpu++; cpuref->cr_hwref = cpuref->cr_cpuid; if (bootverbose) printf("powerpc_smp_next_cpu: cpuid %d\n", cpuref->cr_cpuid); return (0); } static int mpc85xx_smp_get_bsp(platform_t plat, struct cpuref *cpuref) { cpuref->cr_cpuid = mfspr(SPR_PIR); cpuref->cr_hwref = cpuref->cr_cpuid; return (0); } #ifdef SMP static int mpc85xx_smp_start_cpu_epapr(platform_t plat, struct pcpu *pc) { vm_paddr_t rel_pa, bptr; volatile struct cpu_release *rel; vm_offset_t rel_va, rel_page; phandle_t node; int i; /* If we're calling this, the node already exists. */ node = OF_finddevice("/cpus"); for (i = 0, node = OF_child(node); i < pc->pc_cpuid; i++, node = OF_peer(node)) ; if (OF_getencprop(node, "cpu-release-addr", (pcell_t *)&rel_pa, sizeof(rel_pa)) == -1) { return (ENOENT); } rel_page = kva_alloc(PAGE_SIZE); if (rel_page == 0) return (ENOMEM); critical_enter(); rel_va = rel_page + (rel_pa & PAGE_MASK); pmap_kenter(rel_page, rel_pa & ~PAGE_MASK); rel = (struct cpu_release *)rel_va; bptr = ((vm_paddr_t)(uintptr_t)__boot_page - KERNBASE) + kernload; cpu_flush_dcache(__DEVOLATILE(struct cpu_release *,rel), sizeof(*rel)); rel->pir = pc->pc_cpuid; __asm __volatile("sync"); rel->entry_h = (bptr >> 32); rel->entry_l = bptr; __asm __volatile("sync"); cpu_flush_dcache(__DEVOLATILE(struct cpu_release *,rel), sizeof(*rel)); if (bootverbose) printf("Waking up CPU %d via CPU release page %p\n", pc->pc_cpuid, rel); critical_exit(); pmap_kremove(rel_page); kva_free(rel_page, PAGE_SIZE); return (0); } #endif static int mpc85xx_smp_start_cpu(platform_t plat, struct pcpu *pc) { #ifdef SMP vm_paddr_t bptr; uint32_t reg; int timeout; uintptr_t brr; int cpuid; int epapr_boot = 0; uint32_t tgt; if (mpc85xx_is_qoriq()) { reg = ccsr_read4(OCP85XX_COREDISR); cpuid = pc->pc_cpuid; if ((reg & (1 << cpuid)) != 0) { printf("%s: CPU %d is disabled!\n", __func__, pc->pc_cpuid); return (-1); } brr = OCP85XX_BRR; } else { brr = OCP85XX_EEBPCR; cpuid = pc->pc_cpuid + 24; } bp_kernload = kernload; /* * bp_kernload is in the boot page. Sync the cache because ePAPR * booting has the other core(s) already running. */ cpu_flush_dcache(&bp_kernload, sizeof(bp_kernload)); ap_pcpu = pc; __asm __volatile("msync; isync"); /* First try the ePAPR way. */ if (mpc85xx_smp_start_cpu_epapr(plat, pc) == 0) { epapr_boot = 1; goto spin_wait; } reg = ccsr_read4(brr); if ((reg & (1 << cpuid)) != 0) { printf("SMP: CPU %d already out of hold-off state!\n", pc->pc_cpuid); return (ENXIO); } /* Flush caches to have our changes hit DRAM. */ cpu_flush_dcache(__boot_page, 4096); bptr = ((vm_paddr_t)(uintptr_t)__boot_page - KERNBASE) + kernload; KASSERT((bptr & 0xfff) == 0, ("%s: boot page is not aligned (%#jx)", __func__, (uintmax_t)bptr)); if (mpc85xx_is_qoriq()) { /* * Read DDR controller configuration to select proper BPTR target ID. * * On P5020 bit 29 of DDR1_CS0_CONFIG enables DDR controllers * interleaving. If this bit is set, we have to use * OCP85XX_TGTIF_RAM_INTL as BPTR target ID. On other QorIQ DPAA SoCs, * this bit is reserved and always 0. */ reg = ccsr_read4(OCP85XX_DDR1_CS0_CONFIG); if (reg & (1 << 29)) tgt = OCP85XX_TGTIF_RAM_INTL; else tgt = OCP85XX_TGTIF_RAM1; /* * Set BSTR to the physical address of the boot page */ ccsr_write4(OCP85XX_BSTRH, bptr >> 32); ccsr_write4(OCP85XX_BSTRL, bptr); ccsr_write4(OCP85XX_BSTAR, OCP85XX_ENA_MASK | (tgt << OCP85XX_TRGT_SHIFT_QORIQ) | (ffsl(PAGE_SIZE) - 2)); /* Read back OCP85XX_BSTAR to synchronize write */ ccsr_read4(OCP85XX_BSTAR); /* * Enable and configure time base on new CPU. */ /* Set TB clock source to platform clock / 32 */ reg = ccsr_read4(CCSR_CTBCKSELR); ccsr_write4(CCSR_CTBCKSELR, reg & ~(1 << pc->pc_cpuid)); /* Enable TB */ reg = ccsr_read4(CCSR_CTBENR); ccsr_write4(CCSR_CTBENR, reg | (1 << pc->pc_cpuid)); } else { /* * Set BPTR to the physical address of the boot page */ bptr = (bptr >> 12) | 0x80000000u; ccsr_write4(OCP85XX_BPTR, bptr); __asm __volatile("isync; msync"); } /* * Release AP from hold-off state */ reg = ccsr_read4(brr); ccsr_write4(brr, reg | (1 << cpuid)); __asm __volatile("isync; msync"); spin_wait: timeout = 500; while (!pc->pc_awake && timeout--) DELAY(1000); /* wait 1ms */ /* * Disable boot page translation so that the 4K page at the default * address (= 0xfffff000) isn't permanently remapped and thus not * usable otherwise. */ if (!epapr_boot) { if (mpc85xx_is_qoriq()) ccsr_write4(OCP85XX_BSTAR, 0); else ccsr_write4(OCP85XX_BPTR, 0); __asm __volatile("isync; msync"); } if (!pc->pc_awake) panic("SMP: CPU %d didn't wake up.\n", pc->pc_cpuid); return ((pc->pc_awake) ? 0 : EBUSY); #else /* No SMP support */ return (ENXIO); #endif } static void mpc85xx_reset(platform_t plat) { /* * Try the dedicated reset register first. * If the SoC doesn't have one, we'll fall * back to using the debug control register. */ ccsr_write4(OCP85XX_RSTCR, 2); /* Clear DBCR0, disables debug interrupts and events. */ mtspr(SPR_DBCR0, 0); __asm __volatile("isync"); /* Enable Debug Interrupts in MSR. */ mtmsr(mfmsr() | PSL_DE); /* Enable debug interrupts and issue reset. */ mtspr(SPR_DBCR0, mfspr(SPR_DBCR0) | DBCR0_IDM | DBCR0_RST_SYSTEM); printf("Reset failed...\n"); while (1) ; } static void mpc85xx_idle(platform_t plat, int cpu) { uint32_t reg; if (mpc85xx_is_qoriq()) { /* * Base binutils doesn't know what the 'wait' instruction is, so * use the opcode encoding here. */ __asm __volatile("wrteei 1; .long 0x7c00007c"); } else { reg = mfmsr(); /* Freescale E500 core RM section 6.4.1. */ __asm __volatile("msync; mtmsr %0; isync" :: "r" (reg | PSL_WE)); } } static int mpc85xx_idle_wakeup(platform_t plat, int cpu) { return (0); }